master.cfg

# -*- python -*-
# ex: set filetype=python:

import json
import os

from buildbot.plugins import worker, changes, schedulers, secrets, steps, util

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

####### HELPER FUNCTIONS #######

def _path(name):
    return os.path.join(os.path.dirname(__file__), name)

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
# disable statistics submission
c['buildbotNetUsageData'] = None
c['secretsProviders'] = [secrets.SecretInAFile(dirname='secrets', strip=False)]

config = {
    'workersconfig' : _path('workers.json'),
    'htpasswdfile'  : _path('htpasswd'),
    'workerport'    : 9989,
    'httpport'      : 8010,
    'title'         : 'ConTeXt Build Farm',
    'titleURL'      : 'https://build.contextgarden.net/',
    'downloadURL'   : 'https://dl.contextgarden.net/build',
    'downloadPath'  : '/context/www/contextgarden.net/dl/build',
    'db_url'        : 'sqlite:///state.sqlite',
}
####### WORKERS

# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.

from builder_profiles import builder_profiles, builder_list

with open(config['workersconfig']) as f:
    workerdata = json.load(f)

# list of build workers
c['workers'] = [worker.Worker(w['name'], w['pwd']) for w in workerdata['workers']]

# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': config['workerport']}}

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot version of a python hello-world project.

# TeX Live
# - upstream svn
#   - svn://tug.org/texlive/trunk/Build/source
#   - svn://tug.org/texlive/branches/branch20XY/Build/source
# - git mirror
#   - https://github.com/TeX-Live/texlive-source.git
# LuaTeX
# - upstream git
#   - 'git': 'https://gitlab.lisn.upsaclay.fr/texlive/luatex.git',
# - git mirror
#   - https://github.com/TeX-Live/luatex
repositories = {
    'texlive': {
        'svn': 'svn://tug.org/texlive/',
        'git': 'https://github.com/TeX-Live/texlive-source.git',
        'branches': ['trunk', 'branches/branch2023', 'branches/branch2022', 'branches/branch2021', 'branches/branch2020', 'branches/branch2019'],
        'default_branch': 'trunk',
    },
    'luatex': {
        #'git': 'https://github.com/TeX-Live/luatex.git',
        'git': 'https://gitlab.lisn.upsaclay.fr/texlive/luatex.git',
        'branches': ['master'],
        'default_branch': 'master',
    },
    'luametatex': {
        'git': 'https://github.com/contextgarden/luametatex.git',
        'branches': ['main', 'work'],
        'default_branch': 'main',
    },
    'pplib': {
        'git': 'https://github.com/contextgarden/pplib.git',
        'branches': ['master'],
        'default_branch': 'master',
    },
    'lmtx_libs': {
        'git': 'https://github.com/contextgarden/luametatex-optional-libraries.git',
        'branches': ['main'],
        'default_branch': 'main',
    },
}

# TODO: maybe define a subclass of SVNPoller which redefines split_path?
def split_texlive_branches(path):
    pieces = path.split('/')
    if pieces[0] == 'trunk':
        branch = None
        pieces.pop(0)
    elif pieces[0] == 'branches':
        pieces.pop(0)
        branch = 'branches/{}'.format(pieces.pop(0))
    else:
        return None
    if len(pieces) > 2 and pieces[0] == 'Build' and pieces[1] == 'source':
        return dict(branch = branch, path = '/'.join(pieces[2:]))
    else:
        return None

c['change_source'] = []
c['change_source'].append(
    changes.GitPoller(
        repourl    = repositories['luatex']['git'],
        branches   = repositories['luatex']['branches'],
        category   = 'luatex',
        project    = 'luatex',
        workdir    = 'sources_luatex',
        pollInterval = 300,
    )
)
c['change_source'].append(
    changes.SVNPoller(
        repourl      = repositories['texlive']['svn'],
        split_file   = split_texlive_branches,
        category     = 'texlive',
        project      = 'texlive',
        pollInterval = 300,
    )
)
c['change_source'].append(
    changes.GitPoller(
        repourl    = repositories['pplib']['git'],
        branches   = repositories['pplib']['branches'],
        category   = 'pplib',
        project    = 'pplib',
        workdir    = 'sources_pplib',
        pollInterval = 300,
    )
)
c['change_source'].append(
    changes.GitPoller(
        repourl  = repositories['luametatex']['git'],
        branches = repositories['luametatex']['branches'],
        category = 'luametatex',
        project  = 'luametatex',
        workdir  = 'sources_luametatex',
        pollInterval = 120,
        pollAtLaunch = True,
    )
)
c['change_source'].append(
    changes.GitPoller(
        repourl  = repositories['lmtx_libs']['git'],
        branches = repositories['lmtx_libs']['branches'],
        category = 'lmtx_libs',
        project  = 'lmtx_libs',
        workdir  = 'sources_lmtx_libs',
        pollInterval = 300,
    )
)

c['revlink'] = util.RevlinkMatch(
    [r'svn://tug.org/texlive(.*)'],
    r'http://tug.org/svn/texlive?view=revision&revision=%s'
)

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  In this
# case, just kick off a 'runtests' build

build_components = ['texlive', 'luatex', 'pplib', 'luametatex', 'lmtx_libs']

builders = {}
builders_all = []
for component in build_components:
    builders[component] = []
    for builder in builder_list:
        if builder.build[component]:
            builders[component].append('{}.{}'.format(component, builder.name))
    builders_all += builders[component]

c['schedulers'] = []
for component in build_components:
    if component in ['texlive']:
        stableTimer = 3600
    elif component in ['luametatex']:
        stableTimer = 60
    else:
        stableTimer = 300
    c['schedulers'].append(
        schedulers.ForceScheduler(
            name         = 'build-{}'.format(component),
            builderNames = builders[component],
        )
    )
    c['schedulers'].append(
        schedulers.SingleBranchScheduler(
            name            = 'commit-{}'.format(component),
            change_filter   = util.ChangeFilter(project = component),
            builderNames    = builders[component],
            #change_filter   = util.ChangeFilter(branch = repositories[component]['default_branch']),
            treeStableTimer = stableTimer,
        )
    )
    print('Adding SingleBranchScheduler for {}. The scheduler should trigger {} components:'.format(component, len(builders[component])))
    for builder in builders[component]:
        print('- builder: {}'.format(builder))

####### BUILDERS

class CustomFileUpload(steps.MultipleFileUpload):
    def uploadDone(self, result, source, masterdest):
        if self.url:
            basename = os.path.basename(source)
            self.addURL(basename, '{}/{}'.format(self.url, basename))

    def allUploadsDone(self, result, sources, masterdest):
        if self.url:
            numFiles = len(sources)
            if numFiles:
                self.addURL('{} ({:d} files)'.format(self.url, numFiles), self.url)

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them.  Note that any particular build will
# only take place on one worker.

@util.renderer
def get_luametatex_path(props, prefix, fileDest):
    branch = props.getProperty('branch')
    path = [prefix, 'luametatex', branch, fileDest]
    return '/'.join(path)

@util.renderer
def get_texlive_path(props, prefix, fileDest):
    path = [prefix, 'texlive']
    branch = props.getProperty('branch')
    if not branch in ['trunk', '', None]:
        path.append(branch)
    path.append(fileDest)
    return '/'.join(path)

def make_factory_texlive(builder):
    needs_proxy = (builder.code == 'csw')

    build_env                  = builder.env
    build_env['TL_MAKE']       = builder.cmd_make
    build_command              = ['./Build']
    if builder.platform == 'openbsd':
        build_env['ICU_LIBS_EXTRA'] = '-lpthread'
    # On Solaris we want to avoid dependency on non-standard library iconv
    # On FreeBSD there are two iconv libraries which kind-of-conflict in this setup
    if builder.platform == 'solaris' or builder.platform == 'freebsd':
        build_command.append('--without-iconv')
    if 'darwin10' in builder.name:
        build_command.append('--without-x')

    f = util.BuildFactory()
    f.addStep(
        steps.ShellCommand(name='Cleanup', command=['svn', 'cleanup'])
    )
    f.addStep(
        steps.SVN(
            repourl         = util.Interpolate("svn{}://tug.org/texlive/%(src::branch:~trunk)s/Build/source".format('+proxy' if needs_proxy else '')),
            extra_args      = ['--config-option=config:miscellany:use-commit-times=yes'],
            method          = 'clobber',
            #mode           = 'full',
        )
    )
    f.addStep(
        steps.Compile(
            command         = build_command,
            haltOnFailure   = True,
            env             = {**build_env, 'TL_MAKE_FLAGS': 'check_target=skip -j 2'},
            logfiles        = {'build info': './Work/buildinfo.log', 'build environment': './Work/buildenv.log'},
            lazylogfiles    = True,
            description     = util.Interpolate("building %(src::branch:~trunk)s"),
            descriptionDone = util.Interpolate("build %(src::branch:~trunk)s"),
        )
    )
    f.addStep(
        steps.Compile(
            name            = 'test',
            command         = ['./Build', '--no-clean'],
            haltOnFailure   = False,
            flunkOnFailure  = False,
            warnOnFailure   = True,
            env             = {**build_env, 'TL_TARGET': 'check', 'TL_CONFIGURE': 'true'},
            description     = 'testing',
            descriptionDone = 'test',
        )
    )
    if builder.upload:
        fileDest = '{}.tar.xz'.format(builder.tlname)
        f.addStep(
            steps.ShellCommand(
                name          = 'compress',
                #command = ['mv', 'inst/bin/*', 'inst/bin/{}'.format(builder.tlname)]),
                command       = [builder.cmd_tar, 'cvJf', 'inst.tar.xz', 'bin'],
                haltOnFailure = True,
                description   = 'compressing',
                workdir       = os.path.join('build', 'inst'),
            )
        )
        f.addStep(
            steps.FileUpload(
                workersrc  = os.path.join('inst', 'inst.tar.xz'),
                masterdest = get_texlive_path.withArgs(config['downloadPath'], fileDest),
                mode       = 0o644,
                url        = get_texlive_path.withArgs(config['downloadURL'], fileDest),
            )
        )

    return f



def make_factory_metatex(builder):
    f = util.BuildFactory()
    binary_filename = 'luametatex'
    binary_location = ''
    if builder.platform in ['mingw', 'windows']:
        binary_filename = 'luametatex.exe'
    if builder.platform == 'windows':
        binary_source = os.path.join('Release', binary_filename)
    else:
        binary_source = binary_filename

    f.addStep(
        steps.Git(
            repourl = repositories['luametatex']['git'],
            mode = 'full',
            method = 'copy',
        )
    )
    f.addStep(
        steps.CMake(
            generator   = builder.cmake_generator,
            env         = builder.env,
            path        = '.',
            definitions = builder.cmake_defs,
        )
    )
    f.addStep(
        steps.Compile(
            command        = ['cmake', '--build', '.', '--config', 'Release'],
            haltOnFailure  = True,
            warnOnWarnings = True,
            env            = builder.env,
        )
    )
    if builder.upload or True:
        fileDest = '{}/{}'.format(builder.tlname, binary_filename)
        f.addStep(
            steps.FileUpload(
                workersrc  = binary_source,
                masterdest = get_luametatex_path.withArgs(config['downloadPath'], fileDest),
                url        = get_luametatex_path.withArgs(config['downloadURL'], fileDest),
                mode       = 0o755,
            )
        )
    return f

def make_factory_lmtx_libs(builder):
    f = util.BuildFactory()

    f.addStep(
        steps.Git(
            repourl    = repositories['lmtx_libs']['git'],
            branch     = repositories['lmtx_libs']['default_branch'],
            method     = 'clobber',
            submodules = True,
        )
    )
    f.addStep(
        steps.CMake(
            generator   = builder.cmake_generator,
            env         = builder.env,
            path        = '.',
            definitions = builder.cmake_defs,
        )
    )
    f.addStep(
        steps.Compile(
            command        = ['cmake', '--build', '.', '--config', 'Release'],
            haltOnFailure  = True,
            warnOnWarnings = True,
            env            = builder.env,
        )
    )
    return f

def make_factory_pplib(builder):
    build_env = builder.env

    f = util.BuildFactory()
    f.addStep(
        steps.Git(
            repourl    = repositories['pplib']['git'],
            branch     = repositories['pplib']['default_branch'],
            method     = 'clobber',
            submodules = True,
        )
    )
    f.addStep(
        steps.CMake(
            generator   = builder.cmake_generator,
            env         = builder.env,
            path        = '.',
            definitions = builder.cmake_defs,
        )
    )
    f.addStep(
        steps.Compile(
            command        = ['cmake', '--build', '.', '--config', 'Release'],
            haltOnFailure  = True,
            warnOnWarnings = True,
            env            = builder.env,
        )
    )
    return f

def make_factory_luatex(builder):
    luatex_build_env     = builder.env
    luatex_build_command = ['./build.sh', '--parallel', '--luahb']
    build_jit = builder.arch not in ['ppc', 'sparc']
    if build_jit:
        luatex_build_command.append('--jit')
        luatex_build_command.append('--jithb')
    if builder.arch == 'aarch64':
        luatex_build_command.append('--tlopt=--enable-arm-neon=yes')
    elif builder.arch == 'arm64':
        luatex_build_command.append('--tlopt=--enable-arm-neon=no')

    exe_suffix = ''
    builddir   = 'build'
    if builder.platform == 'mingw':
        exe_suffix = '.exe'
        if builder.arch == 'i386':
            luatex_build_command.append('--mingw32')
            builddir = 'build-windows32'
        else:
            luatex_build_command.append('--mingw64')
            builddir = 'build-windows64'

    if build_jit:
        binaries = ['luatex', 'luahbtex', 'luajittex', 'luajithbtex']
    else:
        binaries = ['luatex', 'luahbtex']
    files_to_upload = []
    for f in binaries:
        files_to_upload.append('{}/texk/web2c/{}{}'.format(builddir, f, exe_suffix))

    f = util.BuildFactory()
    f.useProgress = True
    f.addStep(
        steps.Git(
            repourl = repositories['luatex']['git'],
            timeout = 3600
        )
    )
    f.addStep(
        steps.ShellSequence(
            commands = [
                util.ShellArg(command = ['find', '.', '-name', '*.info', '-exec', 'touch', '{}', '+'], logname = 'touch info files'),
                util.ShellArg(command = ['touch', './source/texk/web2c/web2c/web2c-lexer.c'], logname = 'touch web2c-lexer.c'),
                util.ShellArg(command = ['touch', './source/texk/web2c/web2c/web2c-parser.c'], logname = 'touch web2c-parser.c'),
                util.ShellArg(command = ['touch', './source/texk/web2c/web2c/web2c-parser.h'], logname = 'touch web2c-parser.h'),
            ],
            description     = "touching files",
            descriptionDone = "touched files",
        )
    )
    f.addStep(
        steps.Compile(
            command         = luatex_build_command,
            haltOnFailure   = True,
            env             = luatex_build_env,
            description     = util.Interpolate("building %(src::branch:~master)s"),
            descriptionDone = util.Interpolate("build %(src::branch:~master)s")
        )
    )

    # upload it
    if builder.upload:
        for filename in files_to_upload:
            basename = os.path.basename(filename)
            destpath = os.path.join('luatex', builder.tlname, basename)
            f.addStep(
                steps.FileUpload(
                    workersrc       = filename,
                    masterdest      = os.path.join(config['downloadPath'], destpath),
                    url             = '{}/{}'.format(config['downloadURL'], destpath),
                    name            = 'upload_{}'.format(basename),
                    description     = 'upload {}'.format(basename),
                    descriptionDone = 'uploaded {}'.format(basename),
                    mode            = 0o755,
                    doStepIf        = lambda step: step.getProperty('branch') in ['master', '', None],
                )
            )
    if builder.upload and False:
        f.addStep(
            #steps.MultipleFileUpload(
            CustomFileUpload(
                workersrcs = files_to_upload,
                masterdest = os.path.join(config['downloadPath'], 'luatex', builder.tlname),
                url        = '{}/{}/{}'.format(config['downloadURL'], 'luatex', builder.tlname),
                keepstamp  = True,
                mode       = 0o755,
                doStepIf   = lambda step: step.getProperty('branch') in ['trunk', '', None],
            )
        )

    return f

c['builders'] = []
for builder in builder_list:
    tags = ['os.{}'.format(builder.platform)]
    if builder.build['texlive']:
        c['builders'].append(
            util.BuilderConfig(
                name             = 'texlive.{}'.format(builder.name),
                workernames      = [builder.worker],
                factory          = make_factory_texlive(builder),
                workerbuilddir   = 'texlive/{}'.format(builder.name),
                tags             = ['c.texlive'] + tags,
                collapseRequests = True,
                description      = 'Build TeX Live',
            )
        )
    if builder.build['luatex']:
        c['builders'].append(
            util.BuilderConfig(
                name             = 'luatex.{}'.format(builder.name),
                workernames      = [builder.worker],
                factory          = make_factory_luatex(builder), # TODO: why a function to make the factory?
                # Specifies the name of a subdirectory of the master’s basedir
                # in which everything related to this builder will be stored.
                # builddir       = ... # TODO
                # Specifies the name of a subdirectory (under the worker’s configured base directory)
                # in which everything related to this builder will be placed on the worker.
                workerbuilddir   = 'luatex/{}'.format(builder.name),
                tags             = ['c.luatex'] + tags,
                collapseRequests = True,
                # properties     = ... # TODO
                description      = 'Build LuaTeX',
            )
        )
    if builder.build['luametatex']:
        c['builders'].append(
            util.BuilderConfig(
                name             = 'luametatex.{}'.format(builder.name),
                workernames      = [builder.worker],
                factory          = make_factory_metatex(builder),
                workerbuilddir   = 'luametatex/{}'.format(builder.name),
                tags             = ['c.luametatex'] + tags,
                collapseRequests = True,
                description      = 'Build LuaMetaTeX',
        )
    )
    if builder.build['lmtx_libs']:
        c['builders'].append(
            util.BuilderConfig(
                name             = 'lmtx_libs.{}'.format(builder.name),
                workernames      = [builder.worker],
                factory          = make_factory_lmtx_libs(builder),
                workerbuilddir   = 'lmtx_libs/{}'.format(builder.name),
                tags             = ['c.libs'] + tags,
                collapseRequests = True,
                description      = 'Build optional libraries',
            )
        )
    if builder.build['pplib']:
        c['builders'].append(
            util.BuilderConfig(
                name             = 'pplib.{}'.format(builder.name),
                workernames      = [builder.worker],
                factory          = make_factory_pplib(builder),
                workerbuilddir   = 'pplib/{}'.format(builder.name),
                tags             = ['c.pplib'] + tags,
                collapseRequests = True,
                description      = 'Build pplib',
            )
        )

####### BUILDBOT SERVICES

# 'services' is a list of BuildbotService items like reporter targets. The
# status of each build will be pushed to these targets. buildbot/reporters/*.py
# has a variety to choose from, like IRC bots.

c['services'] = []

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').

c['title']    = config['title']
c['titleURL'] = config['titleURL']

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.

c['buildbotURL'] = config['titleURL']

# minimalistic config to activate new web UI
c['www'] = dict(
    port    = config['httpport'],
    plugins = dict(
        waterfall_view = {},
        console_view   = {},
        grid_view      = {},
    )
)
c['www']['ui_default_config'] = {
    #'Waterfall.scaling_waterfall': 0.3,
    'Waterfall.show_builders_without_builds': True,
}

# authentication
authz = util.Authz(
    allowRules = [
        util.AnyControlEndpointMatcher(role = 'admins'),
    ],
    roleMatchers = [
        util.RolesFromUsername(
            roles     = ['admins'],
            usernames = [],
        )
    ]
)
c['www']['auth']  = util.HTPasswdAuth(config['htpasswdfile'])
c['www']['authz'] = authz

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url': config['db_url'],
}