Jenkinsfile.daily

// vim: ts=2:sw=2:expandtab:syntax=groovy
/* https://build.libelektra.org/job/elektra-jenkinsfile-daily/ */

// Set properties for buildjob
properties([
    // Discard all but the latest 30 logs
    buildDiscarder(logRotator(numToKeepStr: '30', artifactNumToKeepStr: '30')),

    // Don't run concurrently
    disableConcurrentBuilds()
])

// Stages
stage("Maintain Reference Repositories") {
    parallel generateRepositoryMaintanenceStages()
}

stage("Maintain docker nodes") {
    parallel generateDockerMaintanenceStages()
}

stage("Maintain docker repositories") {
    parallel generateDockerRepoMaintanenceStages()
}

stage("Cleanup Files") {
    parallel generateFileCleanupStages()
}

/**************/
/* Generators */
/**************/

/* Makes sure all 'gitmirror' nodes have up2date git repositories */
def generateRepositoryMaintanenceStages() {
    // Get all nodes with label 'gitmirror' and the master
    // DO NOT TAG master WITH 'gitmirror' as it will only show up as (null)
    def nodes = nodesByLabel label:"gitmirror"
    nodes.add("master")

    def repos = [
        [
            url: "https://github.com/ElektraInitiative/libelektra.git",
            dest: "libelektra"
        ]
    ]

    def tasks = [:]
    nodes.each { targetNode ->
        repos.each { targetRepo ->
            tasks << mirrorRepoOnNode(targetNode, targetRepo)
        }
    }
    return tasks
}

def mirrorRepoOnNode(targetNode, targetRepo) {
    def taskname = "git/${targetNode}/${targetRepo.dest}"
    return [(taskname): {
        stage(taskname) {
            node(targetNode) {
                syncRepository targetRepo
            }
        }
    }]
}

/* cleanup temp docker files on all docker nodes */
def generateDockerMaintanenceStages() {
    def nodes = nodesByLabel label:"docker"
    def tasks = [:]

    nodes.each { targetNode ->
        tasks << cleanupDocker(targetNode)
    }
    return tasks
}

def cleanupDocker(targetNode) {
    def taskname = "cleanupDocker/${targetNode}"
    return [(taskname): {
        stage(taskname) {
            node(targetNode) {
                dockerImages().each {
                    if (it ==~ /^hub.libelektra.org.*-release$/) {
                        // keep latest release and master image
                        retainNewestImages(it, 5, '', 'master\\|release')
                        retainNewestImages(it, 1, 'release', 'none')
                        retainNewestImages(it, 1, 'master', 'none')
                    } else {
                        retainNewestImages(it, 5)
                    }
                }
                sh "docker system prune -f"
            }
        }
    }]
}

/* Cleanup Docker registries */
def generateDockerRepoMaintanenceStages() {
    def nodes = nodesByLabel label:"docker&&registry"
    def tasks = [:]

    nodes.each { targetNode ->
        tasks << cleanupDockerRegistry(targetNode)
    }
    return tasks
}

def cleanupDockerRegistry(targetNode) {
    def taskname = "cleanupDockerRegistry/${targetNode}"
    return [(taskname): {
        stage(taskname) {
            node(targetNode) {
                withCredentials([usernameColonPassword(credentialsId:
                'docker-hub-elektra-jenkins', variable: 'AUTH')]) {
                    // Remove all but the latest 10 versions of each image in
                    // the registry
                    docker.image('anoxis/registry-cli')
                          .run('--rm', '-l $AUTH -r https://hub.libelektra.org --delete')
                    // remove all images with Elektra packages installed in the registry
                    docker.image('anoxis/registry-cli')
                          .run('--rm', '-l $AUTH -r https://hub.libelektra.org --delete-all --images-like ".*-installed$"')
                }
                def frontendFolder = "${env.HOME}/compose/frontend"
                dir(frontendFolder){
                    // Run garbage collect on the registry
                    sh 'COMPOSE_INTERACTIVE_NO_CLI=1 docker-compose run -T --rm registry bin/registry garbage-collect /etc/docker/registry/config.yml'
                }
            }
        }
    }]
}

def generateFileCleanupStages() {
    def tasks = [:]
    tasks << cleanupDoc('coverage/')
    tasks << cleanupDoc('api/pr/')
    tasks << cleanupAptlyDb()
    return tasks
}

def cleanupDoc(path) {
    def cleanupCommand = """
find /srv/libelektra/${path} \
    -mindepth 1 \
    -maxdepth 1 \
    -type d \
    -ctime +30 \
    -print0 \
    | xargs -r -0 rm -R
"""
    def taskname = "cleanup-${path}"
    return runCommandOnCommunity(taskname, cleanupCommand)
}

def cleanupAptlyDb() {
    def cleanupCommand = 'aptly db cleanup -verbose'
    return runCommandOnCommunity('cleanupAptlyDb', cleanupCommand)
}

/***********/
/* Helpers */
/***********/
def sshExec(remoteConfig, command) {
    sshPublisher(publishers: [
        sshPublisherDesc(
            configName: remoteConfig,
            transfers: [
                sshTransfer(
                    execCommand: command
                )
            ],
            verbose: true
        )
    ])
}

def runCommandOnCommunity(taskname, cleanupCommand) {
    return [(taskname): {
        stage(taskname) {
            node("master") {
                sshExec(
                    'doc.libelektra.org',
                    cleanupCommand
                )
            }
        }
    }]
}

def dockerImages() {
    def s = /docker images --format "{{.Repository}}" | awk '!seen[$0]++'/
    def r = sh returnStdout: true, script: s
    return r.tokenize('\n')
}

def retainNewestImages(image, n, withTag='', excludeTag='none') {
  sh """docker images ${image} --format \"{{.Repository}} {{.Tag}}\" \
| grep -v \"${excludeTag}\" \
| grep \"${withTag}\" \
| sort -k 2,2 \
| head -n -${n} \
| awk \'BEGIN{OFS=\":\"} {print \$1,\$2}' \
| xargs -r -n1 docker rmi -f"""
}

def syncRepository(Map repo) {
    dir("${env.HOME}/git_mirrors") {
        if(fileExists(repo.dest)) {
            echo "Found existing repository mirror"
            dir(repo.dest){
                sh "git fetch --prune"
            }
        } else {
            echo "Create new repository mirror"
            sh "git clone --mirror ${repo.url} ${repo.dest}"
        }
    }
}