Background API

.github/workflows/ci.yml

@@ -0,0 +1,83 @@
+# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml
+
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - 'main'
+
+defaults:
+  run:
+    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail -x {0}
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  build:
+    name: test
+    runs-on: ubuntu-18.04
+    continue-on-error: ${{ matrix.allow_failure }}
+    strategy:
+      fail-fast: true
+      matrix:
+        go: [1.14, 1.15, 1.16]
+        run_long_tests: [no]
+        allow_failure: [false]
+        include:
+          # Disable this for now as it's not clear how to select the latest master branch
+          # version of go using actions/setup-go@v2.
+          # - go: master
+          #   run_long_tests: no
+          #   allow_failure: true
+          - go: 1.17
+            run_long_tests: yes
+            allow_failure: false
+
+    env:
+      NO_FTP_ACCESS: yes
+      BOTO_CONFIG: /dev/null
+      GO111MODULE: "on"
+      GOPROXY: "https://proxy.golang.org"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 2.7.17
+
+      - name: Install O/S packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y python-virtualenv graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
+
+      - name: Install Python packages
+        run: |
+          pip install six packaging appdirs
+          pip install -U pip setuptools
+          pip install -r system/requirements.txt
+
+      - name: Get aptly version
+        run: |
+           make version
+
+      - name: Make prepare
+        run: |
+          make prepare
+
+      - name: Make
+        env:
+          RUN_LONG_TESTS: ${{ matrix.run_long_tests }}
+        run: |
+          make

.gitignore

@@ -41,3 +41,4 @@ build/
 
 pgp/keyrings/aptly2*.gpg
 pgp/keyrings/aptly2*.gpg~
+pgp/keyrings/.#*

.golangci.yml

@@ -3,20 +3,17 @@ run:
 
 
 linters:
-  enable-all: false
   disable-all: true
   enable:
-   - govet
-   - golint
-   - gofmt
    - deadcode
+   - goconst
+   - gofmt
    - goimports
-   - misspell
+   - govet
    - ineffassign
+   - misspell
+   - revive
    - staticcheck
-   - varcheck
    - structcheck
-   - maligned
+   - varcheck
    - vetshadow
-   - goconst
-   - interfacer

AUTHORS

@@ -41,3 +41,6 @@ List of contributors, in chronological order:
 * Raul Benencia (https://github.com/rul)
 * Don Kuntz (https://github.com/dkuntz2)
 * Joshua Colson (https://github.com/freakinhippie)
+* Andre Roth (https://github.com/neolynx)
+* Lorenzo Bolla (https://github.com/lbolla)
+* Benj Fassbind (https://github.com/randombenj)

Makefile

@@ -14,7 +14,7 @@ RUN_LONG_TESTS?=yes
 all: modules test bench check system-test
 
 prepare:
-	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.19.1
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
 
 modules:
 	go mod download
@@ -79,3 +79,4 @@ version:
 	@echo $(VERSION)
 
 .PHONY: man modules version release goxc
+

README.rst

@@ -64,7 +64,7 @@ If you would like to use nightly builds (unstable), please use following reposit
 
 Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.11+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.14+ required)::
 
     git clone https://github.com/aptly-dev/aptly
     cd aptly

api/api.go

@@ -3,12 +3,16 @@ package api
 
 import (
 	"fmt"
+	"log"
+	"net/http"
 	"sort"
-	"time"
+	"strconv"
+	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
 
@@ -35,49 +39,14 @@ type dbRequest struct {
 	err  chan<- error
 }
 
-// Flushes all collections which cache in-memory objects
-func flushColections() {
-	// lock everything to eliminate in-progress calls
-	r := context.CollectionFactory().RemoteRepoCollection()
-	r.Lock()
-	defer r.Unlock()
-
-	l := context.CollectionFactory().LocalRepoCollection()
-	l.Lock()
-	defer l.Unlock()
-
-	s := context.CollectionFactory().SnapshotCollection()
-	s.Lock()
-	defer s.Unlock()
-
-	p := context.CollectionFactory().PublishedRepoCollection()
-	p.Lock()
-	defer p.Unlock()
-
-	// all collections locked, flush them
-	context.CollectionFactory().Flush()
-}
-
-// Periodically flushes CollectionFactory to free up memory used by
-// collections, flushing caches.
-//
-// Should be run in goroutine!
-func cacheFlusher() {
-	ticker := time.Tick(15 * time.Minute)
-
-	for {
-		<-ticker
-
-		flushColections()
-	}
-}
+var dbRequests chan dbRequest
 
 // Acquire database lock and release it when not needed anymore.
 //
 // Should be run in a goroutine!
-func acquireDatabase(requests <-chan dbRequest) {
+func acquireDatabase() {
 	clients := 0
-	for request := range requests {
+	for request := range dbRequests {
 		var err error
 
 		switch request.kind {
@@ -94,7 +63,6 @@ func acquireDatabase(requests <-chan dbRequest) {
 		case releasedb:
 			clients--
 			if clients == 0 {
-				flushColections()
 				err = context.CloseDatabase()
 			} else {
 				err = nil
@@ -105,12 +73,100 @@ func acquireDatabase(requests <-chan dbRequest) {
 	}
 }
 
+// Should be called before database access is needed in any api call.
+// Happens per default for each api call. It is important that you run
+// runTaskInBackground to run a task which accquire database.
+// Important do not forget to defer to releaseDatabaseConnection
+func acquireDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{acquiredb, errCh}
+
+	return <-errCh
+}
+
+// Release database connection when not needed anymore
+func releaseDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{releasedb, errCh}
+	return <-errCh
+}
+
+// runs tasks in background. Acquires database connection first.
+func runTaskInBackground(name string, resources []string, proc task.Process) (task.Task, *task.ResourceConflictError) {
+	return context.TaskList().RunTaskInBackground(name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := acquireDatabaseConnection()
+
+		if err != nil {
+			return nil, err
+		}
+
+		defer releaseDatabaseConnection()
+		return proc(out, detail)
+	})
+}
+
+func truthy(value interface{}) bool {
+	if value == nil {
+		return false
+	}
+	switch value.(type) {
+	case string:
+		switch strings.ToLower(value.(string)) {
+		case "n", "no", "f", "false", "0", "off":
+			return false
+		default:
+			return true
+		}
+	case int:
+		return !(value.(int) == 0)
+	case bool:
+		return value.(bool)
+	}
+	return true
+}
+
+func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, proc task.Process) {
+	// Run this task in background if configured globally or per-request
+	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
+	if background {
+		log.Println("Executing task asynchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			c.AbortWithError(409, conflictErr)
+			return
+		}
+		c.JSON(202, task)
+	} else {
+		log.Println("Executing task synchronously")
+		out := context.Progress()
+		detail := task.Detail{}
+		retValue, err := proc(out, &detail)
+		if err != nil {
+			c.AbortWithError(retValue.Code, err)
+			return
+		}
+		if retValue != nil {
+			c.JSON(retValue.Code, retValue.Value)
+		} else {
+			c.JSON(http.StatusOK, nil)
+		}
+	}
+}
+
 // Common piece of code to show list of packages,
 // with searching & details if requested
-func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
+func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) {
 	result := []*deb.Package{}
 
-	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
 		c.AbortWithError(404, err)
 		return