Rewrite queue

cmd/hook.go

@@ -18,7 +18,6 @@ import (
 	"code.gitea.io/gitea/modules/private"
 	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/util"
 
 	"github.com/urfave/cli"
 )
@@ -141,7 +140,7 @@ func (d *delayWriter) Close() error {
 	if d == nil {
 		return nil
 	}
-	stopped := util.StopTimer(d.timer)
+	stopped := d.timer.Stop()
 	if stopped || d.buf == nil {
 		return nil
 	}

custom/conf/app.example.ini

@@ -926,12 +926,6 @@ ROUTER = console
 ;; Global limit of repositories per user, applied at creation time. -1 means no limit
 ;MAX_CREATION_LIMIT = -1
 ;;
-;; Mirror sync queue length, increase if mirror syncing starts hanging (DEPRECATED: please use [queue.mirror] LENGTH instead)
-;MIRROR_QUEUE_LENGTH = 1000
-;;
-;; Patch test queue length, increase if pull request patch testing starts hanging (DEPRECATED: please use [queue.pr_patch_checker] LENGTH instead)
-;PULL_REQUEST_QUEUE_LENGTH = 1000
-;;
 ;; Preferred Licenses to place at the top of the List
 ;; The name here must match the filename in options/license or custom/options/license
 ;PREFERRED_LICENSES = Apache License 2.0,MIT License
@@ -1376,22 +1370,6 @@ ROUTER = console
 ;; Set to -1 to disable timeout.
 ;STARTUP_TIMEOUT = 30s
 ;;
-;; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue (deprecated - use [queue.issue_indexer])
-;ISSUE_INDEXER_QUEUE_TYPE = levelqueue; **DEPRECATED** use settings in `[queue.issue_indexer]`.
-;;
-;; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the path where the queue will be saved.
-;; This can be overridden by `ISSUE_INDEXER_QUEUE_CONN_STR`.
-;; default is queues/common
-;ISSUE_INDEXER_QUEUE_DIR = queues/common; **DEPRECATED** use settings in `[queue.issue_indexer]`. Relative paths will be made absolute against `%(APP_DATA_PATH)s`.
-;;
-;; When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string.
-;; When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of
-;; the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`.
-;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"; **DEPRECATED** use settings in `[queue.issue_indexer]`.
-;;
-;; Batch queue number, default is 20
-;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Repository Indexer settings
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1418,8 +1396,6 @@ ROUTER = console
 ;; A comma separated list of glob patterns to exclude from the index; ; default is empty
 ;REPO_INDEXER_EXCLUDE =
 ;;
-;;
-;UPDATE_BUFFER_LEN = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 ;MAX_FILE_SIZE = 1048576
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1449,37 +1425,16 @@ ROUTER = console
 ;; Connection string for redis queues this will store the redis connection string.
 ;; When `TYPE` is `persistable-channel`, this provides a directory for the underlying leveldb
 ;; or additional options of the form `leveldb://path/to/db?option=value&....`, and will override `DATADIR`.
-;CONN_STR = "addrs=127.0.0.1:6379 db=0"
+;CONN_STR = "redis://127.0.0.1:6379/0"
 ;;
 ;; Provides the suffix of the default redis/disk queue name - specific queues can be overridden within in their [queue.name] sections.
 ;QUEUE_NAME = "_queue"
 ;;
 ;; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overridden within in their [queue.name] sections.
 ;SET_NAME = "_unique"
 ;;
-;; If the queue cannot be created at startup - level queues may need a timeout at startup - wrap the queue:
-;WRAP_IF_NECESSARY = true
-;;
-;; Attempt to create the wrapped queue at max
-;MAX_ATTEMPTS = 10
-;;
-;; Timeout queue creation
-;TIMEOUT = 15m30s
-;;
-;; Create a pool with this many workers
-;WORKERS = 0
-;;
 ;; Dynamically scale the worker pool to at this many workers
 ;MAX_WORKERS = 10
-;;
-;; Add boost workers when the queue blocks for BLOCK_TIMEOUT
-;BLOCK_TIMEOUT = 1s
-;;
-;; Remove the boost workers after BOOST_TIMEOUT
-;BOOST_TIMEOUT = 5m
-;;
-;; During a boost add BOOST_WORKERS
-;BOOST_WORKERS = 1
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docs/content/doc/administration/config-cheat-sheet.en-us.md

@@ -89,10 +89,6 @@ In addition there is _`StaticRootPath`_ which can be set as a built-in at build
 - `DEFAULT_PUSH_CREATE_PRIVATE`: **true**: Default private when creating a new repository with push-to-create.
 - `MAX_CREATION_LIMIT`: **-1**: Global maximum creation limit of repositories per user,
    `-1` means no limit.
-- `PULL_REQUEST_QUEUE_LENGTH`: **1000**: Length of pull request patch test queue, make it. **DEPRECATED** use `LENGTH` in `[queue.pr_patch_checker]`.
-   as large as possible. Use caution when editing this value.
-- `MIRROR_QUEUE_LENGTH`: **1000**: Patch test queue length, increase if pull request patch
-   testing starts hanging. **DEPRECATED** use `LENGTH` in `[queue.mirror]`.
 - `PREFERRED_LICENSES`: **Apache License 2.0,MIT License**: Preferred Licenses to place at
    the top of the list. Name must match file name in options/license or custom/options/license.
 - `DISABLE_HTTP_GIT`: **false**: Disable the ability to interact with repositories over the
@@ -466,10 +462,6 @@ relation to port exhaustion.
 - `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
 - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_.
 - The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility:
-- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
-- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. **DEPRECATED** use settings in `[queue.issue_indexer]`. Relative paths will be made absolute against `%(APP_DATA_PATH)s`.
-- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
-- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. **DEPRECATED** use settings in `[queue.issue_indexer]`.
 
 - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
 - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`.
@@ -480,7 +472,6 @@ relation to port exhaustion.
 - `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files.
 - `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`.
 - `REPO_INDEXER_EXCLUDE_VENDORED`: **true**: Exclude vendored files from index.
-- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. **DEPRECATED** use settings in `[queue.issue_indexer]`.
 - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
 - `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to -1 to never timeout.
 
@@ -496,15 +487,7 @@ Configuration at `[queue]` will set defaults for queues with overrides for indiv
 - `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overridden in the specific `queue.name` section.
 - `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to
  **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section.
-- `WRAP_IF_NECESSARY`: **true**: Will wrap queues with a timeoutable queue if the selected queue is not ready to be created - (Only relevant for the level queue.)
-- `MAX_ATTEMPTS`: **10**: Maximum number of attempts to create the wrapped queue
-- `TIMEOUT`: **GRACEFUL_HAMMER_TIME + 30s**: Timeout the creation of the wrapped queue if it takes longer than this to create.
-- Queues by default come with a dynamically scaling worker pool. The following settings configure this:
-- `WORKERS`: **0**: Number of initial workers for the queue.
 - `MAX_WORKERS`: **10**: Maximum number of worker go-routines for the queue.
-- `BLOCK_TIMEOUT`: **1s**: If the queue blocks for this time, boost the number of workers - the `BLOCK_TIMEOUT` will then be doubled before boosting again whilst the boost is ongoing.
-- `BOOST_TIMEOUT`: **5m**: Boost workers will timeout after this long.
-- `BOOST_WORKERS`: **1**: This many workers will be added to the worker pool if there is a boost.
 
 Gitea creates the following non-unique queues:
 

models/migrations/base/tests.go

@@ -11,14 +11,14 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
-	"strings"
 	"testing"
 
 	"code.gitea.io/gitea/models/unittest"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/testlogger"
 
 	"github.com/stretchr/testify/assert"
 	"xorm.io/xorm"
@@ -32,7 +32,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
 	t.Helper()
 	ourSkip := 2
 	ourSkip += skip
-	deferFn := PrintCurrentTest(t, ourSkip)
+	deferFn := testlogger.PrintCurrentTest(t, ourSkip)
 	assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
 	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
@@ -110,9 +110,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
 }
 
 func MainTest(m *testing.M) {
-	log.Register("test", NewTestLogger)
-	_, filename, _, _ := runtime.Caller(0)
-	prefix = strings.TrimSuffix(filename, "tests/testlogger.go")
+	log.Register("test", testlogger.NewTestLogger)
 
 	giteaRoot := base.SetupGiteaRoot()
 	if giteaRoot == "" {

models/unittest/testdb.go

@@ -202,6 +202,9 @@ type FixturesOptions struct {
 func CreateTestEngine(opts FixturesOptions) error {
 	x, err := xorm.NewEngine("sqlite3", "file::memory:?cache=shared&_txlock=immediate")
 	if err != nil {
+		if strings.Contains(err.Error(), "unknown driver") {
+			return fmt.Errorf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify%s%w`, "\n", err)
+		}
 		return err
 	}
 	x.SetMapper(names.GonicMapper{})

modules/indexer/code/bleve.go

@@ -273,10 +273,6 @@ func (b *BleveIndexer) Close() {
 	log.Info("PID: %d Repository Indexer closed", os.Getpid())
 }
 
-// SetAvailabilityChangeCallback does nothing
-func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
-}
-
 // Ping does nothing
 func (b *BleveIndexer) Ping() bool {
 	return true