feat: server improvements, validations for data

.gitignore

@@ -16,4 +16,5 @@
 
 barrel*.lock
 barrel*.db
-barrel.hints
+barrel.hints
+config.toml

Makefile

@@ -0,0 +1,30 @@
+LAST_COMMIT := $(shell git rev-parse --short HEAD)
+LAST_COMMIT_DATE := $(shell git show -s --format=%ci ${LAST_COMMIT})
+VERSION := $(shell git describe --abbrev=1)
+BUILDSTR := ${VERSION} (build "\\\#"${LAST_COMMIT} $(shell date '+%Y-%m-%d %H:%M:%S'))
+
+BIN := ./bin/barreldb.bin
+
+.PHONY: build
+build: $(BIN)
+
+$(BIN): $(shell find . -type f -name "*.go")
+	CGO_ENABLED=0 go build -o ${BIN} -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'" ./cmd/server/*.go
+
+.PHONY: run
+run:
+	CGO_ENABLED=0 go run -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'" ./cmd/server --config=cmd/server/config.toml
+
+.PHONY: test
+test:
+	go test ./...
+
+# Use goreleaser to do a dry run producing local builds.
+.PHONY: release-dry
+release-dry:
+	goreleaser --parallelism 1 --rm-dist --snapshot --skip-validate --skip-publish
+
+# Use goreleaser to build production releases and publish them.
+.PHONY: release
+release:
+	goreleaser --parallelism 1 --rm-dist --skip-validate

TODO.md

@@ -23,17 +23,13 @@ Create a `barrel.db` file inside `/data/barrel` which is the working data direct
 
 ### Background
 
-- [ ] Merge old files
+- [x] Merge old files
 - [x] Hints file
 - [x] GC cleanup of old/expired/deleted keys
-- [ ] Compaction routine
+- [x] Compaction routine
 - [x] Rotate file if size increases
 ### Starting program
 
 - [x] Load data from hints file for faster boot time
 
-### Misc
-
-- [ ] Create specific mutex for different things
-
 ## Test Cases

cmd/server/config.sample.toml

@@ -0,0 +1,12 @@
+[server]
+address = ":6379"
+
+[app]
+debug = false # Enable debug logging
+dir = "/data/barreldb" # Directory to store .db files
+read_only = false # Whether to run barreldb in a read only mode. Write operations are not allowed in this mode.
+always_fsync = false # Whether to call `fsync(2)` on every write call. This significantly affects the performance but if data durability is a big concern, consider turning it on. 
+fsync_interval = "5s" # If always_fsync is turned off, barreldb can flush filesystem buffers periodically at this given interval.
+max_file_size = 1000000 # Maximum size of one datafile (.db file) in bytes. After this size has reached, it'll get rotated and a new .db file will be used for storing newer data.
+eval_file_size_interval = "1m" # Periodic interval to check if the size of the active .db file has reached `max_file_size`.
+compaction_interval = "6h" # Periodic interval to perform compaction for optimising disk usage. It cleans up deleted/expired keys and merges old datafiles into a single file.

cmd/server/init.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	flag "github.com/spf13/pflag"
+
+	"github.com/knadh/koanf"
+	"github.com/knadh/koanf/parsers/toml"
+	"github.com/knadh/koanf/providers/env"
+	"github.com/knadh/koanf/providers/file"
+	"github.com/zerodha/logf"
+)
+
+// initLogger initializes logger instance.
+func initLogger(ko *koanf.Koanf) logf.Logger {
+	opts := logf.Opts{EnableCaller: true}
+	if ko.String("app.log") == "debug" {
+		opts.Level = logf.DebugLevel
+		opts.EnableColor = true
+	}
+	return logf.New(opts)
+}
+
+// initConfig loads config to `ko` object.
+func initConfig() (*koanf.Koanf, error) {
+	var (
+		ko = koanf.New(".")
+		f  = flag.NewFlagSet("front", flag.ContinueOnError)
+	)
+
+	// Configure Flags.
+	f.Usage = func() {
+		fmt.Println(f.FlagUsages())
+		os.Exit(0)
+	}
+
+	// Register `--config` flag.
+	cfgPath := f.String("config", "config.sample.toml", "Path to a config file to load.")
+
+	// Parse and Load Flags.
+	err := f.Parse(os.Args[1:])
+	if err != nil {
+		return nil, err
+	}
+
+	err = ko.Load(file.Provider(*cfgPath), toml.Parser())
+	if err != nil {
+		return nil, err
+	}
+	err = ko.Load(env.Provider("BARRELDB_", ".", func(s string) string {
+		return strings.Replace(strings.ToLower(
+			strings.TrimPrefix(s, "BARRELDB_")), "__", ".", -1)
+	}), nil)
+	if err != nil {
+		return nil, err
+	}
+	return ko, nil
+}

cmd/server/main.go

@@ -1,52 +1,69 @@
 package main
 
 import (
-	"log"
+	"context"
+	"fmt"
 	"os"
-	"time"
+	"os/signal"
+	"syscall"
 
 	"github.com/mr-karan/barreldb/pkg/barrel"
 	"github.com/tidwall/redcon"
+	"github.com/zerodha/logf"
 )
 
 var (
 	// Version of the build. This is injected at build-time.
 	buildString = "unknown"
-	lo          = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)
-	addr        = ":6380"
 )
 
 type App struct {
+	lo     logf.Logger
 	barrel *barrel.Barrel
 }
 
 func main() {
-	barrel, err := barrel.Init(barrel.Opts{
-		Dir:                   ".",
-		ReadOnly:              false,
-		AlwaysFSync:           true,
-		MaxActiveFileSize:     1 << 4,
-		SyncInterval:          time.Second * 30,
-		Debug:                 false,
-		CompactInterval:       time.Second * 20,
-		CheckFileSizeInterval: time.Minute * 1,
-	})
+	// Initialise and load the config.
+	ko, err := initConfig()
 	if err != nil {
-		lo.Fatal("error opening barrel db: %w", err)
+		fmt.Printf("error loading config: %v", err)
+		os.Exit(-1)
 	}
 
 	app := &App{
-		barrel: barrel,
+		lo: initLogger(ko),
+	}
+	app.lo.Info("booting barreldb server", "version", buildString)
+
+	barrel, err := barrel.Init(barrel.Opts{
+		Debug:                 ko.Bool("app.debug"),
+		Dir:                   ko.MustString("app.dir"),
+		ReadOnly:              ko.Bool("app.read_only"),
+		AlwaysFSync:           ko.Bool("app.always_fsync"),
+		SyncInterval:          ko.Duration("app.fsync_interval"),
+		MaxActiveFileSize:     ko.Int64("app.max_file_size"),
+		CompactInterval:       ko.Duration("app.compaction_interval"),
+		CheckFileSizeInterval: ko.Duration("app.eval_file_size_interval"),
+	})
+	if err != nil {
+		app.lo.Fatal("error opening barrel db", "error", err)
 	}
 
+	app.barrel = barrel
+
+	// Initialise server.
 	mux := redcon.NewServeMux()
 	mux.HandleFunc("ping", app.ping)
 	mux.HandleFunc("quit", app.quit)
 	mux.HandleFunc("set", app.set)
 	mux.HandleFunc("get", app.get)
 	mux.HandleFunc("del", app.delete)
 
-	if err := redcon.ListenAndServe(addr,
+	// Create a channel to listen for cancellation signals.
+	// Create a new context which is cancelled when `SIGINT`/`SIGTERM` is received.
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+
+	srvr := redcon.NewServer(ko.MustString("server.address"),
 		mux.ServeRESP,
 		func(conn redcon.Conn) bool {
 			// use this function to accept or deny the connection.
@@ -55,7 +72,22 @@ func main() {
 		func(conn redcon.Conn, err error) {
 			// this is called when the connection has been closed
 		},
-	); err != nil {
-		lo.Fatal("error starting server: %w", err)
-	}
+	)
+
+	// Sart the server in a goroutine.
+	go func() {
+		if err := srvr.ListenAndServe(); err != nil {
+			app.lo.Fatal("failed to listen and serve", "error", err)
+		}
+	}()
+
+	// Listen on the close channel indefinitely until a
+	// `SIGINT` or `SIGTERM` is received.
+	<-ctx.Done()
+
+	// Cancel the context to gracefully shutdown and perform
+	// any cleanup tasks.
+	cancel()
+	app.barrel.Shutdown()
+	srvr.Close()
 }

go.mod

@@ -3,12 +3,19 @@ module github.com/mr-karan/barreldb
 go 1.19
 
 require (
+	github.com/knadh/koanf v1.4.4
+	github.com/spf13/pflag v1.0.5
 	github.com/tidwall/redcon v1.6.0
 	github.com/zerodha/logf v0.5.5
 	golang.org/x/sys v0.2.0
 )
 
 require (
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/pelletier/go-toml v1.7.0 // indirect
 	github.com/tidwall/btree v1.1.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 )