Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .github/workflows/checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,5 @@ jobs:
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Install toolchains and tools
run: make install-tools
- name: Run checks and unit tests
run: make test
25 changes: 8 additions & 17 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,10 @@ package-all:
# Generate NOTICE file.
.PHONY: generate-notices
generate-notices:
@go-licenses report ./ctl/... --template ctl/build/notice.tpl > ctl/NOTICE.md --ignore git.beegfs.io --ignore github.com/thinkparq
@go-licenses report ./rst/remote/... --template rst/remote/build/notice.tpl > rst/remote/NOTICE.md --ignore git.beegfs.io --ignore github.com/thinkparq
@go-licenses report ./rst/sync/... --template rst/sync/build/notice.tpl > rst/sync/NOTICE.md --ignore git.beegfs.io --ignore github.com/thinkparq
@go-licenses report ./watch/... --template watch/build/notice.tpl > watch/NOTICE.md --ignore git.beegfs.io --ignore github.com/thinkparq
@go tool go-licenses report ./ctl/... --template ctl/build/notice.tpl > ctl/NOTICE.md --ignore git.beegfs.io --ignore github.com/thinkparq
@go tool go-licenses report ./rst/remote/... --template rst/remote/build/notice.tpl > rst/remote/NOTICE.md --ignore git.beegfs.io --ignore github.com/thinkparq
@go tool go-licenses report ./rst/sync/... --template rst/sync/build/notice.tpl > rst/sync/NOTICE.md --ignore git.beegfs.io --ignore github.com/thinkparq
@go tool go-licenses report ./watch/... --template watch/build/notice.tpl > watch/NOTICE.md --ignore git.beegfs.io --ignore github.com/thinkparq

# Test targets:
# Test targets may make change to the local repository (e.g. running go mod tidy) to
Expand Down Expand Up @@ -100,21 +100,20 @@ check-gofmt:
@if [ -n "$$(gofmt -l ./ | grep -v vendor)" ]; then \
echo "The following files have not been formatted using gofmt:"; \
gofmt -l ./ ;\
echo -e "\nFix individual files with: \ngofmt -w <file> \n"\
"\nOr fix all files with:\nfind . -type d \( -path './vendor' \) -prune -o -name '*.go' -print0 | xargs -0 gofmt -w \n\n" ;\
echo -e "\nFix all files with: \ngo fmt ./... \n";\
exit 1; \
fi

# Run the linters
.PHONY: check-linters
check-linters:
staticcheck ./...
go tool staticcheck ./...
go vet ./...

# Check for vulnerability issues
.PHONY: check-vulnerabilities
check-vulnerabilities:
govulncheck ./...
go tool govulncheck ./...

# Run the unit tests
.PHONY: test-unit
Expand Down Expand Up @@ -150,7 +149,7 @@ check-go-tidy: tidy
.PHONY: check-licenses
check-licenses: generate-notices
@echo "Checking license compliance..."
@go-licenses check ./... \
@go tool go-licenses check ./... \
--disallowed_types=forbidden,permissive,reciprocal,restricted,unknown \
--ignore git.beegfs.io \
--ignore github.com/thinkparq \
Expand All @@ -172,14 +171,6 @@ check-licenses: generate-notices
exit 1; \
fi


# Targets for installation of various prerequisites:
.PHONY: install-tools
install-tools:
go install honnef.co/go/tools/cmd/staticcheck@latest
go install github.com/google/go-licenses@v1.6.0
go install golang.org/x/vuln/cmd/govulncheck@latest

.PHONY: tidy
tidy :
@go mod tidy
13 changes: 6 additions & 7 deletions ctl/pkg/ctl/entry/chooser.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package entry

import (
"context"
"errors"
"fmt"
"math/rand/v2"
Expand Down Expand Up @@ -64,7 +63,6 @@ func getRandomIDChooser() func(fromDstIDs []uint16, idsAlreadyInStripePattern []
// IMPORTANT: dstTargets and dstGroups should not contain any duplicates but must be provided as
// slices to optimize copying the slice for randomization.
func getMigrationForEntry(
ctx context.Context,
entry *GetEntryCombinedInfo,
srcTargets map[uint16]struct{},
srcGroups map[uint16]struct{},
Expand All @@ -83,35 +81,36 @@ func getMigrationForEntry(
dstIDs = make([]uint16, 0, 4)
unmodifiedIDs = make([]uint16, 0, 4)
var rebalanceIDType msg.RebalanceIDType = msg.RebalanceIDTypeInvalid
if entry.Entry.Pattern.Type == beegfs.StripePatternBuddyMirror {
switch entry.Entry.Pattern.Type {
case beegfs.StripePatternBuddyMirror:
rebalanceIDType = msg.RebalanceIDTypeGroup
for _, group := range entry.Entry.Pattern.TargetIDs {
if _, ok := srcGroups[group]; ok {
randomID, err := targetChooser(dstGroups, entry.Entry.Pattern.TargetIDs)
if err != nil {
return msg.RebalanceIDTypeInvalid, nil, nil, nil, fmt.Errorf("insufficient destination groups to migrate entry away from the specified groups (entry is currently assigned to groups %v)", entry.Entry.Pattern.TargetIDs)
return msg.RebalanceIDTypeInvalid, nil, nil, nil, fmt.Errorf("insufficient available destination groups to migrate entry away from the specified groups (entry is currently assigned to groups %v)", entry.Entry.Pattern.TargetIDs)
}
srcIDs = append(srcIDs, group)
dstIDs = append(dstIDs, randomID)
} else {
unmodifiedIDs = append(unmodifiedIDs, group)
}
}
} else if entry.Entry.Pattern.Type == beegfs.StripePatternRaid0 {
case beegfs.StripePatternRaid0:
rebalanceIDType = msg.RebalanceIDTypeTarget
for _, target := range entry.Entry.Pattern.TargetIDs {
if _, ok := srcTargets[target]; ok {
randomID, err := targetChooser(dstTargets, entry.Entry.Pattern.TargetIDs)
if err != nil {
return msg.RebalanceIDTypeInvalid, nil, nil, nil, fmt.Errorf("insufficient destination targets to migrate entry away from the specified targets (entry is currently assigned to targets %v)", entry.Entry.Pattern.TargetIDs)
return msg.RebalanceIDTypeInvalid, nil, nil, nil, fmt.Errorf("insufficient available destination targets to migrate entry away from the specified targets (entry is currently assigned to targets %v)", entry.Entry.Pattern.TargetIDs)
}
srcIDs = append(srcIDs, target)
dstIDs = append(dstIDs, randomID)
} else {
unmodifiedIDs = append(unmodifiedIDs, target)
}
}
} else {
default:
return msg.RebalanceIDTypeInvalid, nil, nil, nil, fmt.Errorf("unsupported pattern type: %s", entry.Entry.Pattern.Type)
}

Expand Down
3 changes: 1 addition & 2 deletions ctl/pkg/ctl/entry/chooser_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package entry

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -167,7 +166,7 @@ func TestStartRebalancingJobs(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

rebalanceIDType, srcIDs, dstIDs, unmodifiedIDs, err := getMigrationForEntry(context.Background(), tt.entry, tt.srcTargets, tt.srcGroups, tt.dstTargets, tt.dstGroups)
rebalanceIDType, srcIDs, dstIDs, unmodifiedIDs, err := getMigrationForEntry(tt.entry, tt.srcTargets, tt.srcGroups, tt.dstTargets, tt.dstGroups)
if tt.wantErr {
assert.Error(t, err, "expected an error getting migrations for entry")
assert.Empty(t, srcIDs, "expected no srcIDs when no rebalance started")
Expand Down
7 changes: 4 additions & 3 deletions ctl/pkg/ctl/entry/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,8 @@ func checkPoolForPattern(storagePool pool.GetStoragePools_Result, pattern beegfs
// files to be created using targets in different pools unless forced.
func checkAndGetTargets(force bool, mappings *util.Mappings, storagePool pool.GetStoragePools_Result, pattern beegfs.StripePatternType, targetsOrBuddies []beegfs.EntityId) ([]uint16, error) {
ids := map[uint16]struct{}{}
if pattern == beegfs.StripePatternRaid0 {
switch pattern {
case beegfs.StripePatternRaid0:
targetMap := map[beegfs.EntityIdSet]struct{}{}
for _, t := range storagePool.Targets {
targetMap[t] = struct{}{}
Expand All @@ -218,7 +219,7 @@ func checkAndGetTargets(force bool, mappings *util.Mappings, storagePool pool.Ge
ids[uint16(t.LegacyId.NumId)] = struct{}{}
}
}
} else if pattern == beegfs.StripePatternBuddyMirror {
case beegfs.StripePatternBuddyMirror:
buddyMap := map[beegfs.EntityIdSet]struct{}{}
for _, b := range storagePool.BuddyGroups {
buddyMap[b] = struct{}{}
Expand All @@ -236,7 +237,7 @@ func checkAndGetTargets(force bool, mappings *util.Mappings, storagePool pool.Ge
ids[uint16(b.LegacyId.NumId)] = struct{}{}
}
}
} else {
default:
return nil, fmt.Errorf("unknown stripe pattern: %s", pattern)
}

Expand Down
2 changes: 1 addition & 1 deletion ctl/pkg/ctl/entry/disposal.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ type disposalCleaner struct {
log *logger.Logger
}

// CleanupDisposals() returns two channels where the results or errors running the disposal cleaner
// CleanupDisposals returns two channels where the results or errors running the disposal cleaner
// will be sent. It immediately returns an error if any initialization fails. The DisposalResult
// channel will be closed once all results have been returned, or if a fatal error occurs.
func CleanupDisposals(ctx context.Context, cfg DisposalCfg) (<-chan DisposalResult, <-chan error, error) {
Expand Down
11 changes: 6 additions & 5 deletions ctl/pkg/ctl/entry/entry.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// Package entry provides functions and types for interacting with entries in BeeGFS.
package entry

import (
Expand Down Expand Up @@ -807,14 +808,14 @@ func SetFileRstIds(ctx context.Context, entry msg.EntryInfo, ownerNode beegfs.No
return err
}

rstIdRequest := &msg.SetFilePatternRequest{EntryInfo: entry, RST: msg.RemoteStorageTarget{RSTIDs: rstIds}}
rstIdResp := &msg.SetFilePatternResponse{}
err = store.RequestTCP(ctx, ownerNode.Uid, rstIdRequest, rstIdResp)
rstIDRequest := &msg.SetFilePatternRequest{EntryInfo: entry, RST: msg.RemoteStorageTarget{RSTIDs: rstIds}}
rstIDResp := &msg.SetFilePatternResponse{}
err = store.RequestTCP(ctx, ownerNode.Uid, rstIDRequest, rstIDResp)
if err != nil {
return err
}
if rstIdResp.Result != beegfs.OpsErr_SUCCESS {
return fmt.Errorf("server returned an error configuring file targets, %s: %w", path, rstIdResp.Result)
if rstIDResp.Result != beegfs.OpsErr_SUCCESS {
return fmt.Errorf("server returned an error configuring file targets, %s: %w", path, rstIDResp.Result)
}

return nil
Expand Down
2 changes: 1 addition & 1 deletion ctl/pkg/ctl/entry/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ func migrateEntry(ctx context.Context, mappings *util.Mappings, migration migrat
}

// Determine if the entry needs migration and if so, if there are enough targets/groups:
rebalanceType, srcIDs, destIDs, unmodifiedIDs, err := getMigrationForEntry(ctx, entry, migration.srcTargets, migration.srcGroups, migration.dstTargets, migration.dstGroups)
rebalanceType, srcIDs, destIDs, unmodifiedIDs, err := getMigrationForEntry(entry, migration.srcTargets, migration.srcGroups, migration.dstTargets, migration.dstGroups)
if err != nil {
if errors.Is(err, ErrEntryHasNoTargets) {
result.StartingIDs = "(unavailable)"
Expand Down
2 changes: 1 addition & 1 deletion ctl/pkg/ctl/entry/set.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"go.uber.org/zap"
)

// Equivalent of the original MODESETPATTERN args. Fields that are nil are unmodified.
// SetEntryCfg is equivalent to the original MODESETPATTERN args. Nil fields are unmodified.
//
// IMPORTANT: When updating this struct, add any fields that can only be modified by root to the
// checks in Validate().
Expand Down
32 changes: 31 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
module github.com/thinkparq/beegfs-go

go 1.23.12
go 1.25.3

tool (
github.com/google/go-licenses
golang.org/x/vuln/cmd/govulncheck
honnef.co/go/tools/cmd/staticcheck
)

require (
github.com/aws/aws-sdk-go-v2 v1.36.4
Expand Down Expand Up @@ -31,6 +37,7 @@ require (
)

require (
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 // indirect
Expand All @@ -48,29 +55,52 @@ require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/expr-lang/expr v1.17.5
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/go-licenses v1.6.0 // indirect
github.com/google/licenseclassifier v0.0.0-20210722185704-3043a050f148 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/otiai10/copy v1.6.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/sagikazarmark/locafero v0.9.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/cast v1.9.2 // indirect
github.com/src-d/gcfg v1.4.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/xanzy/ssh-agent v0.2.1 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.36.0 // indirect
go.opentelemetry.io/otel/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.39.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/tools v0.33.0 // indirect
golang.org/x/vuln v1.1.4 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect
gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
honnef.co/go/tools v0.6.1 // indirect
k8s.io/klog/v2 v2.80.1 // indirect
)
Loading
Loading