Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: redundancy retrieve api #4529

Merged
merged 52 commits into from
Feb 7, 2024
Merged
Changes from 1 commit
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
cf63d66
fix: copyright header year
zelig Dec 20, 2023
9bb8112
fix: openapi param name
zelig Dec 20, 2023
d628dc4
refactor(getter): add delayed and forgetting store mocks
zelig Dec 20, 2023
a202707
fix(getter): timeout header parsing for time duration
zelig Dec 20, 2023
df9a82b
feat(api): extensive api tests for redundancy
zelig Dec 21, 2023
b54e832
fix(api): redundancy param strict = !fallbackmode (value of header)
zelig Dec 21, 2023
0738117
test: tmp, trying to figure out context
zelig Dec 21, 2023
14c8627
fix(api): context passing
zelig Dec 21, 2023
fad1062
fix(api): amend value passing through context
zelig Dec 21, 2023
37e4b5f
fix(pipeline): capitalisation
zelig Jan 1, 2024
44b5261
refactor(pseudorand): fix and add match
zelig Jan 1, 2024
bd4acad
refactor(storer/mock): redesign forgettingstore
zelig Jan 1, 2024
817816d
feat(joiner): introduce fetchtimeout on decoder get calls
zelig Jan 1, 2024
c01ceb0
fix(redundancy/getter): fill incomplete shard, recover from cancelled…
zelig Jan 1, 2024
808ba4c
test: add extensive multilevel joiner tests for redundancy
zelig Jan 1, 2024
2de2007
fix: check seek error and add Size func
zelig Jan 4, 2024
fdb02ec
fix: remove spurious selector in import
zelig Jan 4, 2024
2dcaef4
fix: improve GetParamsFromContext and remove debug lines
zelig Jan 4, 2024
65e3f23
fix: add comments and improve test
zelig Jan 4, 2024
5a59b65
fix: disable langos and lookahead buffer on top of joiner
zelig Jan 4, 2024
2ac6937
fix: bzz upload/download tests with redundancy and improve range quer…
zelig Jan 4, 2024
b7b1a76
fix(getter): fix default assignment to fetcher timeout
zelig Jan 4, 2024
7846bcd
fix: langos back, debug print lines removed
zelig Jan 4, 2024
89a756c
test: temporary debug lines in redundancy tests
zelig Jan 4, 2024
57f430e
refactor(replicas,redundancy): move level in context accessors
zelig Jan 4, 2024
867e1cd
fix(api): typo
zelig Jan 4, 2024
484310a
fix: set NONE as default replicas getter
zelig Jan 5, 2024
f208167
test: remove t.Parallel from inclusion proof test
zelig Jan 5, 2024
eb8fb71
fix: heed linter complaints
zelig Jan 5, 2024
56ba972
chore: remove langos from deps
zelig Jan 6, 2024
a2194d3
test: speed up
zelig Jan 6, 2024
050a3e8
fix: remove unused consts
zelig Jan 6, 2024
2f52ef3
test: speed up
zelig Jan 6, 2024
e563c82
fix(getter): lastIdx unnecessary
zelig Jan 6, 2024
a0b2dc9
test: increase timeout
zelig Jan 6, 2024
5f8862a
fix: typo
zelig Jan 10, 2024
0624ea4
fix: incorrectly placed context timeouts
zelig Jan 10, 2024
2e97467
fix: comment
zelig Jan 19, 2024
02ddf61
fix: slow redundancy test (#4537)
notanatol Jan 19, 2024
218fe7e
fix(redunadncy): amend parity numbers (#4547)
dysordys Jan 20, 2024
6139ab0
test(getter): decrease flakiness
zelig Jan 20, 2024
74bd848
chore(pseudorand): move from ioutil to testutil
zelig Jan 20, 2024
68c3483
refactor(redundancy): introduce getter Config and improve default params
zelig Jan 23, 2024
2821ff5
fix: simplify error
zelig Jan 23, 2024
606f2a8
test: add race detection detection
zelig Jan 23, 2024
6d4f2cb
fix: contexts
zelig Jan 23, 2024
3dc5df9
test: fix timeout
zelig Jan 24, 2024
5dc1802
test: increase timeout
zelig Jan 25, 2024
a1eb893
fix: file header added to new files
zelig Jan 25, 2024
3a47dc6
fix(joiner): reintroduces langos restrict no change in behaviour, loo…
zelig Jan 25, 2024
ec66cc4
fix(api): return after Serve
zelig Feb 3, 2024
969a134
test(file): fix timeouts
zelig Feb 3, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: slow redundancy test (#4537)
notanatol authored Jan 19, 2024

Verified

This commit was signed with the committer’s verified signature.
frapell Franco Pellegrini
commit 02ddf615d30d0c314f6cb6be134e81979d2b1ce9
5 changes: 4 additions & 1 deletion pkg/api/bzz_test.go
Original file line number Diff line number Diff line change
@@ -68,6 +68,7 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) {
fileDownloadResource := func(addr string) string { return "/bzz/" + addr + "/" }

testRedundancy := func(t *testing.T, rLevel redundancy.Level, encrypt bool, levels int, chunkCnt int, shardCnt int, parityCnt int) {
t.Helper()
seed, err := pseudorand.NewSeed()
if err != nil {
t.Fatal(err)
@@ -197,6 +198,7 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) {
})
}
for _, rLevel := range []redundancy.Level{1, 2, 3, 4} {
rLevel := rLevel
t.Run(fmt.Sprintf("level=%d", rLevel), func(t *testing.T) {
for _, encrypt := range []bool{false, true} {
encrypt := encrypt
@@ -216,17 +218,18 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) {
case 3:
chunkCnt = shardCnt*shardCnt + 1
}
levels := levels
t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d", encrypt, levels, chunkCnt), func(t *testing.T) {
if levels > 2 && (encrypt == (rLevel%2 == 1)) {
t.Skip("skipping to save time")
}
t.Parallel()
testRedundancy(t, rLevel, encrypt, levels, chunkCnt, shardCnt, parityCnt)
})
}
}
})
}

}

func TestBzzFiles(t *testing.T) {
5 changes: 4 additions & 1 deletion pkg/file/joiner/joiner_test.go
Original file line number Diff line number Diff line change
@@ -1128,10 +1128,11 @@ func TestJoinerRedundancy(t *testing.T) {
}
i := 0
eg, ectx := errgroup.WithContext(ctx)
scnt:
for ; i < shardCnt; i++ {
select {
case <-ectx.Done():
break
break scnt
default:
}
i := i
@@ -1226,6 +1227,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) {
getter.StrategyTimeout = 100 * time.Millisecond

test := func(t *testing.T, rLevel redundancy.Level, encrypt bool, size int) {
t.Helper()
store := mockstorer.NewForgettingStore(inmemchunkstore.New())
testutil.CleanupCloser(t, store)
seed, err := pseudorand.NewSeed()
@@ -1319,6 +1321,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) {
rLevel := rLevel
// speeding up tests by skipping some of them
t.Run(fmt.Sprintf("rLevel=%v", rLevel), func(t *testing.T) {
t.Parallel()
for _, encrypt := range []bool{false, true} {
encrypt := encrypt
shardCnt := rLevel.GetMaxShards()
21 changes: 4 additions & 17 deletions pkg/replicas/getter.go
Original file line number Diff line number Diff line change
@@ -24,20 +24,7 @@ import (
// then the probability of Swarmageddon is less than 0.000001
// assuming the error rate of chunk retrievals stays below the level expressed
// as depth by the publisher.
type ErrSwarmageddon struct {
error
}

func (err *ErrSwarmageddon) Unwrap() []error {
if err == nil || err.error == nil {
return nil
}
var uwe interface{ Unwrap() []error }
if !errors.As(err.error, &uwe) {
return nil
}
return uwe.Unwrap()
}
var ErrSwarmageddon = errors.New("swarmageddon has begun")

// getter is the private implementation of storage.Getter, an interface for
// retrieving chunks. This getter embeds the original simple chunk getter and extends it
@@ -69,7 +56,7 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e
resultC := make(chan swarm.Chunk)
// errc collects the errors
errc := make(chan error, 17)
var errs []error
var errs error
errcnt := 0

// concurrently call to retrieve chunk using original CAC address
@@ -108,10 +95,10 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e
return chunk, nil

case err = <-errc:
errs = append(errs, err)
errs = errors.Join(errs, err)
errcnt++
if errcnt > total {
return nil, &ErrSwarmageddon{errors.Join(errs...)}
return nil, errors.Join(ErrSwarmageddon, errs)
}

// ticker switches on the address channel
14 changes: 3 additions & 11 deletions pkg/replicas/getter_test.go
Original file line number Diff line number Diff line change
@@ -195,18 +195,11 @@ func TestGetter(t *testing.T) {
}

t.Run("returns correct error", func(t *testing.T) {
var esg *replicas.ErrSwarmageddon
if !errors.As(err, &esg) {
if !errors.Is(err, replicas.ErrSwarmageddon) {
t.Fatalf("incorrect error. want Swarmageddon. got %v", err)
}
errs := esg.Unwrap()
for _, err := range errs {
if !errors.Is(err, tc.failure.err) {
t.Fatalf("incorrect error. want it to wrap %v. got %v", tc.failure.err, err)
}
}
if len(errs) != tc.count+1 {
t.Fatalf("incorrect error. want %d. got %d", tc.count+1, len(errs))
if !errors.Is(err, tc.failure.err) {
t.Fatalf("incorrect error. want it to wrap %v. got %v", tc.failure.err, err)
}
})
}
@@ -265,7 +258,6 @@ func TestGetter(t *testing.T) {
}
}
})

})
}
}