-
Notifications
You must be signed in to change notification settings - Fork 344
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
docs: redundancy strategies openapi #4510
Changes from all commits
8189b95
7b74ac7
738b403
83d4ecf
79ef2c8
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -945,6 +945,17 @@ components: | |
Add redundancy to the data being uploaded so that downloaders can download it with better UX. | ||
0 value is default and does not add any redundancy to the file. | ||
SwarmRedundancyStrategyParameter: | ||
in: header | ||
name: swarm-redundancy-parameter | ||
schema: | ||
type: integer | ||
enum: [0, 1, 2, 3, 4] | ||
required: false | ||
description: > | ||
Force different retrieve strategies on redundant data. | ||
The mumbers stand for NONE, DATA, PROX and RACE, respectively. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 5 values, but only 4 words? Respectively doesn't seem to align? Also "mumbers" should be "numbers". |
||
ContentTypePreserved: | ||
in: header | ||
name: Content-Type | ||
|
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
package getter | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"time" | ||
) | ||
|
||
var ( | ||
StrategyTimeout = 500 * time.Millisecond // timeout for each strategy | ||
) | ||
|
||
type Strategy = int | ||
|
||
const ( | ||
NONE Strategy = iota // no prefetching and no decoding | ||
DATA // just retrieve data shards no decoding | ||
PROX // proximity driven selective fetching | ||
RACE // aggressive fetching racing all chunks | ||
strategyCnt | ||
) | ||
|
||
func (g *getter) prefetch(ctx context.Context, strategy int, strict bool) { | ||
if strict && strategy == NONE { | ||
return | ||
} | ||
|
||
ctx, cancel := context.WithCancel(ctx) | ||
defer cancel() | ||
|
||
run := func(s Strategy) error { | ||
if s == PROX { // NOT IMPLEMENTED | ||
return fmt.Errorf("strategy %d not implemented", s) | ||
} | ||
|
||
var stop <-chan time.Time | ||
if s < RACE { | ||
timer := time.NewTimer(StrategyTimeout) | ||
defer timer.Stop() | ||
stop = timer.C | ||
} | ||
|
||
prefetch(ctx, g, s) | ||
|
||
select { | ||
// successfully retrieved shardCnt number of chunks | ||
case <-g.ready: | ||
case <-stop: | ||
return fmt.Errorf("prefetching with strategy %d timed out", s) | ||
case <-ctx.Done(): | ||
return nil | ||
} | ||
// call the erasure decoder | ||
// if decoding is successful terminate the prefetch loop | ||
return g.recover(ctx) // context to cancel when shardCnt chunks are retrieved | ||
} | ||
var err error | ||
for s := strategy; s == strategy || (err != nil && !strict && s < strategyCnt); s++ { | ||
err = run(s) | ||
} | ||
} | ||
|
||
// prefetch launches the retrieval of chunks based on the strategy | ||
func prefetch(ctx context.Context, g *getter, s Strategy) { | ||
var m []int | ||
switch s { | ||
case NONE: | ||
return | ||
case DATA: | ||
// only retrieve data shards | ||
m = g.missing() | ||
case PROX: | ||
// proximity driven selective fetching | ||
// NOT IMPLEMENTED | ||
case RACE: | ||
// retrieve all chunks at once enabling race among chunks | ||
m = g.missing() | ||
for i := g.shardCnt; i < len(g.addrs); i++ { | ||
m = append(m, i) | ||
} | ||
} | ||
for _, i := range m { | ||
i := i | ||
g.wg.Add(1) | ||
go func() { | ||
g.fetch(ctx, i) | ||
g.wg.Done() | ||
}() | ||
} | ||
} |
Unchanged files with check annotations Beta
// Copyright 2020 The Swarm Authors. All rights reserved. | ||
// Use of this source code is governed by a BSD-style | ||
// license that can be found in the LICENSE file. | ||
return | ||
} | ||
sAddresses, pAddresses := file.ChunkAddresses(data[:pSize], parity, j.refLength) | ||
getter := getter.New(sAddresses, pAddresses, j.getter, j.putter) | ||
Check failure on line 158 in pkg/file/joiner/joiner.go
|
||
for cursor := 0; cursor < len(data); cursor += j.refLength { | ||
if bytesToRead == 0 { | ||
break | ||
return err | ||
} | ||
sAddresses, pAddresses := file.ChunkAddresses(data[:eSize], parity, j.refLength) | ||
getter := getter.New(sAddresses, pAddresses, j.getter, j.putter) | ||
Check failure on line 309 in pkg/file/joiner/joiner.go
|
||
for cursor := 0; cursor < len(data); cursor += j.refLength { | ||
ref := data[cursor : cursor+j.refLength] | ||
var reportAddr swarm.Address |
"errors" | ||
"fmt" | ||
"github.com/ethersphere/bee/pkg/file/joiner" | ||
"github.com/ethersphere/bee/pkg/file/loadsave" | ||
"github.com/ethersphere/bee/pkg/manifest" | ||
"github.com/ethersphere/bee/pkg/manifest/mantaray" |
mtx.Lock() | ||
bins[bin]++ | ||
totaldur += dur.Seconds() | ||
peers = append(peers, peer{snapshot, dur, addr, bin, s.reserve.IsWithinStorageRadius(addr)}) | ||
mtx.Unlock() | ||
}() | ||
return false, false, nil | ||
} | ||
selfHealth := true | ||
if s.reserve.StorageRadius() != networkRadius { | ||
selfHealth = false | ||
s.logger.Warning("node is unhealthy due to storage radius discrepency", "self_radius", s.reserve.StorageRadius(), "network_radius", networkRadius) | ||
} | ||
s.isSelfHealthy.Store(selfHealth) |
"github.com/ethersphere/bee/pkg/node" | ||
"github.com/ethersphere/bee/pkg/postage" | ||
"github.com/ethersphere/bee/pkg/storage" | ||
"github.com/ethersphere/bee/pkg/storer" | ||
"github.com/ethersphere/bee/pkg/swarm" | ||
"github.com/spf13/cobra" |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Would it be good to mention which is the default if the header is not specified?