Skip to content
This repository has been archived by the owner on Jun 27, 2023. It is now read-only.

Commit

Permalink
sync: update CI config files (#125)
Browse files Browse the repository at this point in the history
  • Loading branch information
@web3-bot
web3-bot authored Aug 26, 2022
1 parent 4571136 commit 2c23c3e
Show file tree
Hide file tree
Showing 15 changed files with 166 additions and 164 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/go-check.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ jobs:
env:
RUNGOGENERATE: false
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
with:
submodules: recursive
- uses: actions/setup-go@v2
- uses: actions/setup-go@v3
with:
go-version: "1.18.x"
go-version: "1.19.x"
- name: Run repo-specific setup
uses: ./.github/actions/go-check-setup
if: hashFiles('./.github/actions/go-check-setup') != ''
Expand All @@ -27,7 +27,7 @@ jobs:
echo "RUNGOGENERATE=true" >> $GITHUB_ENV
fi
- name: Install staticcheck
run: go install honnef.co/go/tools/cmd/staticcheck@d7e217c1ff411395475b2971c0824e1e7cc1af98 # 2022.1 (v0.3.0)
run: go install honnef.co/go/tools/cmd/staticcheck@376210a89477dedbe6fdc4484b233998650d7b3c # 2022.1.3 (v0.3.3)
- name: Check that go.mod is tidy
uses: protocol/multiple-go-modules@v1.2
with:
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/go-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@ jobs:
fail-fast: false
matrix:
os: [ "ubuntu", "windows", "macos" ]
go: [ "1.17.x", "1.18.x" ]
go: [ "1.18.x", "1.19.x" ]
env:
COVERAGES: ""
runs-on: ${{ format('{0}-latest', matrix.os) }}
name: ${{ matrix.os }} (go ${{ matrix.go }})
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
with:
submodules: recursive
- uses: actions/setup-go@v2
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Go information
Expand All @@ -43,7 +43,7 @@ jobs:
# Use -coverpkg=./..., so that we include cross-package coverage.
# If package ./A imports ./B, and ./A's tests also cover ./B,
# this means ./B's coverage will be significantly higher than 0%.
run: go test -v -coverprofile=module-coverage.txt -coverpkg=./... ./...
run: go test -v -shuffle=on -coverprofile=module-coverage.txt -coverpkg=./... ./...
- name: Run tests (32 bit)
if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX.
uses: protocol/multiple-go-modules@v1.2
Expand All @@ -52,7 +52,7 @@ jobs:
with:
run: |
export "PATH=${{ env.PATH_386 }}:$PATH"
go test -v ./...
go test -v -shuffle=on ./...
- name: Run tests with race detector
if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow
uses: protocol/multiple-go-modules@v1.2
Expand All @@ -62,7 +62,7 @@ jobs:
shell: bash
run: echo "COVERAGES=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_ENV
- name: Upload coverage to Codecov
uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0
uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
with:
files: '${{ env.COVERAGES }}'
env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }}
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,4 @@ require (
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
)

go 1.17
go 1.18
16 changes: 9 additions & 7 deletions hamt/hamt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@
// wikipedia article is the collapsing of empty shards.
// Given the following tree: ( '[' = shards, '{' = values )
// [ 'A' ] -> [ 'B' ] -> { "ABC" }
// | L-> { "ABD" }
// L-> { "ASDF" }
//
// | L-> { "ABD" }
// L-> { "ASDF" }
//
// If we simply removed "ABC", we would end up with a tree where shard 'B' only
// has a single child. This causes two issues, the first, is that now we have
// an extra lookup required to get to "ABD". The second issue is that now we
Expand Down Expand Up @@ -460,11 +462,11 @@ func (ds *Shard) walkChildren(processLinkValues func(formattedLink *ipld.Link) e

// parallelShardWalk is quite similar to the DAG walking algorithm from https://github.com/ipfs/go-merkledag/blob/594e515f162e764183243b72c2ba84f743424c8c/merkledag.go#L464
// However, there are a few notable differences:
// 1. Some children are actualized Shard structs and some are in the blockstore, this will leverage walking over the in memory Shards as well as the stored blocks
// 2. Instead of just passing each child into the worker pool by itself we group them so that we can leverage optimizations from GetMany.
// This optimization also makes the walk a little more biased towards depth (as opposed to BFS) in the earlier part of the DAG.
// This is particularly helpful for operations like estimating the directory size which should complete quickly when possible.
// 3. None of the extra options from that package are needed
// 1. Some children are actualized Shard structs and some are in the blockstore, this will leverage walking over the in memory Shards as well as the stored blocks
// 2. Instead of just passing each child into the worker pool by itself we group them so that we can leverage optimizations from GetMany.
// This optimization also makes the walk a little more biased towards depth (as opposed to BFS) in the earlier part of the DAG.
// This is particularly helpful for operations like estimating the directory size which should complete quickly when possible.
// 3. None of the extra options from that package are needed
func parallelShardWalk(ctx context.Context, root *Shard, dserv ipld.DAGService, processShardValues func(formattedLink *ipld.Link) error) error {
const concurrency = 32

Expand Down
11 changes: 5 additions & 6 deletions importer/balanced/balanced_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
mrand "math/rand"
"testing"

Expand Down Expand Up @@ -53,7 +52,7 @@ func getTestDag(t *testing.T, ds ipld.DAGService, size int64, blksize int64) (*d
return nd, data
}

//Test where calls to read are smaller than the chunk size
// Test where calls to read are smaller than the chunk size
func TestSizeBasedSplit(t *testing.T) {
if testing.Short() {
t.SkipNow()
Expand Down Expand Up @@ -119,7 +118,7 @@ func arrComp(a, b []byte) error {
}

func dagrArrComp(t *testing.T, r io.Reader, should []byte) {
out, err := ioutil.ReadAll(r)
out, err := io.ReadAll(r)
if err != nil {
t.Fatal(err)
}
Expand All @@ -138,7 +137,7 @@ func TestIndirectBlocks(t *testing.T) {
t.Fatal(err)
}

out, err := ioutil.ReadAll(reader)
out, err := io.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -179,7 +178,7 @@ func TestSeekToBegin(t *testing.T) {
t.Fatal(err)
}

n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
n, err := io.CopyN(io.Discard, rs, 1024*4)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -207,7 +206,7 @@ func TestSeekToAlmostBegin(t *testing.T) {
t.Fatal(err)
}

n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
n, err := io.CopyN(io.Discard, rs, 1024*4)
if err != nil {
t.Fatal(err)
}
Expand Down
176 changes: 88 additions & 88 deletions importer/balanced/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,36 +17,37 @@
// that the UnixFS node.
//
// Notes:
// 1. In the implementation. `FSNodeOverDag` structure is used for representing
// the UnixFS node encoded inside the DAG node.
// (see https://github.com/ipfs/go-ipfs/pull/5118.)
// 2. `TFile` is used for backwards-compatibility. It was a bug causing the leaf
// nodes to be generated with this type instead of `TRaw`. The former one
// should be used (like the trickle builder does).
// (See https://github.com/ipfs/go-ipfs/pull/5120.)
//
// +-------------+
// | Root 4 |
// +-------------+
// |
// +--------------------------+----------------------------+
// | |
// +-------------+ +-------------+
// | Node 2 | | Node 5 |
// +-------------+ +-------------+
// | |
// +-------------+-------------+ +-------------+
// | | |
// +-------------+ +-------------+ +-------------+
// | Node 1 | | Node 3 | | Node 6 |
// +-------------+ +-------------+ +-------------+
// | | |
// +------+------+ +------+------+ +------+
// | | | | |
// +=========+ +=========+ +=========+ +=========+ +=========+
// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 | | Chunk 5 |
// +=========+ +=========+ +=========+ +=========+ +=========+
// 1. In the implementation. `FSNodeOverDag` structure is used for representing
// the UnixFS node encoded inside the DAG node.
// (see https://github.com/ipfs/go-ipfs/pull/5118.)
//
// 2. `TFile` is used for backwards-compatibility. It was a bug causing the leaf
// nodes to be generated with this type instead of `TRaw`. The former one
// should be used (like the trickle builder does).
// (See https://github.com/ipfs/go-ipfs/pull/5120.)
//
// +-------------+
// | Root 4 |
// +-------------+
// |
// +--------------------------+----------------------------+
// | |
// +-------------+ +-------------+
// | Node 2 | | Node 5 |
// +-------------+ +-------------+
// | |
// +-------------+-------------+ +-------------+
// | | |
// +-------------+ +-------------+ +-------------+
// | Node 1 | | Node 3 | | Node 6 |
// +-------------+ +-------------+ +-------------+
// | | |
// +------+------+ +------+------+ +------+
// | | | | |
// +=========+ +=========+ +=========+ +=========+ +=========+
// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 | | Chunk 5 |
// +=========+ +=========+ +=========+ +=========+ +=========+
package balanced

import (
Expand Down Expand Up @@ -80,55 +81,54 @@ import (
// offset in the file the graph represents: each internal node uses the file size
// of its children as an index when seeking.
//
// `Layout` creates a root and hands it off to be filled:
//
// +-------------+
// | Root 1 |
// +-------------+
// |
// ( fillNodeRec fills in the )
// ( chunks on the root. )
// |
// +------+------+
// | |
// + - - - - + + - - - - +
// | Chunk 1 | | Chunk 2 |
// + - - - - + + - - - - +
// `Layout` creates a root and hands it off to be filled:
//
// ↓
// When the root is full but there's more data...
// ↓
// +-------------+
// | Root 1 |
// +-------------+
// |
// ( fillNodeRec fills in the )
// ( chunks on the root. )
// |
// +------+------+
// | |
// + - - - - + + - - - - +
// | Chunk 1 | | Chunk 2 |
// + - - - - + + - - - - +
//
// +-------------+
// | Root 1 |
// +-------------+
// |
// +------+------+
// | |
// +=========+ +=========+ + - - - - +
// | Chunk 1 | | Chunk 2 | | Chunk 3 |
// +=========+ +=========+ + - - - - +
// ↓
// When the root is full but there's more data...
// ↓
//
// ↓
// ...Layout's job is to create a new root.
// ↓
// +-------------+
// | Root 1 |
// +-------------+
// |
// +------+------+
// | |
// +=========+ +=========+ + - - - - +
// | Chunk 1 | | Chunk 2 | | Chunk 3 |
// +=========+ +=========+ + - - - - +
//
// +-------------+
// | Root 2 |
// +-------------+
// |
// +-------------+ - - - - - - - - +
// | |
// +-------------+ ( fillNodeRec creates the )
// | Node 1 | ( branch that connects )
// +-------------+ ( "Root 2" to "Chunk 3." )
// | |
// +------+------+ + - - - - -+
// | | |
// +=========+ +=========+ + - - - - +
// | Chunk 1 | | Chunk 2 | | Chunk 3 |
// +=========+ +=========+ + - - - - +
// ↓
// ...Layout's job is to create a new root.
// ↓
//
// +-------------+
// | Root 2 |
// +-------------+
// |
// +-------------+ - - - - - - - - +
// | |
// +-------------+ ( fillNodeRec creates the )
// | Node 1 | ( branch that connects )
// +-------------+ ( "Root 2" to "Chunk 3." )
// | |
// +------+------+ + - - - - -+
// | | |
// +=========+ +=========+ + - - - - +
// | Chunk 1 | | Chunk 2 | | Chunk 3 |
// +=========+ +=========+ + - - - - +
func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
if db.Done() {
// No data, return just an empty node.
Expand Down Expand Up @@ -185,22 +185,22 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
// root, and those children will in turn be filled (calling `fillNodeRec`
// recursively).
//
// +-------------+
// | `node` |
// | (new root) |
// +-------------+
// |
// +-------------+ - - - - - - + - - - - - - - - - - - +
// | | |
// +--------------+ + - - - - - + + - - - - - +
// | (old root) | | new child | | |
// +--------------+ + - - - - - + + - - - - - +
// | | |
// +------+------+ + - - + - - - +
// | | | |
// +=========+ +=========+ + - - - - + + - - - - +
// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 |
// +=========+ +=========+ + - - - - + + - - - - +
// +-------------+
// | `node` |
// | (new root) |
// +-------------+
// |
// +-------------+ - - - - - - + - - - - - - - - - - - +
// | | |
// +--------------+ + - - - - - + + - - - - - +
// | (old root) | | new child | | |
// +--------------+ + - - - - - + + - - - - - +
// | | |
// +------+------+ + - - + - - - +
// | | | |
// +=========+ +=========+ + - - - - + + - - - - +
// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 |
// +=========+ +=========+ + - - - - + + - - - - +
//
// The `node` to be filled uses the `FSNodeOverDag` abstraction that allows adding
// child nodes without packing/unpacking the UnixFS layer node (having an internal
Expand Down
19 changes: 10 additions & 9 deletions importer/helpers/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,18 @@ var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name + protobuf

// DefaultLinksPerBlock governs how the importer decides how many links there
// will be per block. This calculation is based on expected distributions of:
// * the expected distribution of block sizes
// * the expected distribution of link sizes
// * desired access speed
// - the expected distribution of block sizes
// - the expected distribution of link sizes
// - desired access speed
//
// For now, we use:
//
// var roughLinkBlockSize = 1 << 13 // 8KB
// var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name
// // + protobuf framing
// var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
// = ( 8192 / 47 )
// = (approximately) 174
// var roughLinkBlockSize = 1 << 13 // 8KB
// var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name
// // + protobuf framing
// var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
// = ( 8192 / 47 )
// = (approximately) 174
var DefaultLinksPerBlock = roughLinkBlockSize / roughLinkSize

// ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit.
Expand Down
Loading

0 comments on commit 2c23c3e

Please sign in to comment.