Skip to content

Commit

Permalink
block: expose GatherFileStats and use it
Browse files Browse the repository at this point in the history
Signed-off-by: Giedrius Statkevičius <giedriuswork@gmail.com>
  • Loading branch information
GiedriusS committed Dec 9, 2020
1 parent 4ca451f commit 259c70b
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 24 deletions.
7 changes: 4 additions & 3 deletions pkg/block/block.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func Upload(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bdir st
return errors.New("empty external labels are not allowed for Thanos block.")
}

meta.Thanos.Files, err = gatherFileStats(bdir, hf)
meta.Thanos.Files, err = GatherFileStats(bdir, hf)
if err != nil {
return errors.Wrap(err, "gather meta file stats")
}
Expand Down Expand Up @@ -279,8 +279,9 @@ func GetSegmentFiles(blockDir string) []string {
return result
}

// GatherFileStats returns an array with data about all files in a given block dir.
// TODO(bwplotka): Gather stats when dirctly uploading files.
func gatherFileStats(blockDir string, hf metadata.HashFunc) (res []metadata.File, _ error) {
func GatherFileStats(blockDir string, hf metadata.HashFunc) (res []metadata.File, _ error) {
files, err := ioutil.ReadDir(filepath.Join(blockDir, ChunksDirname))
if err != nil {
return nil, errors.Wrapf(err, "read dir %v", filepath.Join(blockDir, ChunksDirname))
Expand All @@ -290,7 +291,7 @@ func gatherFileStats(blockDir string, hf metadata.HashFunc) (res []metadata.File
RelPath: filepath.Join(ChunksDirname, f.Name()),
SizeBytes: f.Size(),
}
if hf != metadata.NoneFunc {
if hf != metadata.NoneFunc && !f.IsDir() {
h, err := metadata.CalculateHash(filepath.Join(blockDir, ChunksDirname, f.Name()), hf)
if err != nil {
return nil, errors.Wrapf(err, "calculate hash %v", filepath.Join(ChunksDirname, f.Name()))
Expand Down
24 changes: 3 additions & 21 deletions pkg/testutil/e2eutil/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/prometheus/prometheus/tsdb/index"
"golang.org/x/sync/errgroup"

"github.com/thanos-io/thanos/pkg/block"
"github.com/thanos-io/thanos/pkg/block/metadata"
"github.com/thanos-io/thanos/pkg/runutil"
"github.com/thanos-io/thanos/pkg/testutil"
Expand Down Expand Up @@ -482,29 +483,10 @@ func createBlock(

files := []metadata.File{}
if addHashes {
paths := []string{}
err := filepath.Walk(blockDir, func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
return filepath.SkipDir
}
paths = append(paths, path)
return nil
})
files, err = block.GatherFileStats(blockDir, metadata.SHA256Func)
if err != nil {
return id, errors.Wrapf(err, "walking %s", dir)
return id, errors.Wrapf(err, "gathering %s file stats", blockDir)
}

for _, p := range paths {
pHash, err := metadata.CalculateHash(blockDir+p, metadata.SHA256Func)
if err != nil {
return id, errors.Wrapf(err, "calculating hash of %s", blockDir+p)
}
files = append(files, metadata.File{
RelPath: p,
Hash: &pHash,
})
}

}

if _, err = metadata.InjectThanos(log.NewNopLogger(), blockDir, metadata.Thanos{
Expand Down

0 comments on commit 259c70b

Please sign in to comment.