Skip to content
This repository has been archived by the owner on Aug 13, 2019. It is now read-only.

Open db in Read only mode #588

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
0997a15
Added db read only open mode and use it for the tsdb cli.
Apr 24, 2019
cd08753
add the DBReadOnly struct
Apr 24, 2019
8128c53
fix wal closing
Apr 24, 2019
83d3683
refactored to update blocks on domand
Apr 25, 2019
a704541
rename to DBView
Apr 25, 2019
a16cd17
revert renaming
Apr 25, 2019
ce25efc
remove metrics
Apr 25, 2019
825d6c0
detach db from openBlocks()
May 6, 2019
ca72a4c
Merge remote-tracking branch 'upstream/master' into read-only-alterna…
May 6, 2019
93af048
Merge remote-tracking branch 'upstream/master' into read-only-alterna…
May 29, 2019
c2d809b
refactor and add tests
May 30, 2019
1b68f0d
added db.Close to close all open blocks for windows
May 30, 2019
4778d73
add a read only interface for a block.
May 31, 2019
32d5aae
nits
Jun 3, 2019
b722198
Merge remote-tracking branch 'upstream/master' into read-only-alterna…
Jun 6, 2019
add6f1e
nits
krasi-georgiev Jun 11, 2019
a2726b4
Merge remote-tracking branch 'upstream/master' into read-only-alterna…
krasi-georgiev Jun 11, 2019
6092063
simplified
krasi-georgiev Jun 12, 2019
fbea5e8
refactored to use the Blockreader API.
krasi-georgiev Jun 13, 2019
3b76b2f
non blocking head closing and use dir hash to ensure read only.
krasi-georgiev Jun 14, 2019
95ba508
Merge branch 'master' into read-only-alternative
krasi-georgiev Jun 24, 2019
764d307
fix wal corruption metrics and head test
krasi-georgiev Jun 25, 2019
bd79d07
nits
krasi-georgiev Jun 25, 2019
106d0e9
nit
krasi-georgiev Jun 25, 2019
2b4ddbc
nits
krasi-georgiev Jul 2, 2019
32ae42e
Merge branch 'master' into read-only-alternative
krasi-georgiev Jul 4, 2019
fb75682
refactor error handling for DirHash and DirSize
krasi-georgiev Jul 9, 2019
1cf409f
NumSeries in Meta() of Head
codesome Jul 16, 2019
0a645ab
Merge pull request #5 from codesome/num-series-read-only
krasi-georgiev Jul 16, 2019
af10877
nits
krasi-georgiev Jul 17, 2019
b8c4781
Merge branch 'read-only-alternative' of github.com:krasi-georgiev/tsd…
krasi-georgiev Jul 17, 2019
df4a374
use channel for the db closing and remove mutex
krasi-georgiev Jul 18, 2019
b3f7774
add ErrClosed and add a test for it.
krasi-georgiev Jul 18, 2019
1a97569
handle mutli errors.
krasi-georgiev Jul 19, 2019
03b33b1
nits
krasi-georgiev Jul 19, 2019
beeaefa
head meta comment
krasi-georgiev Jul 22, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions cmd/tsdb/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,21 @@ func main() {
if err != nil {
exitWithError(err)
}
printBlocks(db.Blocks(), listCmdHumanReadable)
blocks, err := db.Blocks()
if err != nil {
exitWithError(err)
}
printBlocks(blocks, listCmdHumanReadable)
case analyzeCmd.FullCommand():
db, err := tsdb.OpenReadOnly(*analyzePath, nil, nil)

krasi-georgiev marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
exitWithError(err)
}
blocks := db.Blocks()
blocks, err := db.Blocks()
if err != nil {
exitWithError(err)
}
var block *tsdb.Block
if *analyzeBlockID != "" {
for _, b := range blocks {
Expand Down
93 changes: 53 additions & 40 deletions db.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,34 +247,73 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics {

// DBReadOnly provides APIs for read only operations on a database.
type DBReadOnly struct {
db *DB
logger log.Logger
dir string
registerer prometheus.Registerer
krasi-georgiev marked this conversation as resolved.
Show resolved Hide resolved
}

// OpenReadOnly returns a new DB in the given directory only for read operations.
func OpenReadOnly(dir string, l log.Logger, r prometheus.Registerer) (db *DBReadOnly, err error) {
func OpenReadOnly(dir string, l log.Logger, r prometheus.Registerer) (*DBReadOnly, error) {
if _, err := os.Stat(dir); err != nil {
return nil, err
krasi-georgiev marked this conversation as resolved.
Show resolved Hide resolved
}

if l == nil {
l = log.NewNopLogger()
}

if _, err := os.Stat(dir); err != nil {
db := &DBReadOnly{
krasi-georgiev marked this conversation as resolved.
Show resolved Hide resolved
logger: l,
dir: dir,
registerer: r,
}

return db, nil
}

// Querier loads the wal and returns a new querier over the data partition for the given time range.
krasi-georgiev marked this conversation as resolved.
Show resolved Hide resolved
// A goroutine must not handle more than one open Querier.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you mean? It is not thread safe? You can only invoke Querier once from single DbReadOnly? Can reword comment for that then?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a direct comment copy from the existing db.Querier.

It was added here, but can't tell why. Maybe you can give an idea why?
3065be9

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 Not sure either, I think it might mean similar one querier = one goroutine, but no sure 100% (:
That will work for now.

func (dbRead *DBReadOnly) Querier(mint, maxt int64) (Querier, error) {
head, err := NewHead(dbRead.registerer, dbRead.logger, nil, 1)
if err != nil {
return nil, err
}

head, err := NewHead(r, l, nil, 1)
blocks, err := dbRead.Blocks()
if err != nil {
return nil, err
}

db = &DBReadOnly{
&DB{
dir: dir,
logger: l,
head: head,
},
db := &DB{
dir: dbRead.dir,
logger: dbRead.logger,
head: head,
blocks: blocks,
}

// Set the min valid time for the ingested wal samples
// to be no lower than the maxt of the last block.
minValidTime := int64(math.MinInt64)
if len(blocks) > 0 {
minValidTime = blocks[len(blocks)-1].Meta().MaxTime
}

if err := db.head.Init(minValidTime); err != nil {
return nil, errors.Wrap(err, "read WAL")
}

return db.Querier(mint, maxt)
}

// Blocks returns the databases persisted blocks.
func (dbRead *DBReadOnly) Blocks() ([]*Block, error) {
db := &DB{
krasi-georgiev marked this conversation as resolved.
Show resolved Hide resolved
dir: dbRead.dir,
logger: dbRead.logger,
}
db.db.metrics = newDBMetrics(db.db, r)
db.metrics = newDBMetrics(db, dbRead.registerer)

loadable, corrupted, err := db.db.openBlocks()
loadable, corrupted, err := db.openBlocks()
if err != nil {
return nil, err
}
Expand All @@ -297,40 +336,14 @@ func OpenReadOnly(dir string, l log.Logger, r prometheus.Registerer) (db *DBRead
return loadable[i].Meta().MinTime < loadable[j].Meta().MinTime
})

db.db.blocks = loadable

blockMetas := make([]BlockMeta, 0, len(loadable))
for _, b := range loadable {
blockMetas = append(blockMetas, b.Meta())
}
if overlaps := OverlappingBlocks(blockMetas); len(overlaps) > 0 {
level.Warn(db.db.logger).Log("msg", "overlapping blocks found during opening", "detail", overlaps.String())
level.Warn(dbRead.logger).Log("msg", "overlapping blocks found during opening", "detail", overlaps.String())
}
return db, nil
}

// Querier loads the wal and returns a new querier over the data partition for the given time range.
// A goroutine must not handle more than one open Querier.
func (db *DBReadOnly) Querier(mint, maxt int64) (Querier, error) {

// Set the min valid time for the ingested wal samples
// to be no lower than the maxt of the last block.
blocks := db.db.Blocks()
minValidTime := int64(math.MinInt64)
if len(blocks) > 0 {
minValidTime = blocks[len(blocks)-1].Meta().MaxTime
}

if err := db.db.head.Init(minValidTime); err != nil {
return nil, errors.Wrap(err, "read WAL")
}

return db.db.Querier(mint, maxt)
}

// Blocks returns the databases persisted blocks.
func (db *DBReadOnly) Blocks() []*Block {
return db.db.Blocks()
return loadable, nil
}

// Open returns a new DB in the given directory.
Expand Down