From 304942d935d6854b80b9a7849557b15db7cdce27 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 9 Nov 2015 16:57:47 -0800 Subject: [PATCH 1/3] DRY up the minimum retention duration variable --- meta/errors.go | 2 +- meta/store.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/meta/errors.go b/meta/errors.go index c076e841b83..750ed6612ef 100644 --- a/meta/errors.go +++ b/meta/errors.go @@ -67,7 +67,7 @@ var ( // ErrRetentionPolicyDurationTooLow is returned when updating a retention // policy that has a duration lower than the allowed minimum. ErrRetentionPolicyDurationTooLow = newError(fmt.Sprintf("retention policy duration must be at least %s", - RetentionPolicyMinDuration)) + MinRetentionPolicyDuration)) // ErrReplicationFactorTooLow is returned when the replication factor is not in an // acceptable range. diff --git a/meta/store.go b/meta/store.go index 853de96f606..34900b93466 100644 --- a/meta/store.go +++ b/meta/store.go @@ -46,7 +46,6 @@ const ExecMagic = "EXEC" const ( AutoCreateRetentionPolicyName = "default" AutoCreateRetentionPolicyPeriod = 0 - RetentionPolicyMinDuration = time.Hour // MaxAutoCreatedRetentionPolicyReplicaN is the maximum replication factor that will // be set for auto-created retention policies. @@ -1020,7 +1019,7 @@ func (s *Store) RetentionPolicies(database string) (a []RetentionPolicyInfo, err // CreateRetentionPolicy creates a new retention policy for a database. func (s *Store) CreateRetentionPolicy(database string, rpi *RetentionPolicyInfo) (*RetentionPolicyInfo, error) { - if rpi.Duration < RetentionPolicyMinDuration && rpi.Duration != 0 { + if rpi.Duration < MinRetentionPolicyDuration && rpi.Duration != 0 { return nil, ErrRetentionPolicyDurationTooLow } if err := s.exec(internal.Command_CreateRetentionPolicyCommand, internal.E_CreateRetentionPolicyCommand_Command, From bbe905804126a19d1b3ab4a82dec1673fcd64f82 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 9 Nov 2015 16:58:12 -0800 Subject: [PATCH 2/3] Don't SHOW SHARDS for deleted shard groups Fixes issue #4709. --- CHANGELOG.md | 1 + meta/statement_executor.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 844958998f7..6ee986b9bed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ - [#4278](https://github.com/influxdb/influxdb/pull/4278): Fix error marshalling across the cluster - [#4149](https://github.com/influxdb/influxdb/pull/4149): Fix derivative unnecessarily requires aggregate function. Thanks @peekeri! - [#4674](https://github.com/influxdb/influxdb/pull/4674): Fix panic during restore. Thanks @simcap. +- [#4725](https://github.com/influxdb/influxdb/pull/4725): Don't list deleted shards during SHOW SHARDS. - [#4237](https://github.com/influxdb/influxdb/issues/4237): DERIVATIVE() edge conditions - [#4263](https://github.com/influxdb/influxdb/issues/4263): derivative does not work when data is missing - [#4293](https://github.com/influxdb/influxdb/pull/4293): Ensure shell is invoked when touching PID file. Thanks @christopherjdickson diff --git a/meta/statement_executor.go b/meta/statement_executor.go index 2928c6c2948..9b2672e2d3c 100644 --- a/meta/statement_executor.go +++ b/meta/statement_executor.go @@ -363,6 +363,12 @@ func (e *StatementExecutor) executeShowShardsStatement(stmt *influxql.ShowShards row := &models.Row{Columns: []string{"id", "database", "retention_policy", "start_time", "end_time", "expiry_time", "owners"}, Name: di.Name} for _, rpi := range di.RetentionPolicies { for _, sgi := range rpi.ShardGroups { + // Shards associated with deleted shard groups are effectively deleted. + // Don't list them. + if sgi.Deleted() { + continue + } + for _, si := range sgi.Shards { ownerIDs := make([]uint64, len(si.Owners)) for i, owner := range si.Owners { From bece8fed2ae86e77204b3cbbd617f49d64e81441 Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Mon, 9 Nov 2015 17:22:24 -0800 Subject: [PATCH 3/3] Better retention enforcement logging Fixes issue #4727. --- services/retention/service.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/services/retention/service.go b/services/retention/service.go index a83721ebe37..0ec03e69a45 100644 --- a/services/retention/service.go +++ b/services/retention/service.go @@ -106,22 +106,28 @@ func (s *Service) deleteShards() { case <-ticker.C: s.logger.Println("retention policy shard deletion check commencing") - deletedShardIDs := make(map[uint64]struct{}, 0) + type deletionInfo struct { + db string + rp string + } + deletedShardIDs := make(map[uint64]deletionInfo, 0) s.MetaStore.VisitRetentionPolicies(func(d meta.DatabaseInfo, r meta.RetentionPolicyInfo) { for _, g := range r.DeletedShardGroups() { for _, sh := range g.Shards { - deletedShardIDs[sh.ID] = struct{}{} + deletedShardIDs[sh.ID] = deletionInfo{db: d.Name, rp: r.Name} } } }) for _, id := range s.TSDBStore.ShardIDs() { - if _, ok := deletedShardIDs[id]; ok { + if di, ok := deletedShardIDs[id]; ok { if err := s.TSDBStore.DeleteShard(id); err != nil { - s.logger.Printf("failed to delete shard ID %d: %s", id, err.Error()) + s.logger.Printf("failed to delete shard ID %d from database %s, retention policy %s: %s", + id, di.db, di.rp, err.Error()) continue } - s.logger.Printf("shard ID %d deleted", id) + s.logger.Printf("shard ID %d from database %s, retention policy %s, deleted", + id, di.db, di.rp) } } }