Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improves stream trimming performance by approximating #1

Open
wants to merge 8 commits into
base: redis-streams
Choose a base branch
from
69 changes: 42 additions & 27 deletions .rubocop_todo.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# This configuration was generated by
# `rubocop --auto-gen-config`
# on 2018-11-07 23:41:56 +0000 using RuboCop version 0.60.0.
# on 2018-11-08 00:58:02 +0000 using RuboCop version 0.60.0.
# The point is for the user to remove these configuration records
# one by one as the offenses are removed from the code base.
# Note that changes in the inspected code, or installation of new
Expand Down Expand Up @@ -54,12 +54,13 @@ Lint/LiteralAsCondition:
Exclude:
- 'examples/chat/chat.rb'

# Offense count: 3
# Offense count: 4
Lint/NonLocalExitFromIterator:
Exclude:
- 'lib/message_bus.rb'
- 'lib/message_bus/backends/postgres.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'

# Offense count: 1
# Cop supports --auto-correct.
Expand All @@ -78,7 +79,7 @@ Lint/StringConversionInInterpolation:
Exclude:
- 'spec/lib/message_bus/multi_process_spec.rb'

# Offense count: 30
# Offense count: 36
Metrics/AbcSize:
Max: 112

Expand All @@ -88,21 +89,21 @@ Metrics/AbcSize:
Metrics/BlockLength:
Max: 289

# Offense count: 3
# Offense count: 4
# Configuration parameters: CountBlocks.
Metrics/BlockNesting:
Max: 4

# Offense count: 8
# Offense count: 10
# Configuration parameters: CountComments.
Metrics/ClassLength:
Max: 307
Max: 312

# Offense count: 18
# Offense count: 21
Metrics/CyclomaticComplexity:
Max: 30

# Offense count: 39
# Offense count: 46
# Configuration parameters: CountComments, ExcludedMethods.
Metrics/MethodLength:
Max: 105
Expand All @@ -112,7 +113,7 @@ Metrics/MethodLength:
Metrics/ModuleLength:
Max: 415

# Offense count: 18
# Offense count: 21
Metrics/PerceivedComplexity:
Max: 34

Expand All @@ -124,7 +125,7 @@ Naming/MemoizedInstanceVariableName:
- 'lib/message_bus.rb'
- 'spec/assets/support/jasmine_helper.rb'

# Offense count: 2
# Offense count: 3
# Configuration parameters: NamePrefix, NamePrefixBlacklist, NameWhitelist, MethodDefinitionMacros.
# NamePrefix: is_, has_, have_
# NamePrefixBlacklist: is_, has_, have_
Expand All @@ -135,6 +136,7 @@ Naming/PredicateName:
- 'spec/**/*'
- 'lib/message_bus.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'

# Offense count: 6
# Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames.
Expand All @@ -158,11 +160,12 @@ Performance/RedundantMerge:
Exclude:
- 'spec/spec_helper.rb'

# Offense count: 3
# Offense count: 5
# Cop supports --auto-correct.
Performance/RegexpMatch:
Exclude:
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/rack/diagnostics.rb'

# Offense count: 1
Expand Down Expand Up @@ -211,7 +214,7 @@ Style/CaseEquality:
- 'lib/message_bus/distributed_cache.rb'
- 'spec/lib/message_bus/rack/middleware_spec.rb'

# Offense count: 24
# Offense count: 26
# Cop supports --auto-correct.
# Configuration parameters: AutoCorrect, EnforcedStyle.
# SupportedStyles: nested, compact
Expand All @@ -221,6 +224,7 @@ Style/ClassAndModuleChildren:
- 'lib/message_bus/backends/memory.rb'
- 'lib/message_bus/backends/postgres.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/client.rb'
- 'lib/message_bus/connection_manager.rb'
- 'lib/message_bus/diagnostics.rb'
Expand All @@ -236,21 +240,22 @@ Style/ClassVars:
Exclude:
- 'lib/message_bus/rack/diagnostics.rb'

# Offense count: 3
# Offense count: 4
# Cop supports --auto-correct.
# Configuration parameters: Keywords.
# Keywords: TODO, FIXME, OPTIMIZE, HACK, REVIEW
Style/CommentAnnotation:
Exclude:
- 'lib/message_bus/backends/postgres.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/rack/middleware.rb'

# Offense count: 33
# Offense count: 37
Style/Documentation:
Enabled: false

# Offense count: 6
# Offense count: 7
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: empty, nil, both
Expand All @@ -260,6 +265,7 @@ Style/EmptyElse:
- 'lib/message_bus/backends/memory.rb'
- 'lib/message_bus/backends/postgres.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'spec/lib/message_bus/backend_spec.rb'

# Offense count: 1
Expand Down Expand Up @@ -305,14 +311,15 @@ Style/GlobalVars:
- 'examples/chat/chat.rb'
- 'lib/message_bus/diagnostics.rb'

# Offense count: 12
# Offense count: 15
# Configuration parameters: MinBodyLength.
Style/GuardClause:
Exclude:
- 'lib/message_bus.rb'
- 'lib/message_bus/backends/memory.rb'
- 'lib/message_bus/backends/postgres.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/client.rb'
- 'lib/message_bus/rack/middleware.rb'

Expand All @@ -321,7 +328,7 @@ Style/IfInsideElse:
Exclude:
- 'lib/message_bus.rb'

# Offense count: 22
# Offense count: 30
# Cop supports --auto-correct.
Style/IfUnlessModifier:
Exclude:
Expand All @@ -330,6 +337,7 @@ Style/IfUnlessModifier:
- 'lib/message_bus/backends/memory.rb'
- 'lib/message_bus/backends/postgres.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/client.rb'
- 'lib/message_bus/rack/diagnostics.rb'
- 'lib/message_bus/rack/middleware.rb'
Expand Down Expand Up @@ -362,11 +370,12 @@ Style/MutableConstant:
- 'lib/message_bus/backends/postgres.rb'
- 'spec/spec_helper.rb'

# Offense count: 2
# Offense count: 3
# Cop supports --auto-correct.
Style/NegatedWhile:
Exclude:
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/timer_thread.rb'

# Offense count: 2
Expand All @@ -378,22 +387,23 @@ Style/Next:
- 'lib/message_bus/connection_manager.rb'
- 'lib/message_bus/timer_thread.rb'

# Offense count: 4
# Offense count: 6
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: predicate, comparison
Style/NilComparison:
Exclude:
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'spec/lib/message_bus/distributed_cache_spec.rb'

# Offense count: 5
# Offense count: 7
# Cop supports --auto-correct.
# Configuration parameters: Strict.
Style/NumericLiterals:
MinDigits: 15

# Offense count: 15
# Offense count: 17
# Cop supports --auto-correct.
# Configuration parameters: AutoCorrect, EnforcedStyle, IgnoredMethods.
# SupportedStyles: predicate, comparison
Expand All @@ -404,6 +414,7 @@ Style/NumericPredicate:
- 'lib/message_bus/backends/memory.rb'
- 'lib/message_bus/backends/postgres.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/client.rb'
- 'lib/message_bus/rack/middleware.rb'
- 'lib/message_bus/timer_thread.rb'
Expand All @@ -423,20 +434,22 @@ Style/PercentLiteralDelimiters:
- 'Guardfile'
- 'message_bus.gemspec'

# Offense count: 2
# Offense count: 3
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: compact, exploded
Style/RaiseArgs:
Exclude:
- 'lib/message_bus.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'

# Offense count: 3
# Offense count: 4
# Cop supports --auto-correct.
Style/RedundantBegin:
Exclude:
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/diagnostics.rb'

# Offense count: 2
Expand Down Expand Up @@ -489,7 +502,7 @@ Style/RescueModifier:
- 'lib/message_bus/client.rb'
- 'lib/message_bus/connection_manager.rb'

# Offense count: 17
# Offense count: 19
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: implicit, explicit
Expand All @@ -500,6 +513,7 @@ Style/RescueStandardError:
- 'lib/message_bus/backends/memory.rb'
- 'lib/message_bus/backends/postgres.rb'
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/client.rb'
- 'lib/message_bus/connection_manager.rb'
- 'lib/message_bus/diagnostics.rb'
Expand Down Expand Up @@ -540,7 +554,7 @@ Style/SpecialGlobalVars:
- 'message_bus.gemspec'
- 'spec/spec_helper.rb'

# Offense count: 647
# Offense count: 658
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
# SupportedStyles: single_quotes, double_quotes
Expand Down Expand Up @@ -639,16 +653,17 @@ Style/YodaCondition:
Exclude:
- 'spec/lib/message_bus/timer_thread_spec.rb'

# Offense count: 6
# Offense count: 7
# Cop supports --auto-correct.
Style/ZeroLengthPredicate:
Exclude:
- 'lib/message_bus/backends/redis.rb'
- 'lib/message_bus/backends/redis_streams.rb'
- 'lib/message_bus/client.rb'
- 'lib/message_bus/rack/middleware.rb'
- 'spec/lib/message_bus_spec.rb'

# Offense count: 104
# Offense count: 115
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
# URISchemes: http, https
Metrics/LineLength:
Expand Down
7 changes: 6 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,15 @@ gemfile:
- Gemfile
addons:
postgresql: "9.4"
apt:
packages:
- redis-server
env:
- PGUSER=postgres
before_script:
- redis-server --version
- sudo sed -i -e 's/bind 127.0.0.1 ::1/bind 127.0.0.1/g' /etc/redis/redis.conf
- sudo service redis-server start
- psql -c 'create database message_bus_test;' -U postgres
services:
- redis-server
- postgresql
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,8 @@ MessageBus.reliable_pub_sub.max_backlog_age = 100

```

The primary Redis-based implementation uses Redis PubSub and sorted sets. An alternative implementation based on [Redis Streams](https://redis.io/topics/streams-intro) (available in Redis 5.0) is available by setting `backend: :redis_streams`.

### PostgreSQL

message_bus also supports PostgreSQL as the backend:
Expand Down
6 changes: 5 additions & 1 deletion Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,18 @@ run_spec = proc do |backend|
end
end

task spec: [:spec_memory, :spec_redis, :spec_postgres, :spec_client_js, :rubocop]
task spec: [:spec_memory, :spec_redis, :spec_redis_streams, :spec_postgres, :spec_client_js, :rubocop]

task spec_client_js: 'jasmine:ci'

task :spec_redis do
run_spec.call('redis')
end

task :spec_redis_streams do
run_spec.call('redis_streams')
end

task :spec_memory do
run_spec.call('memory')
end
Expand Down
Loading