Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add ability to configure global Redis namespace prefix #134

Merged
merged 9 commits into from
May 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions src/mosquito/backend.cr
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ module Mosquito
end

def self.build_key(*parts)
KeyBuilder.build KEY_PREFIX, *parts
KeyBuilder.build Mosquito.configuration.global_prefix, KEY_PREFIX, *parts
end

def build_key(*parts)
Expand Down Expand Up @@ -78,7 +78,7 @@ module Mosquito
abstract def dequeue : JobRun?
abstract def schedule(job_run : JobRun, at scheduled_time : Time) : JobRun
abstract def deschedule : Array(JobRun)
abstract def finish(job_run : JobRun) # should this be called succeed?
abstract def finish(job_run : JobRun) # should this be called succeed?
abstract def terminate(job_run : JobRun) # should this be called fail?
abstract def flush : Nil
abstract def size(include_dead : Bool = true) : Int64
Expand All @@ -88,6 +88,5 @@ module Mosquito
{% end %}

abstract def scheduled_job_run_time(job_run : JobRun) : String?

end
end
2 changes: 1 addition & 1 deletion src/mosquito/configuration.cr
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ module Mosquito
property use_distributed_lock : Bool = true

property run_from : Array(String) = [] of String
property global_prefix : String? = nil
property backend : Mosquito::Backend.class = Mosquito::RedisBackend

property validated = false
Expand Down Expand Up @@ -45,6 +46,5 @@ module Mosquito
raise message
end
end

end
end
2 changes: 2 additions & 0 deletions src/mosquito/key_builder.cr
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ module Mosquito
end
when Number
id << part.to_s
when Nil
# do nothing
else
raise "#{part.class} is not a keyable type"
end
Expand Down
10 changes: 5 additions & 5 deletions src/mosquito/periodic_job_run.cr
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@ module Mosquito
class PeriodicJobRun
property class : Mosquito::PeriodicJob.class
property interval : Time::Span | Time::MonthSpan
getter metadata : Metadata { Metadata.new(Backend.build_key("periodic_jobs", @class.name)) }

# The last executed timestamp for this periodicjob tracked by the backend.
def last_executed_at?
if timestamp = @metadata["last_executed_at"]?
if timestamp = metadata["last_executed_at"]?
Time.unix(timestamp.to_i)
else
nil
Expand All @@ -26,19 +27,18 @@ module Mosquito
#
# A month is approximated to 2635200 seconds, or 30.5 days.
def last_executed_at=(time : Time)
@metadata["last_executed_at"] = time.to_unix.to_s
metadata["last_executed_at"] = time.to_unix.to_s

case interval_ = interval
when Time::Span
@metadata.delete(in: interval_ * 3)
metadata.delete(in: interval_ * 3)
when Time::MonthSpan
seconds_in_an_average_month = 2_635_200.seconds
@metadata.delete(in: seconds_in_an_average_month * interval_.value * 3)
metadata.delete(in: seconds_in_an_average_month * interval_.value * 3)
end
end

def initialize(@class, @interval)
@metadata = Metadata.new(Backend.build_key("periodic_jobs", @class.name))
end

# Check the last executed timestamp against the current time,
Expand Down
10 changes: 5 additions & 5 deletions src/mosquito/redis_backend.cr
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ module Mosquito
end

def self.delete(key : String, in ttl : Int64 = 0) : Nil
if (ttl > 0)
if ttl > 0
redis.expire key, ttl
else
redis.del key
Expand Down Expand Up @@ -124,14 +124,14 @@ module Mosquito
key = build_key(LIST_OF_QUEUES_KEY)
list_queues = redis.zrange(key, 0, -1).as(Array)

return [] of String unless list_queues.any?
return [] of String if list_queues.empty?

list_queues.compact_map(&.as(String))
end

def self.list_runners : Array(String)
runner_prefix = "mosquito:runners:"
Redis.instance.keys("#{runner_prefix}*")
runner_prefix = build_key(LIST_OF_QUEUES_KEY)
Redis.instance.keys("#{runner_prefix}:*")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice improvement. The use of #keys here is still a performance implication and can abuse redis servers regardless of the search string, but that's an unrelated issue.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps it's possible to use build_key once here, and then cache the value in the getter runner_prefix to improve performance, but I'm not sure.

.map(&.as(String))
.map(&.sub(runner_prefix, ""))
end
Expand Down Expand Up @@ -159,7 +159,7 @@ module Mosquito
time = Time.utc
overdue_job_runs = redis.zrangebyscore(scheduled_q, "0", time.to_unix_ms.to_s).as(Array)

return [] of JobRun unless overdue_job_runs.any?
return [] of JobRun if overdue_job_runs.empty?
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Normally, I'd ask for ameba fixes to be submitted separately, just so the changes history doesn't get too muddled. In this case, 👍 and thank you.


overdue_job_runs.compact_map do |job_run_id|
redis.zrem scheduled_q, job_run_id.to_s
Expand Down
20 changes: 20 additions & 0 deletions test/mosquito/configuration_test.cr
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,24 @@ describe "Mosquito Config" do
assert_equal test_value, Mosquito.configuration.failed_job_ttl
end
end

it "allows setting global_prefix string" do
test_value = "yolo"

Mosquito.temp_config do
Mosquito.configuration.global_prefix = test_value
assert_equal test_value, Mosquito.configuration.global_prefix
Mosquito.configuration.backend.build_key("test").must_equal "yolo:mosquito:test"
end
end

it "allows setting global_prefix nillable" do
test_value = nil

Mosquito.temp_config do
Mosquito.configuration.global_prefix = test_value
assert_equal test_value, Mosquito.configuration.global_prefix
Mosquito.configuration.backend.build_key("test").must_equal "mosquito:test"
end
end
end
Loading