Skip to content

Commit

Permalink
storage: add test for time based retention capacity metric
Browse files Browse the repository at this point in the history
Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
  • Loading branch information
dotnwat committed May 19, 2023
1 parent b6a772e commit 91f59f8
Showing 1 changed file with 46 additions and 0 deletions.
46 changes: 46 additions & 0 deletions tests/rptest/tests/full_disk_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,52 @@ def observed_data_size(pred):
backoff_sec=2)


class LocalDiskReportTimeTest(RedpandaTest):
topics = (TopicSpec(segment_bytes=2**20,
partition_count=1,
retention_bytes=-1,
retention_ms=60 * 60 * 1000,
cleanup_policy=TopicSpec.CLEANUP_DELETE), )

def __init__(self, test_ctx):
extra_rp_conf = dict(log_compaction_interval_ms=3600 * 1000)
super().__init__(test_context=test_ctx, extra_rp_conf=extra_rp_conf)

@cluster(num_nodes=3)
def test_target_min_capacity_wanted_time_based(self):
admin = Admin(self.redpanda)
default_segment_size = admin.get_cluster_config()["log_segment_size"]

# produce 8 mb with 12.5 seconds of sleep. through examination it looks
# like the observed span of time of data ends up being around 17
# seconds, which makes sense because its should be including the time
# between when the topic is created during test setup. this works out to
# about 493447 bytes per second. The retention time is 3600 seconds so
# there should be around 1776411105 bytes of capacity needed.
kafka_tools = KafkaCliTools(self.redpanda)
for _ in range(8):
kafka_tools.produce(self.topic, 1024, 1024, acks=-1)
sleep(1.5)

node = self.redpanda.nodes[0]
reported = admin.get_local_storage_usage(
node)["target_min_capacity_wanted"]

# params
size = 8 * 2**20
time = 17
retention = 3600
expected = retention * (size / time)

# factor in the 2 segments worth of space for controller log
diff = abs(reported - expected - 2 * default_segment_size)

# there is definitely going to be some fuzz factor needed here and may
# need updated, but after many runs 50mb was a good amount of slack.
assert diff <= (50 *
2**20), f"{diff}, {default_segment_size}, {reported}"


class LocalDiskReportTest(RedpandaTest):
topics = (
TopicSpec(segment_bytes=2**30,
Expand Down

0 comments on commit 91f59f8

Please sign in to comment.