rocksdb_put_cf() return "No space left on device" when compacting with 60G free space

[meilihao]

env:

• os: oracle linux 7.9
• rocksdb: 9.1.2

rocksdb options, and CFOptions "default" is saving data.:

# This is a RocksDB option file.
#
# For detailed file format spec, please refer to the example file
# in examples/rocksdb_option_file_example.ini
#

[Version]
  rocksdb_version=9.1.2
  options_file_version=1.1

[DBOptions]
  max_background_flushes=1
  compaction_readahead_size=2097152
  wal_bytes_per_sync=524288
  bytes_per_sync=1048576
  max_open_files=40960
  stats_history_buffer_size=1048576
  stats_dump_period_sec=600
  max_background_compactions=2
  max_total_wal_size=4294967296
  max_background_jobs=4
  strict_bytes_per_sync=false
  delayed_write_rate=16777216
  writable_file_max_buffer_size=1048576
  max_subcompactions=3
  avoid_flush_during_shutdown=false
  stats_persist_period_sec=600
  delete_obsolete_files_period_micros=21600000000
  enforce_single_del_contracts=true
  file_checksum_gen_factory=nullptr
  bgerror_resume_retry_interval=1000000
  max_bgerror_resume_count=2147483647
  log_readahead_size=0
  avoid_unnecessary_blocking_io=false
  two_write_queues=false
  allow_ingest_behind=false
  avoid_flush_during_recovery=false
  write_thread_max_yield_usec=100
  write_thread_slow_yield_usec=3
  manual_wal_flush=false
  enable_write_thread_adaptive_yield=true
  wal_recovery_mode=kPointInTimeRecovery
  unordered_write=false
  wal_compression=kNoCompression
  enable_pipelined_write=false
  compaction_verify_record_count=true
  allow_mmap_reads=false
  use_direct_io_for_flush_and_compaction=false
  max_write_batch_group_size_bytes=1048576
  WAL_ttl_seconds=0
  db_host_id=__hostname__
  error_if_exists=false
  paranoid_checks=true
  is_fd_close_on_exec=true
  allow_mmap_writes=false
  manifest_preallocation_size=4194304
  use_direct_reads=false
  best_efforts_recovery=false
  flush_verify_memtable_count=true
  info_log_level=INFO_LEVEL
  recycle_log_file_num=0
  enable_thread_tracking=false
  wal_filter=nullptr
  allow_data_in_errors=false
  allow_2pc=false
  skip_checking_sst_file_sizes_on_db_open=false
  use_adaptive_mutex=false
  dump_malloc_stats=false
  verify_sst_unique_id_in_manifest=true
  write_dbid_to_manifest=false
  fail_if_options_file_error=true
  atomic_flush=false
  allow_fallocate=true
  advise_random_on_open=true
  create_missing_column_families=true
  random_access_max_buffer_size=1048576
  max_log_file_size=268435456
  create_if_missing=true
  lowest_used_cache_tier=kNonVolatileBlockTier
  track_and_verify_wals_in_manifest=false
  log_file_time_to_roll=0
  max_manifest_file_size=134217728
  skip_stats_update_on_db_open=false
  keep_log_file_num=4
  use_fsync=false
  max_file_opening_threads=16
  table_cache_numshardbits=6
  db_write_buffer_size=0
  WAL_size_limit_MB=0
  allow_concurrent_memtable_write=true
  persist_stats_to_disk=false
  

[CFOptions "default"]
  bottommost_compression_opts={checksum=false;max_compressed_bytes_per_kb=896;strategy=0;window_bits=-14;max_dict_bytes=0;zstd_max_train_bytes=0;parallel_threads=1;level=32767;enabled=false;use_zstd_dict_trainer=true;max_dict_buffer_bytes=0;}
  compression_opts={checksum=false;max_compressed_bytes_per_kb=896;strategy=0;window_bits=-14;max_dict_bytes=0;zstd_max_train_bytes=0;parallel_threads=1;level=32767;enabled=false;use_zstd_dict_trainer=true;max_dict_buffer_bytes=0;}
  experimental_mempurge_threshold=0.000000
  bottommost_compression=kNoCompression
  prepopulate_blob_cache=kDisable
  blob_file_starting_level=0
  blob_compaction_readahead_size=0
  blob_garbage_collection_age_cutoff=0.250000
  blob_compression_type=kNoCompression
  block_protection_bytes_per_key=0
  blob_file_size=268435456
  min_blob_size=0
  default_write_temperature=kUnknown
  arena_block_size=1048576
  max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
  ttl=2592000
  inplace_update_num_locks=10000
  memtable_protection_bytes_per_key=0
  level0_slowdown_writes_trigger=30
  max_compaction_bytes=2147483648
  target_file_size_multiplier=1
  periodic_compaction_seconds=0
  sample_for_compression=0
  paranoid_file_checks=false
  disable_auto_compactions=false
  level0_file_num_compaction_trigger=4
  compaction_options_fifo={file_temperature_age_thresholds=;max_table_files_size=1073741824;age_for_warm=0;allow_compaction=false;}
  report_bg_io_stats=false
  bottommost_file_compaction_delay=0
  level0_stop_writes_trigger=36
  max_successive_merges=0
  enable_blob_garbage_collection=false
  hard_pending_compaction_bytes_limit=1099511627776
  memtable_max_range_deletions=0
  soft_pending_compaction_bytes_limit=206158430208
  max_write_buffer_number=5
  strict_max_successive_merges=false
  max_bytes_for_level_base=536870912
  memtable_huge_page_size=0
  compression_per_level=kNoCompression:kNoCompression:kNoCompression:kNoCompression:kNoCompression:kZSTD:kZSTD
  blob_garbage_collection_force_threshold=1.000000
  write_buffer_size=134217728
  memtable_prefix_bloom_size_ratio=0.000000
  memtable_whole_key_filtering=false
  max_bytes_for_level_multiplier=10.000000
  max_sequential_skip_in_iterations=8
  enable_blob_files=false
  compression=kNoCompression
  prefix_extractor=rocksdb.Noop
  target_file_size_base=134217728
  compaction_options_universal={allow_trivial_move=false;incremental=false;stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;size_ratio=1;min_merge_width=2;max_merge_width=4294967295;max_size_amplification_percent=200;}
  last_level_temperature=kUnknown
  sst_partitioner_factory=nullptr
  compaction_pri=kMinOverlappingRatio
  compaction_filter_factory=nullptr
  merge_operator=nullptr
  persist_user_defined_timestamps=true
  comparator=leveldb.BytewiseComparator
  inplace_update_support=false
  compaction_filter=nullptr
  table_properties_collectors={{id=CompactOnDeletionCollector;deletion_ratio=0.000000;deletion_trigger=1000;window_size=10000;}}
  max_write_buffer_size_to_maintain=0
  compaction_style=kCompactionStyleLevel
  min_write_buffer_number_to_merge=2
  optimize_filters_for_hits=false
  preclude_last_level_data_seconds=0
  level_compaction_dynamic_level_bytes=true
  force_consistency_checks=true
  max_write_buffer_number_to_maintain=0
  num_levels=7
  default_temperature=kUnknown
  bloom_locality=0
  preserve_internal_time_seconds=0
  memtable_insert_with_hint_prefix_extractor=nullptr
  memtable_factory=SkipListFactory
  table_factory=BlockBasedTable
  
[TableOptions/BlockBasedTable "default"]
  initial_auto_readahead_size=8192
  max_auto_readahead_size=262144
  metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;}
  block_align=false
  read_amp_bytes_per_bit=0
  verify_compression=false
  detect_filter_construct_corruption=false
  num_file_reads_for_auto_readahead=2
  format_version=6
  checksum=kCRC32c
  index_shortening=kShortenSeparators
  index_type=kBinarySearch
  enable_index_compression=true
  flush_block_policy_factory=FlushBlockBySizePolicyFactory
  cache_index_and_filter_blocks_with_high_priority=true
  no_block_cache=false
  data_block_index_type=kDataBlockBinarySearch
  block_restart_interval=16
  pin_top_level_index_and_filter=true
  index_block_restart_interval=1
  cache_index_and_filter_blocks=true
  pin_l0_filter_and_index_blocks_in_cache=true
  metadata_block_size=4096
  data_block_hash_table_util_ratio=0.750000
  block_size=32768
  block_size_deviation=10
  prepopulate_block_cache=kDisable
  partition_filters=false
  optimize_filters_for_memory=false
  use_delta_encoding=true
  filter_policy=nullptr
  whole_key_filtering=false
  

[CFOptions "other"]
  bottommost_compression_opts={checksum=false;max_compressed_bytes_per_kb=896;strategy=0;window_bits=-14;max_dict_bytes=0;zstd_max_train_bytes=0;parallel_threads=1;level=32767;enabled=false;use_zstd_dict_trainer=true;max_dict_buffer_bytes=0;}
  compression_opts={checksum=false;max_compressed_bytes_per_kb=896;strategy=0;window_bits=-14;max_dict_bytes=0;zstd_max_train_bytes=0;parallel_threads=1;level=32767;enabled=false;use_zstd_dict_trainer=true;max_dict_buffer_bytes=0;}
  experimental_mempurge_threshold=0.000000
  bottommost_compression=kNoCompression
  prepopulate_blob_cache=kDisable
  blob_file_starting_level=0
  blob_compaction_readahead_size=0
  blob_garbage_collection_age_cutoff=0.250000
  blob_compression_type=kNoCompression
  block_protection_bytes_per_key=0
  blob_file_size=268435456
  min_blob_size=0
  default_write_temperature=kUnknown
  arena_block_size=1048576
  max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
  ttl=2592000
  inplace_update_num_locks=10000
  memtable_protection_bytes_per_key=0
  level0_slowdown_writes_trigger=30
  max_compaction_bytes=2147483648
  target_file_size_multiplier=1
  periodic_compaction_seconds=0
  sample_for_compression=0
  paranoid_file_checks=false
  disable_auto_compactions=false
  level0_file_num_compaction_trigger=4
  compaction_options_fifo={file_temperature_age_thresholds=;max_table_files_size=1073741824;age_for_warm=0;allow_compaction=false;}
  report_bg_io_stats=false
  bottommost_file_compaction_delay=0
  level0_stop_writes_trigger=36
  max_successive_merges=0
  enable_blob_garbage_collection=false
  hard_pending_compaction_bytes_limit=1099511627776
  memtable_max_range_deletions=0
  soft_pending_compaction_bytes_limit=206158430208
  max_write_buffer_number=5
  strict_max_successive_merges=false
  max_bytes_for_level_base=536870912
  memtable_huge_page_size=0
  compression_per_level=kNoCompression:kNoCompression:kNoCompression:kNoCompression:kNoCompression:kZSTD:kZSTD
  blob_garbage_collection_force_threshold=1.000000
  write_buffer_size=134217728
  memtable_prefix_bloom_size_ratio=0.100000
  memtable_whole_key_filtering=false
  max_bytes_for_level_multiplier=10.000000
  max_sequential_skip_in_iterations=8
  enable_blob_files=false
  compression=kNoCompression
  prefix_extractor=rocksdb.FixedPrefix.1
  target_file_size_base=134217728
  compaction_options_universal={allow_trivial_move=false;incremental=false;stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;size_ratio=1;min_merge_width=2;max_merge_width=4294967295;max_size_amplification_percent=200;}
  last_level_temperature=kUnknown
  sst_partitioner_factory=nullptr
  compaction_pri=kMinOverlappingRatio
  compaction_filter_factory=nullptr
  merge_operator=nullptr
  persist_user_defined_timestamps=true
  comparator=leveldb.BytewiseComparator
  inplace_update_support=false
  compaction_filter=nullptr
  table_properties_collectors={{id=CompactOnDeletionCollector;deletion_ratio=0.000000;deletion_trigger=1000;window_size=10000;}}
  max_write_buffer_size_to_maintain=0
  compaction_style=kCompactionStyleLevel
  min_write_buffer_number_to_merge=2
  optimize_filters_for_hits=true
  preclude_last_level_data_seconds=0
  level_compaction_dynamic_level_bytes=true
  force_consistency_checks=true
  max_write_buffer_number_to_maintain=0
  num_levels=7
  default_temperature=kUnknown
  bloom_locality=0
  preserve_internal_time_seconds=0
  memtable_insert_with_hint_prefix_extractor=nullptr
  memtable_factory=SkipListFactory
  table_factory=BlockBasedTable
  
[TableOptions/BlockBasedTable "other"]
  initial_auto_readahead_size=8192
  max_auto_readahead_size=262144
  metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;}
  block_align=false
  read_amp_bytes_per_bit=0
  verify_compression=false
  detect_filter_construct_corruption=false
  num_file_reads_for_auto_readahead=2
  format_version=6
  checksum=kCRC32c
  index_shortening=kShortenSeparators
  index_type=kBinarySearch
  enable_index_compression=true
  flush_block_policy_factory=FlushBlockBySizePolicyFactory
  cache_index_and_filter_blocks_with_high_priority=true
  no_block_cache=false
  data_block_index_type=kDataBlockBinarySearch
  block_restart_interval=16
  pin_top_level_index_and_filter=true
  index_block_restart_interval=1
  cache_index_and_filter_blocks=true
  pin_l0_filter_and_index_blocks_in_cache=true
  metadata_block_size=4096
  data_block_hash_table_util_ratio=0.750000
  block_size=32768
  block_size_deviation=10
  prepopulate_block_cache=kDisable
  partition_filters=false
  optimize_filters_for_memory=false
  use_delta_encoding=true
  filter_policy=bloomfilter
  whole_key_filtering=true

How to evaluate how much free space is needed for rocksdb compaction?

Expected behavior

rocksdb_put_cf() is ok when 60G free

my app log:

ps:

1. filesystem is using zfs filesystem to save my data, all space is 240G.
2. GetDBSizeByConfig() is using the du command to count used space(free=240-179≈60G)

rocksdb log:

Actual behavior

rocksdb_put_cf() return "No space left on device" when compacting with 60G free.

Steps to reproduce the behavior

Insert data into the db using nanosecond timestamp as key, then delete expired data (data older than 1 hour) and compact db every 30 minutes.

[meilihao]

Could anyone give me some advice please?