Skip to content

Commit f35dbe9

Browse files
martinsumnermartinsumner
committed
Merge branch 'develop-3.4' into mas-d32-leveled.i388-logger
2 parents ae48016 + 5db277b commit f35dbe9

8 files changed

+311
-175
lines changed

‎src/leveled_cdb.erl

+35-21
Original file line numberDiff line numberDiff line change
@@ -1720,11 +1720,11 @@ build_hashtree_binary(SlotMap, IndexLength) ->
17201720

17211721
build_hashtree_binary([], IdxLen, SlotPos, Bin) ->
17221722
case SlotPos of
1723-
IdxLen ->
1724-
lists:reverse(Bin);
1723+
N when N == IdxLen ->
1724+
Bin;
17251725
N when N < IdxLen ->
17261726
ZeroLen = (IdxLen - N) * 64,
1727-
lists:reverse([<<0:ZeroLen>>|Bin])
1727+
[<<0:ZeroLen>>|Bin]
17281728
end;
17291729
build_hashtree_binary([{TopSlot, TopBin}|SlotMapTail], IdxLen, SlotPos, Bin) ->
17301730
case TopSlot of
@@ -1772,21 +1772,23 @@ write_hash_tables([], _HashTree, _CurrPos, _BasePos,
17721772
IndexList, HT_BinList, {T1, T2, T3}) ->
17731773
leveled_log:log(cdb14, [T1, T2, T3]),
17741774
IL = lists:reverse(IndexList),
1775-
{IL, list_to_binary(HT_BinList)};
1775+
{IL, list_to_binary(lists:reverse(HT_BinList))};
17761776
write_hash_tables([Index|Rest], HashTree, CurrPos, BasePos,
17771777
IndexList, HT_BinList, Timers) ->
17781778
SW1 = os:timestamp(),
17791779
SlotMap = to_slotmap(HashTree, Index),
17801780
T1 = timer:now_diff(os:timestamp(), SW1) + element(1, Timers),
17811781
case SlotMap of
17821782
[] ->
1783-
write_hash_tables(Rest,
1784-
HashTree,
1785-
CurrPos,
1786-
BasePos,
1787-
[{Index, BasePos, 0}|IndexList],
1788-
HT_BinList,
1789-
Timers);
1783+
write_hash_tables(
1784+
Rest,
1785+
HashTree,
1786+
CurrPos,
1787+
BasePos,
1788+
[{Index, BasePos, 0}|IndexList],
1789+
HT_BinList,
1790+
Timers
1791+
);
17901792
_ ->
17911793
SW2 = os:timestamp(),
17921794
IndexLength = length(SlotMap) * 2,
@@ -1795,13 +1797,15 @@ write_hash_tables([Index|Rest], HashTree, CurrPos, BasePos,
17951797
SW3 = os:timestamp(),
17961798
NewSlotBin = build_hashtree_binary(SortedMap, IndexLength),
17971799
T3 = timer:now_diff(os:timestamp(), SW3) + element(3, Timers),
1798-
write_hash_tables(Rest,
1799-
HashTree,
1800-
CurrPos + IndexLength * ?DWORD_SIZE,
1801-
BasePos,
1802-
[{Index, CurrPos, IndexLength}|IndexList],
1803-
HT_BinList ++ NewSlotBin,
1804-
{T1, T2, T3})
1800+
write_hash_tables(
1801+
Rest,
1802+
HashTree,
1803+
CurrPos + IndexLength * ?DWORD_SIZE,
1804+
BasePos,
1805+
[{Index, CurrPos, IndexLength}|IndexList],
1806+
lists:append(NewSlotBin, HT_BinList),
1807+
{T1, T2, T3}
1808+
)
18051809
end.
18061810

18071811

@@ -1947,7 +1951,12 @@ build_hashtree_bunchedatend_binary_test() ->
19471951
{14, <<15:32, 500:32>>},
19481952
{15, <<16:32, 600:32>>},
19491953
{15, <<17:32, 700:32>>}],
1950-
Bin = list_to_binary(build_hashtree_binary(SlotMap, 16)),
1954+
Bin =
1955+
list_to_binary(
1956+
lists:reverse(
1957+
build_hashtree_binary(SlotMap, 16)
1958+
)
1959+
),
19511960
ExpBinP1 = <<16:32, 600:32, 10:32, 0:32, 17:32, 700:32, 0:64>>,
19521961
ExpBinP2 = <<11:32, 100:32, 0:192, 12:32, 200:32, 13:32, 300:32, 0:256>>,
19531962
ExpBinP3 = <<14:32, 400:32, 15:32, 500:32>>,
@@ -1963,7 +1972,12 @@ build_hashtree_bunchedatstart_binary_test() ->
19631972
{6, <<15:32, 500:32>>},
19641973
{7, <<16:32, 600:32>>},
19651974
{8, <<17:32, 700:32>>}],
1966-
Bin = list_to_binary(build_hashtree_binary(SlotMap, 16)),
1975+
Bin =
1976+
list_to_binary(
1977+
lists:reverse(
1978+
build_hashtree_binary(SlotMap, 16)
1979+
)
1980+
),
19671981
ExpBinP1 = <<0:64, 10:32, 0:32, 11:32, 100:32, 12:32, 200:32>>,
19681982
ExpBinP2 = <<13:32, 300:32, 14:32, 400:32, 15:32, 500:32, 16:32, 600:32>>,
19691983
ExpBinP3 = <<17:32, 700:32, 0:448>>,
@@ -1986,7 +2000,7 @@ build_hashtree_test() ->
19862000
[<<2424915712:32, 300:32>>, <<0:64>>] ++
19872001
[<<2424903936:32, 400:32>>, <<2424907008:32, 500:32>>] ++
19882002
[<<2424913408:32, 600:32>>],
1989-
?assertMatch(ExpOut, BinList).
2003+
?assertMatch(ExpOut, lists:reverse(BinList)).
19902004

19912005

19922006
find_firstzero_test() ->

‎src/leveled_ebloom.erl

+2-2
Original file line numberDiff line numberDiff line change
@@ -92,11 +92,11 @@ check_hash({_SegHash, Hash}, BloomBin) when is_binary(BloomBin)->
9292
map_hashes([], HashListTuple, _SlotCount) ->
9393
HashListTuple;
9494
map_hashes([Hash|Rest], HashListTuple, SlotCount) ->
95-
{Slot, Hashes} = split_hash(element(2, Hash), SlotCount),
95+
{Slot, [H0, H1]} = split_hash(element(2, Hash), SlotCount),
9696
SlotHL = element(Slot + 1, HashListTuple),
9797
map_hashes(
9898
Rest,
99-
setelement(Slot + 1, HashListTuple, Hashes ++ SlotHL),
99+
setelement(Slot + 1, HashListTuple, [H0, H1 | SlotHL]),
100100
SlotCount).
101101

102102
-spec split_hash(external_hash(), slot_count())

‎src/leveled_head.erl

+11-9
Original file line numberDiff line numberDiff line change
@@ -421,9 +421,9 @@ decode_maybe_binary(<<0, Bin/binary>>) ->
421421
decode_maybe_binary(<<_Other:8, Bin/binary>>) ->
422422
Bin.
423423

424-
-spec diff_index_data([{binary(), index_value()}],
425-
[{binary(), index_value()}]) ->
426-
[{index_op(), binary(), index_value()}].
424+
-spec diff_index_data(
425+
[{binary(), index_value()}], [{binary(), index_value()}]) ->
426+
[{index_op(), binary(), index_value()}].
427427
diff_index_data(OldIndexes, AllIndexes) ->
428428
OldIndexSet = ordsets:from_list(OldIndexes),
429429
AllIndexSet = ordsets:from_list(AllIndexes),
@@ -435,18 +435,20 @@ diff_specs_core(AllIndexSet, OldIndexSet) ->
435435
RemoveIndexSet =
436436
ordsets:subtract(OldIndexSet, AllIndexSet),
437437
NewIndexSpecs =
438-
assemble_index_specs(ordsets:subtract(NewIndexSet, OldIndexSet),
439-
add),
438+
assemble_index_specs(
439+
ordsets:subtract(NewIndexSet, OldIndexSet),
440+
add
441+
),
440442
RemoveIndexSpecs =
441-
assemble_index_specs(RemoveIndexSet,
442-
remove),
443+
assemble_index_specs(RemoveIndexSet, remove),
443444
NewIndexSpecs ++ RemoveIndexSpecs.
444445

445446
%% @doc Assemble a list of index specs in the
446447
%% form of triplets of the form
447448
%% {IndexOperation, IndexField, IndexValue}.
448-
-spec assemble_index_specs([{binary(), binary()}], index_op()) ->
449-
[{index_op(), binary(), binary()}].
449+
-spec assemble_index_specs(
450+
[{binary(), binary()}], index_op()) ->
451+
[{index_op(), binary(), binary()}].
450452
assemble_index_specs(Indexes, IndexOp) ->
451453
[{IndexOp, Index, Value} || {Index, Value} <- Indexes].
452454

‎src/leveled_pclerk.erl

+64-45
Original file line numberDiff line numberDiff line change
@@ -216,21 +216,27 @@ merge(SrcLevel, Manifest, RootPath, OptsSST) ->
216216
Src =
217217
leveled_pmanifest:mergefile_selector(Manifest, SrcLevel, SelectMethod),
218218
NewSQN = leveled_pmanifest:get_manifest_sqn(Manifest) + 1,
219-
SinkList = leveled_pmanifest:merge_lookup(Manifest,
220-
SrcLevel + 1,
221-
Src#manifest_entry.start_key,
222-
Src#manifest_entry.end_key),
219+
SinkList =
220+
leveled_pmanifest:merge_lookup(
221+
Manifest,
222+
SrcLevel + 1,
223+
Src#manifest_entry.start_key,
224+
Src#manifest_entry.end_key
225+
),
223226
Candidates = length(SinkList),
224227
leveled_log:log(pc008, [SrcLevel, Candidates]),
225228
case Candidates of
226229
0 ->
227230
NewLevel = SrcLevel + 1,
228231
leveled_log:log(pc009, [Src#manifest_entry.filename, NewLevel]),
229232
leveled_sst:sst_switchlevels(Src#manifest_entry.owner, NewLevel),
230-
Man0 = leveled_pmanifest:switch_manifest_entry(Manifest,
231-
NewSQN,
232-
SrcLevel,
233-
Src),
233+
Man0 =
234+
leveled_pmanifest:switch_manifest_entry(
235+
Manifest,
236+
NewSQN,
237+
SrcLevel,
238+
Src
239+
),
234240
{Man0, []};
235241
_ ->
236242
SST_RP = leveled_penciller:sst_rootpath(RootPath),
@@ -252,69 +258,82 @@ notify_deletions([Head|Tail], Penciller) ->
252258
%%
253259
%% SrcLevel is the level of the src sst file, the sink should be srcLevel + 1
254260

255-
perform_merge(Manifest,
256-
Src, SinkList, SrcLevel,
257-
RootPath, NewSQN,
258-
OptsSST) ->
261+
perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN, OptsSST) ->
259262
leveled_log:log(pc010, [Src#manifest_entry.filename, NewSQN]),
260263
SrcList = [{next, Src, all}],
261264
MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner),
262265
SinkLevel = SrcLevel + 1,
263266
SinkBasement = leveled_pmanifest:is_basement(Manifest, SinkLevel),
264267
Additions =
265-
do_merge(SrcList, SinkList,
266-
SinkLevel, SinkBasement,
267-
RootPath, NewSQN, MaxSQN,
268-
OptsSST,
269-
[]),
268+
do_merge(
269+
SrcList, SinkList,
270+
SinkLevel, SinkBasement,
271+
RootPath, NewSQN, MaxSQN,
272+
OptsSST,
273+
[]
274+
),
270275
RevertPointerFun =
271276
fun({next, ME, _SK}) ->
272277
ME
273278
end,
274279
SinkManifestList = lists:map(RevertPointerFun, SinkList),
275-
Man0 = leveled_pmanifest:replace_manifest_entry(Manifest,
276-
NewSQN,
277-
SinkLevel,
278-
SinkManifestList,
279-
Additions),
280-
Man2 = leveled_pmanifest:remove_manifest_entry(Man0,
281-
NewSQN,
282-
SrcLevel,
283-
Src),
280+
Man0 =
281+
leveled_pmanifest:replace_manifest_entry(
282+
Manifest,
283+
NewSQN,
284+
SinkLevel,
285+
SinkManifestList,
286+
Additions
287+
),
288+
Man2 =
289+
leveled_pmanifest:remove_manifest_entry(
290+
Man0,
291+
NewSQN,
292+
SrcLevel,
293+
Src
294+
),
284295
{Man2, [Src|SinkManifestList]}.
285296

286297
do_merge([], [], SinkLevel, _SinkB, _RP, NewSQN, _MaxSQN, _Opts, Additions) ->
287298
leveled_log:log(pc011, [NewSQN, SinkLevel, length(Additions)]),
288-
Additions;
299+
lists:reverse(Additions);
289300
do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, OptsSST, Additions) ->
290-
FileName = leveled_penciller:sst_filename(NewSQN,
291-
SinkLevel,
292-
length(Additions)),
301+
FileName =
302+
leveled_penciller:sst_filename(
303+
NewSQN, SinkLevel, length(Additions)
304+
),
293305
leveled_log:log(pc012, [NewSQN, FileName, SinkB]),
294306
TS1 = os:timestamp(),
295307
case leveled_sst:sst_newmerge(RP, FileName,
296308
KL1, KL2, SinkB, SinkLevel, MaxSQN,
297309
OptsSST) of
298310
empty ->
299311
leveled_log:log(pc013, [FileName]),
300-
do_merge([], [],
301-
SinkLevel, SinkB,
302-
RP, NewSQN, MaxSQN,
303-
OptsSST,
304-
Additions);
312+
do_merge(
313+
[], [],
314+
SinkLevel, SinkB,
315+
RP, NewSQN, MaxSQN,
316+
OptsSST,
317+
Additions
318+
);
305319
{ok, Pid, Reply, Bloom} ->
306320
{{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply,
307-
Entry = #manifest_entry{start_key=SmallestKey,
308-
end_key=HighestKey,
309-
owner=Pid,
310-
filename=FileName,
311-
bloom=Bloom},
321+
Entry =
322+
#manifest_entry{
323+
start_key=SmallestKey,
324+
end_key=HighestKey,
325+
owner=Pid,
326+
filename=FileName,
327+
bloom=Bloom
328+
},
312329
leveled_log:log_timer(pc015, [], TS1),
313-
do_merge(KL1Rem, KL2Rem,
314-
SinkLevel, SinkB,
315-
RP, NewSQN, MaxSQN,
316-
OptsSST,
317-
Additions ++ [Entry])
330+
do_merge(
331+
KL1Rem, KL2Rem,
332+
SinkLevel, SinkB,
333+
RP, NewSQN, MaxSQN,
334+
OptsSST,
335+
[Entry|Additions]
336+
)
318337
end.
319338

320339
-spec grooming_scorer(

‎src/leveled_penciller.erl

+3
Original file line numberDiff line numberDiff line change
@@ -2001,6 +2001,9 @@ format_status_test() ->
20012001
?assertMatch(redacted, ST#state.levelzero_cache),
20022002
?assertMatch(redacted, ST#state.levelzero_index),
20032003
?assertMatch(redacted, ST#state.levelzero_astree),
2004+
NormStatus = format_status(#{reason => normal, state => S}),
2005+
NST = maps:get(state, NormStatus),
2006+
?assert(is_integer(array:size(element(2, NST#state.manifest)))),
20042007
clean_testdir(RootPath).
20052008

20062009
close_no_crash_test_() ->

0 commit comments

Comments
 (0)