Mas d31 i433re2capture

.gitignore

@@ -10,3 +10,5 @@ cover_*
 .eqc-info
 leveled_data/*
 compile_commands.json
+*parser.erl
+*lexer.erl

rebar.config

@@ -4,11 +4,11 @@
 
 {xref_checks,
   [undefined_function_calls,undefined_functions,
-    locals_not_used,
     deprecated_function_calls, deprecated_functions]}.
 
 {cover_excl_mods,
-  [testutil,
+  [leveled_filterlexer, leveled_filterparser, leveled_evallexer, leveled_evalparser,
+    testutil,
     appdefined_SUITE, basic_SUITE, iterator_SUITE,
     perf_SUITE, recovery_SUITE, riak_SUITE, tictac_SUITE]}.
 
@@ -23,6 +23,7 @@
    ]},
   {test, [{extra_src_dirs, ["test/end_to_end", "test/property"]}
    ]},
+  {perf_fexp, [{erl_opts, [{d, test_filter_expression}]}]},
   {perf_full, [{erl_opts, [{d, performance, riak_fullperf}]}]},
   {perf_mini, [{erl_opts, [{d, performance, riak_miniperf}]}]},
   {perf_prof, [{erl_opts, [{d, performance, riak_profileperf}]}]}

src/leveled_bookie.erl

@@ -696,7 +696,7 @@ book_returnfolder(Pid, RunnerType) ->
                                      IndexVal::term(),
                                      Start::IndexVal,
                                      End::IndexVal,
-                                     ReturnTerms::boolean(),
+                                     ReturnTerms::boolean()|binary(),
                                      TermRegex :: leveled_codec:regular_expression().
 
 book_indexfold(Pid, Constraint, FoldAccT, Range, TermHandling)

src/leveled_codec.erl

@@ -112,10 +112,17 @@
         {index_specs(), infinity|integer()}. % {KeyChanges, TTL}
 -type maybe_lookup() ::
         lookup|no_lookup.
+-type actual_regex() ::
+        reference()|{re_pattern, term(), term(), term(), term()}.
+-type capture_value() :: binary()|integer().
+-type capture_filter_fun() ::
+        fun((#{binary() => capture_value()}) -> boolean()).
+-type capture_eval_fun() ::
+        fun((binary(), binary()) -> #{binary() => capture_value()}).
+-type capture_reference() :: 
+        {eval, capture_eval_fun(), capture_filter_fun()}.
 -type regular_expression() ::
-        {re_pattern, term(), term(), term(), term()}|undefined. 
-    % first element must be re_pattern, but tuple may change legnth with
-    % versions
+        actual_regex()|undefined|capture_reference().
 
 -type value_fetcher() ::
     {fun((pid(), leveled_codec:journal_key()) -> any()),
@@ -155,6 +162,7 @@
             last_moddate/0,
             lastmod_range/0,
             regular_expression/0,
+            actual_regex/0,
             value_fetcher/0,
             proxy_object/0,
             slimmed_key/0
@@ -292,8 +300,9 @@ maybe_accumulate(
     maybe_accumulate(T, Acc, Count, Filter, AccFun).
 
 -spec accumulate_index(
-        {boolean(), undefined|leveled_runner:mp()}, leveled_runner:acc_fun())
-            -> any().
+        {boolean()|binary(),
+        regular_expression()},
+        leveled_runner:acc_fun()) -> any().
 accumulate_index({false, undefined}, FoldKeysFun) ->
     fun({?IDX_TAG, Bucket, _IndexInfo, ObjKey}, _Value, Acc) ->
         FoldKeysFun(Bucket, ObjKey, Acc)
@@ -302,9 +311,19 @@ accumulate_index({true, undefined}, FoldKeysFun) ->
     fun({?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) ->
         FoldKeysFun(Bucket, {IdxValue, ObjKey}, Acc)
     end;
+accumulate_index(
+        {AddTerm, {eval, EvalFun, FilterFun}}, FoldKeysFun) ->
+    fun({?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) ->
+        CptMap = EvalFun(IdxValue, ObjKey),
+        check_captured_terms(
+            CptMap,
+            FilterFun, AddTerm, FoldKeysFun,
+            Bucket, IdxValue, ObjKey,
+            Acc)
+    end;
 accumulate_index({AddTerm, TermRegex}, FoldKeysFun) ->
     fun({?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) ->
-        case leveled_util:regex_run(IdxValue, TermRegex) of
+        case leveled_util:regex_run(IdxValue, TermRegex, []) of
             nomatch ->
                 Acc;
             _ ->
@@ -317,6 +336,29 @@ accumulate_index({AddTerm, TermRegex}, FoldKeysFun) ->
         end
     end.
 
+check_captured_terms(
+        CptMap, FilterFun, AddTerm, FoldKeysFun, B, IdxValue, ObjKey, Acc) ->
+    case FilterFun(CptMap) of
+        true ->
+            case AddTerm of
+                true ->
+                    FoldKeysFun(B, {IdxValue, ObjKey}, Acc);
+                false ->
+                    FoldKeysFun(B, ObjKey, Acc);
+                CptKey when is_binary(CptKey) ->
+                    case maps:get(CptKey, CptMap, undefined) of
+                        undefined ->
+                            Acc;
+                        CptValue ->  
+                            FoldKeysFun(B, {CptValue, ObjKey}, Acc)
+                    end
+            end;
+        false ->
+            Acc
+    end.
+
+
+
 -spec key_dominates(ledger_kv(), ledger_kv()) -> boolean().
 %% @doc
 %% When comparing two keys in the ledger need to find if one key comes before