Add facilities for diffing large lists

.github/workflows/ci.yml

@@ -13,13 +13,19 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        otp: [25.2, 21.3.8.21]
+        otp: [27, 21]
     container:
       image: erlang:${{ matrix.otp }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+    - name: Fix repository ownership
+      run: git config --global --add safe.directory /__w/snabbkaffe/snabbkaffe
     - name: Compile and run tests
-      run: make
+      run: |
+        export DEBIAN_FRONTEND=noninteractive
+        apt-get update
+        apt-get install -y gawk
+        make
     - name: Test with concuerror
       run: make concuerror_test

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.0.10
+
+### Features
+
+- Added new module `snabbkaffe_diff` that helps to compare large lists
+- Added a new macro `?defer_assert(...)` that allows the run stage to continue after encountering a failure, and fails the testcase in the check stage instead.
+
+### Fixes
+- Quote a new reserved word (`maybe`) for compatibility with OTP27
+
 ## 1.0.9
 ### Features
 - Now it is possible to redefine what `?snk_kind` macro translates to in prod mode by defining `SNK_PROD_KIND` macro.

doc/src/extract_tests

@@ -1,16 +1,16 @@
-#!/usr/bin/awk -f
+#!/usr/bin/gawk -f
 # Poor man's TANGLE
 BEGIN {
     print "%% Generated code, do not edit"
+    # print gensub(/(.+)\.md/, "\\1", FILENAME)
+    print "-module(" gensub(/.*\/([^/]+)\.md/, "\\1_example", 1, ARGV[1]) ")."
     printLine = 0
     isSrcBlockStart = 0
-    isEmpty = 1
 }
 
 {
-    if (isSrcBlockStart && ($0 ~ /_test_?\(\) *->$/ || $0 ~ /^-module/)) {
+    if (isSrcBlockStart && ($0 ~ /_test_?\(\) *->$/ || $0 ~ /^-include/)) {
         # Beginning of the src block
-        isEmpty = 0
         printLine = 1
     } else if ($0 ~ /^``` *$/) {
         # End of the src block
@@ -22,10 +22,3 @@ BEGIN {
 
     isSrcBlockStart = $0 ~ /^```erlang/
 }
-
-END {
-    # Generate a dummy module if nothing has been extracted from the file:
-    if (isEmpty) {
-        system("echo \"-module($(basename " FILENAME " .md)_example).\"")
-    }
-}

doc/src/fault_injection.md

@@ -14,8 +14,6 @@ Note: injected crashes are global, they work on the remote nodes.
 Example:
 
 ```erlang
--module(fault_injection_example).
-
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 -include_lib("eunit/include/eunit.hrl").
 

doc/src/offline_analysis.md

@@ -14,8 +14,6 @@ It finds pairs of events matching "cause" and "effect" patterns, and verifies th
 For example, the following test verifies that every received message eventually gets processed:
 
 ```erlang
--module(offline_analysis_example).
-
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
@@ -260,3 +258,10 @@ projection_test() ->
 
 `snabbkaffe:unique(Trace)` will raise an exception of some event in the trace is repeated.
 It ignores the timestamps.
+
+## snabbkaffe_diff
+
+`snabbkaffe_diff` is a helper module that contains routines for comparison of Erlang terms (currently it focuses on the lists).
+
+Functions such as `snabbkaffe_diff:assert_lists_eq/3` and `snabbkaffe_diff:diff_lists/3` help to find and highlight differences between two large lists.
+They try to provide the context while limiting the amount of logs printed when the number of differences is large.

doc/src/running.md

@@ -7,15 +7,13 @@ Most of the work is done by `?check_trace` macro, which can be placed inside eun
 In the most basic form, Snabbkaffe tests look like this:
 
 ```erlang
--module(running_example).
-
--compile(nowarn_export_all).
--compile(export_all).
-
 -include_lib("proper/include/proper.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
+-compile(nowarn_export_all).
+-compile(export_all).
+
 basic_test() ->
   ?check_trace(
      %% Run stage:
@@ -162,3 +160,25 @@ simple_prop() ->
 ```
 
 It is equivalent to the previous example.
+
+## Deferring asserts
+
+Snabbkaffe provides a useful macro `?defer_assert(BODY)` that allows the run stage to continue after encountering an error.
+This is useful for checking multiple properties at once.
+
+Deferred assertions that fail instead are reported as error in the check stage:
+
+```erlang
+deferred_assertion_test() ->
+  ?assertError(
+    _,
+    ?check_trace(
+      begin
+          ?defer_assert(?assert(false, "First assert")),
+          ?defer_assert(?assert(false, "Second assert"))
+      end,
+      [])
+  ).
+```
+
+In the above example both asserts are executed, the run stage succeeds, while the check stage fails.

doc/src/scheduling_injection.md

@@ -22,8 +22,6 @@ Parameters:
 
 Example:
 ```erlang
--module(scheduling_injection_example).
-
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 

doc/src/waiting_events.md

@@ -14,8 +14,6 @@ In the simplest case, `?block_until` macro can be used.
 As the name suggests, it blocks execution of the testcase until an event matching a pattern is emitted:
 
 ```erlang
--module(waiting_events_example).
-
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 

include/common.hrl

@@ -13,4 +13,11 @@
 -define(snk_span, '$span').
 -endif.
 
+%% Redefining this macro should be impossible, because when snabbkaffe
+%% library is compiled with different settings, it would lose the
+%% events.
+-define(snk_deferred_assert, 'Deferred assertion failed').
+
+-compile(nowarn_update_literal).
+
 -endif.

include/snabbkaffe.hrl

@@ -4,4 +4,14 @@
 -include("trace.hrl").
 -include("test_macros.hrl").
 
+-define(defer_assert(BODY),
+        (fun() ->
+             try BODY
+             catch
+               EC:Err:Stack ->
+                 ?tp(error, ?snk_deferred_assert, #{EC => Err, stacktrace => Stack})
+             end,
+             ok
+         end)()).
+
 -endif.

src/snabbkaffe.erl

@@ -1,4 +1,4 @@
-%% Copyright 2021-2023 snabbkaffe contributors
+%% Copyright 2021-2024 snabbkaffe contributors
 %% Copyright 2019-2020 Klarna Bank AB
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
@@ -101,7 +101,7 @@
 -type maybe_pair() :: {pair, timed_event(), timed_event()}
                     | {unmatched_cause | unmatched_effect, timed_event()}.
 
--type maybe(A) :: {just, A} | nothing.
+-type 'maybe'(A) :: {just, A} | nothing.
 
 -type run_config() ::
         #{ bucket          => integer()
@@ -123,8 +123,8 @@
                            | [trace_spec(Result) | {string(), trace_spec(Result)}].
 
 -export_type([ kind/0, timestamp/0, event/0, timed_event/0, trace/0
-             , maybe_pair/0, maybe/1, metric/0, run_config/0, predicate/0
-             , predicate2/0, trace_spec/1, trace_specs/1
+             , maybe_pair/0, 'maybe'/1, metric/0, run_config/0, predicate/0
+             , predicate2/0, trace_spec/1, trace_specs/1, filter/0
              ]).
 
 %%====================================================================
@@ -762,7 +762,8 @@ run_stage(Run, Config) ->
 -spec check_stage(trace_specs(Result), Result, trace(), run_config()) -> boolean() | {error, _}.
 check_stage(Fun, Result, Trace, Config) when is_function(Fun) ->
   check_stage([{"check stage", Fun}], Result, Trace, Config);
-check_stage(Specs, Result, Trace, Config) ->
+check_stage(Specs0, Result, Trace, Config) ->
+  Specs = [{"Deferred asserts", fun check_deferred/1} | Specs0],
   Failed = [Spec || Spec <- Specs, not run_trace_spec(Spec, Result, Trace, Config)],
   case Failed of
     [] ->
@@ -773,6 +774,12 @@ check_stage(Specs, Result, Trace, Config) ->
       {error, check_stage_failed}
   end.
 
+check_deferred(Trace) ->
+  case ?of_kind(?snk_deferred_assert, Trace) of
+    [] -> true;
+    _  -> false
+  end.
+
 %% @private
 run_trace_spec(Spec, Result, Trace, Config) ->
   case Spec of

src/snabbkaffe_collector.erl

@@ -1,4 +1,4 @@
-%% Copyright 2021-2023 snabbkaffe contributors
+%% Copyright 2021-2024 snabbkaffe contributors
 %% Copyright 2019-2020 Klarna Bank AB
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
@@ -44,7 +44,7 @@
 -export([ do_forward_trace/1
         ]).
 
--export_type([async_action/0, subscription/0]).
+-export_type([async_action/0, subscription/0, datapoints/0]).
 
 -define(SERVER, ?MODULE).
 
@@ -116,7 +116,7 @@ get_stats() ->
   gen_server:call(?SERVER, get_stats, infinity).
 
 %% NOTE: Concuerror only supports `Timeout=0'
--spec flush_trace() -> snabbkaffe:timed_trace().
+-spec flush_trace() -> snabbkaffe:trace().
 flush_trace() ->
   {ok, Trace} = gen_server:call(?SERVER, flush_trace, infinity),
   Trace.