diff --git a/.circleci/config.yml b/.circleci/config.yml index a76a14fcd..f98022599 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,11 +155,11 @@ jobs: at: . - run: cp -r benchmark/ ~/benchmark_backup/ - run: cp mix.exs ~/benchmark_backup/ - - run: docker run -e MIX_ENV=benchmark -v ./:/root/app -v ~/results:/root/results -w /root/app membraneframeworklabs/docker_membrane mix run benchmark/run.exs /root/results/results.res - - run: git checkout master + - run: docker run -e MIX_ENV=benchmark -v ./:/root/app -v ~/results:/root/results -w /root/app membraneframeworklabs/docker_membrane mix run benchmark/run.exs /root/results/feature_branch_results + - run: git checkout -f master - run: cp ~/benchmark_backup/mix.exs ~/app - - run: docker run -e MIX_ENV=benchmark -v ./:/root/app -v ~/results:/root/results -v ~/benchmark_backup/benchmark:/root/app/benchmark -w /root/app membraneframeworklabs/docker_membrane mix run benchmark/run.exs /root/results/results_ref.res - - run: docker run -e MIX_ENV=benchmark -v ./:/root/app -v ~/results:/root/results -v ~/benchmark_backup/benchmark:/root/app/benchmark -w /root/app membraneframeworklabs/docker_membrane mix run benchmark/compare.exs /root/results/results.res /root/results/results_ref.res + - run: docker run -e MIX_ENV=benchmark -v ./:/root/app -v ~/results:/root/results -v ~/benchmark_backup/benchmark:/root/app/benchmark -w /root/app membraneframeworklabs/docker_membrane mix run benchmark/run.exs /root/results/master_results + - run: docker run -e MIX_ENV=benchmark -v ./:/root/app -v ~/results:/root/results -v ~/benchmark_backup/benchmark:/root/app/benchmark -w /root/app membraneframeworklabs/docker_membrane mix run benchmark/compare.exs /root/results/feature_branch_results /root/results/master_results - run: command: rm ~/results/* when: always diff --git a/benchmark/compare.exs b/benchmark/compare.exs index f82536f66..104522a5e 100644 --- a/benchmark/compare.exs +++ b/benchmark/compare.exs @@ -8,7 +8,7 @@ defmodule Benchmark.Compare do require Logger - def run(results, ref_results) do + def run(results, ref_results, results_name, ref_results_name) do if Map.keys(results) != Map.keys(ref_results), do: raise("Incompatible performance test result files!") @@ -20,17 +20,17 @@ defmodule Benchmark.Compare do Enum.map(Map.keys(test_case_results), fn metric_module -> """ METRIC: #{metric_module} - #{inspect(Map.get(test_case_results, metric_module), pretty: true)} - vs - #{inspect(Map.get(test_case_results_ref, metric_module), pretty: true)} - + 1. In #{results_name}: + #{inspect(Map.get(test_case_results_ref, metric_module), pretty: true, limit: :infinity)} + 2. In #{ref_results_name}: + #{inspect(Map.get(test_case_results, metric_module), pretty: true, limit: :infinity)} """ end) |> Enum.join() Logger.debug(""" TEST CASE: - #{inspect(test_case, pretty: true)} + #{inspect(test_case, pretty: true, limit: :infinity)} #{results_str} @@ -50,4 +50,6 @@ end [results_filename, ref_results_filename] = System.argv() |> Enum.take(2) results = File.read!(results_filename) |> :erlang.binary_to_term() ref_results = File.read!(ref_results_filename) |> :erlang.binary_to_term() -Benchmark.Compare.run(results, ref_results) +results_name = String.split(results_filename, "/") |> Enum.at(-1) +ref_results_name = String.split(ref_results_filename, "/") |> Enum.at(-1) +Benchmark.Compare.run(results, ref_results, results_name, ref_results_name) diff --git a/benchmark/metric/final_memory.ex b/benchmark/metric/final_memory.ex deleted file mode 100644 index d3b14c052..000000000 --- a/benchmark/metric/final_memory.ex +++ /dev/null @@ -1,32 +0,0 @@ -defmodule Benchmark.Metric.FinalMemory do - @behaviour Benchmark.Metric - - @allowed_worsening_factor 0.5 - - @impl true - def assert(final_memory, final_memory_ref, test_case) do - if final_memory > final_memory_ref * (1 + @allowed_worsening_factor), - do: - raise( - "The memory performance has got worse! For test case: #{inspect(test_case, pretty: true)} - the final memory used to be: #{final_memory_ref} MB and now it is: #{final_memory} MB" - ) - - :ok - end - - @impl true - def average(final_memory_samples) do - Enum.sum(final_memory_samples) / (length(final_memory_samples) * 1_000_000) - end - - @impl true - def start_meassurement(_opts \\ nil) do - :erlang.memory(:total) - end - - @impl true - def stop_meassurement(starting_memory) do - :erlang.memory(:total) - starting_memory - end -end diff --git a/benchmark/metric/in_progress_memory.ex b/benchmark/metric/in_progress_memory.ex index 9d5a36996..71925924a 100644 --- a/benchmark/metric/in_progress_memory.ex +++ b/benchmark/metric/in_progress_memory.ex @@ -1,7 +1,7 @@ defmodule Benchmark.Metric.InProgressMemory do @behaviour Benchmark.Metric - @allowed_worsening_factor 0.5 + @tolerance_factor 0.5 @sampling_period 100 @impl true @@ -9,7 +9,7 @@ defmodule Benchmark.Metric.InProgressMemory do cumulative_memory = integrate(memory_samples) cumulative_memory_ref = integrate(memory_samples_ref) - if cumulative_memory > cumulative_memory_ref * (1 + @allowed_worsening_factor), + if cumulative_memory > cumulative_memory_ref * (1 + @tolerance_factor), do: raise( "The memory performance has got worse! For test case: #{inspect(test_case, pretty: true)} diff --git a/benchmark/metric/message_queues_length.ex b/benchmark/metric/message_queues_length.ex index 911f94e96..61ad10494 100644 --- a/benchmark/metric/message_queues_length.ex +++ b/benchmark/metric/message_queues_length.ex @@ -1,7 +1,7 @@ defmodule Benchmark.Metric.MessageQueuesLength do @behaviour Benchmark.Metric - @allowed_worsening_factor 0.5 + @tolerance_factor 0.5 @sampling_period 100 @impl true @@ -10,7 +10,7 @@ defmodule Benchmark.Metric.MessageQueuesLength do cumulative_queues_length_ref = integrate(queues_lengths_ref) if cumulative_queues_length > - cumulative_queues_length_ref * (1 + @allowed_worsening_factor), + cumulative_queues_length_ref * (1 + @tolerance_factor), do: raise( "The cumulative queues length has got worse! For test case: #{inspect(test_case, pretty: true)} diff --git a/benchmark/metric/time.ex b/benchmark/metric/time.ex index 0517bb53a..211738f49 100644 --- a/benchmark/metric/time.ex +++ b/benchmark/metric/time.ex @@ -1,11 +1,11 @@ defmodule Benchmark.Metric.Time do @behaviour Benchmark.Metric - @allowed_worsening_factor 0.1 + @tolerance_factor 0.15 @impl true def assert(time, time_ref, test_case) do - if time > time_ref * (1 + @allowed_worsening_factor), + if time > time_ref * (1 + @tolerance_factor), do: raise( "The time performance has got worse! For test case: #{inspect(test_case, pretty: true)} the test diff --git a/benchmark/run.exs b/benchmark/run.exs index 5125244fc..162cb76bc 100644 --- a/benchmark/run.exs +++ b/benchmark/run.exs @@ -54,7 +54,7 @@ defmodule Benchmark.Run do alias Membrane.Pad alias Benchmark.Run.BranchedFilter alias Benchmark.Run.LinearFilter - alias Benchmark.Metric.{FinalMemory, InProgressMemory, MessageQueuesLength, Time} + alias Benchmark.Metric.{InProgressMemory, MessageQueuesLength, Time} require Logger require Membrane.RCPipeline @@ -72,7 +72,7 @@ defmodule Benchmark.Run do reductions: 1_000, max_random: 1, number_of_filters: 100, - number_of_buffers: 50000, + number_of_buffers: 50_000, buffer_size: 1 ], linear: [ @@ -92,7 +92,7 @@ defmodule Benchmark.Run do with_branches: [ struct: [{1, 2}, {1, 2}, {2, 1}, {2, 1}], reductions: 100, - number_of_buffers: 50_000, + number_of_buffers: 500_000, buffer_size: 1, max_random: 10 ] @@ -245,7 +245,6 @@ defmodule Benchmark.Run do spec = prepare_pipeline(test_type, params) time_meassurement = Time.start_meassurement() - final_memory_meassurement = FinalMemory.start_meassurement() {:ok, _supervisor_pid, pipeline_pid} = Benchmark.Run.Pipeline.start( @@ -267,7 +266,6 @@ defmodule Benchmark.Run do end time = Time.stop_meassurement(time_meassurement) - final_memory = FinalMemory.stop_meassurement(final_memory_meassurement) in_progress_memory = InProgressMemory.stop_meassurement(in_progress_memory_meassurment) message_queues_length = @@ -277,7 +275,6 @@ defmodule Benchmark.Run do %{ Time => time, - FinalMemory => final_memory, InProgressMemory => in_progress_memory, MessageQueuesLength => message_queues_length }