From f9f0e3a7732e6ed135949ab894f673b3b1aabb58 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 27 Apr 2022 14:01:09 +0100 Subject: [PATCH 01/92] testing ic against py_ic --- Jenkinsfile | 28 +++++++++---------- .../py_c_frame_compare/test_ic_frame.py | 23 +++++++++------ 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5fc561dfb..3c5542495 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { } environment { REPO = 'sw_avona' - VIEW = getViewName(REPO) + VIEW = 'sw_avona_ic_test' FULL_TEST = """${(params.FULL_TEST_OVERRIDE || env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'main' @@ -235,6 +235,19 @@ pipeline { } } } + stage('IC Python C equivalence') { + steps { + dir("${REPO}/test/lib_ic/py_c_frame_compare") { + viewEnv() { + withVenv { + runPython("python build_ic_frame_proc.py") + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" + } + } + } + } + } stage('VAD vad_unit_tests') { steps { dir("${REPO}/test/lib_vad/vad_unit_tests") { @@ -321,19 +334,6 @@ pipeline { } } } - stage('IC Python C equivalence') { - steps { - dir("${REPO}/test/lib_ic/py_c_frame_compare") { - viewEnv() { - withVenv { - runPython("python build_ic_frame_proc.py") - sh "pytest -s --junitxml=pytest_result.xml" - junit "pytest_result.xml" - } - } - } - } - } stage('IC test profile') { steps { dir("${REPO}/test/lib_ic/test_ic_profile") { diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 24ba32bd8..cff546fed 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -14,7 +14,7 @@ package_dir = os.path.dirname(os.path.abspath(__file__)) att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -py_ic_path = os.path.join(package_dir,'../../../../lib_interference_canceller/python') +py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic') pvc_path = os.path.join(package_dir, '../../shared/python') sys.path.append(att_path) @@ -38,18 +38,20 @@ def pre_test_stuff(request): class ic_comparison: def __init__(self): - self.ic = IC.adaptive_interference_canceller(frame_advance, proc_frame_length, num_phases, 0, + self.ic = IC.adaptive_interference_canceller(frame_advance, proc_frame_length, num_phases, + #0, mu = 0.36956599983386695, delta = 7.450580593454381e-09, #two_mic_stereo.json - delay = 0, + #delay = 0, K = 1, lamda = 0.9995117188, gamma = 2.0, leakage = 0.995, channel_delay = x_channel_delay, remove_NQ = True, - use_noise_minimisation = False, - output_beamform_on_ch1 = True) + #use_noise_minimisation = False, + #output_beamform_on_ch1 = True + ) ic_test_lib.test_init() self.input_delay_py = np.zeros(frame_advance + x_channel_delay) @@ -64,9 +66,11 @@ def process_frame(self, frame): #first copy the input data for C ver before we modfiy it frame_int = pvc.float_to_int32(frame) #now delay y samples for the python version (not done internally) - frame[0] = self.delay_y_samples(frame[0]) + #####frame[0] = self.delay_y_samples(frame[0]) + all_channels_output, Error_ap = self.ic.process_frame(frame) - self.ic.adapt(Error_ap) + + #####self.ic.adapt(Error_ap) output_py = all_channels_output[0,:] #Grab a pointer to the data storage of the numpy arrays @@ -75,8 +79,9 @@ def process_frame(self, frame): output_c = np.zeros((240), dtype=np.int32) output_c_ptr = ffi.cast("int32_t *", ffi.from_buffer(output_c.data)) ic_test_lib.test_filter(y_data, x_data, output_c_ptr) - vad = int(0) - ic_test_lib.test_adapt(vad, output_c_ptr) + + ####vad = int(0) + ####ic_test_lib.test_adapt(vad, output_c_ptr) state = ic_test_lib.test_get_state() # print(pvc.float_s32_to_float(state.mu[0][0])) From 43623c0f8fd8a6aa45b8465bc331260220d50704 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 27 Apr 2022 14:31:59 +0100 Subject: [PATCH 02/92] fixing cmake issue --- examples/bare-metal/shared_src/external_deps/fetch_deps.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/bare-metal/shared_src/external_deps/fetch_deps.cmake b/examples/bare-metal/shared_src/external_deps/fetch_deps.cmake index c9f37da3d..ca389ef9a 100644 --- a/examples/bare-metal/shared_src/external_deps/fetch_deps.cmake +++ b/examples/bare-metal/shared_src/external_deps/fetch_deps.cmake @@ -5,7 +5,7 @@ FetchContent_Declare( xs3_math GIT_REPOSITORY https://github.com/xmos/lib_xs3_math.git GIT_TAG b440211755185d97dac8ea72f83f704f8025220f - GIT_SHALLOW TRUE + GIT_SHALLOW FALSE SOURCE_DIR ${CMAKE_BINARY_DIR}/avona_deps/lib_xs3_math ) FetchContent_Populate(xs3_math) @@ -14,7 +14,7 @@ FetchContent_Declare( xscope_fileio GIT_REPOSITORY https://github.com/xmos/xscope_fileio.git GIT_TAG 86add5101d73d98d4addea9aaeb238072e461b63 - GIT_SHALLOW TRUE + GIT_SHALLOW FALSE SOURCE_DIR ${CMAKE_BINARY_DIR}/avona_deps/xscope_fileio ) FetchContent_Populate(xscope_fileio) From b6b31642b913f0f738bb766bb47980c0af67141e Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 27 Apr 2022 14:45:21 +0100 Subject: [PATCH 03/92] typo --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index cff546fed..1b7704590 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -47,7 +47,7 @@ def __init__(self): lamda = 0.9995117188, gamma = 2.0, leakage = 0.995, - channel_delay = x_channel_delay, + y_channel_delay = x_channel_delay, remove_NQ = True, #use_noise_minimisation = False, #output_beamform_on_ch1 = True From aae050a5f5689ebfe59ec37343c5821c54139415 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 27 Apr 2022 15:25:19 +0100 Subject: [PATCH 04/92] testing other test --- test/lib_ic/characterise_c_py/characterise_c_py.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 2199ffc05..286c96956 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -11,7 +11,7 @@ package_dir = os.path.dirname(os.path.abspath(__file__)) att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -py_ic_path = os.path.join(package_dir,'../../../../lib_interference_canceller/python') +py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic') sys.path.append(att_path) sys.path.append(py_ic_path) From 623dd3da4cb20ca639213ea3ce962798e6676bc5 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 28 Apr 2022 09:53:19 +0100 Subject: [PATCH 05/92] fixing mu and leakage for characterisation test --- examples/bare-metal/ic/src/ic_test_task.c | 2 +- test/lib_ic/test_ic_spec/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/bare-metal/ic/src/ic_test_task.c b/examples/bare-metal/ic/src/ic_test_task.c index 2526edff3..e04c71345 100644 --- a/examples/bare-metal/ic/src/ic_test_task.c +++ b/examples/bare-metal/ic/src/ic_test_task.c @@ -95,7 +95,7 @@ void ic_task(const char *input_file_name, const char *output_file_name) { ic_init(&state); prof(1, "end_ic_init"); - #if IC_SPEC_TEST + #if DISABLE_ADAPTION_CONTROLLER state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = 0; state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha = float_to_float_s32(1.0); //From test_wav_ic #endif diff --git a/test/lib_ic/test_ic_spec/CMakeLists.txt b/test/lib_ic/test_ic_spec/CMakeLists.txt index f8e38dfa4..bf8f8c796 100644 --- a/test/lib_ic/test_ic_spec/CMakeLists.txt +++ b/test/lib_ic/test_ic_spec/CMakeLists.txt @@ -1,4 +1,4 @@ #same as test_wav_ic so reuse cmake file set( APP_NAME test_ic_spec ) -set(ADDITIONAL_COMPILE_FLAGS IC_SPEC_TEST=1 ) #Fixed mu and leakage=1.0 +set(ADDITIONAL_COMPILE_FLAGS DISABLE_ADAPTION_CONTROLLER=1 ) #Fixed mu and leakage=1.0 include(../test_wav_ic/CMakeLists.txt) From ab95c52797108431f782ebb5a88acad8715e928e Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 28 Apr 2022 10:40:18 +0100 Subject: [PATCH 06/92] enabling adaptation without adaptation controller --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 1b7704590..aff171c56 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -69,8 +69,8 @@ def process_frame(self, frame): #####frame[0] = self.delay_y_samples(frame[0]) all_channels_output, Error_ap = self.ic.process_frame(frame) - - #####self.ic.adapt(Error_ap) + self.ic.adapt(Error_ap) + output_py = all_channels_output[0,:] #Grab a pointer to the data storage of the numpy arrays @@ -79,12 +79,16 @@ def process_frame(self, frame): output_c = np.zeros((240), dtype=np.int32) output_c_ptr = ffi.cast("int32_t *", ffi.from_buffer(output_c.data)) ic_test_lib.test_filter(y_data, x_data, output_c_ptr) - - ####vad = int(0) - ####ic_test_lib.test_adapt(vad, output_c_ptr) + state = ic_test_lib.test_get_state() + state.adaption_controller_config.leakage_alpha.mant = int(1) + state.adaption_controller_config.leakage_alpha.exp = int(0) + state.adaption_controller_config.enable_adaption_controller = pvc.float_to_uint8(0.0) + vad = int(0) + ic_test_lib.test_adapt(vad, output_c_ptr) state = ic_test_lib.test_get_state() - # print(pvc.float_s32_to_float(state.mu[0][0])) + print(pvc.float_s32_to_float(state.mu[0][0])) + print(pvc.float_s32_to_float(state.adaption_controller_config.leakage_alpha)) # print(pvc.float_s32_to_float(state.config_params.delta)) return output_py, pvc.int32_to_float(output_c) From beba9de9a4b5217d2f08628ac28980a84130f116 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 28 Apr 2022 10:56:01 +0100 Subject: [PATCH 07/92] fixing states names --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index aff171c56..e1f237efe 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -70,7 +70,7 @@ def process_frame(self, frame): all_channels_output, Error_ap = self.ic.process_frame(frame) self.ic.adapt(Error_ap) - + output_py = all_channels_output[0,:] #Grab a pointer to the data storage of the numpy arrays @@ -80,15 +80,15 @@ def process_frame(self, frame): output_c_ptr = ffi.cast("int32_t *", ffi.from_buffer(output_c.data)) ic_test_lib.test_filter(y_data, x_data, output_c_ptr) state = ic_test_lib.test_get_state() - state.adaption_controller_config.leakage_alpha.mant = int(1) - state.adaption_controller_config.leakage_alpha.exp = int(0) - state.adaption_controller_config.enable_adaption_controller = pvc.float_to_uint8(0.0) + state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha.mant = int(1) + state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha.exp = int(0) + state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = pvc.float_to_uint8(0.0) vad = int(0) ic_test_lib.test_adapt(vad, output_c_ptr) state = ic_test_lib.test_get_state() print(pvc.float_s32_to_float(state.mu[0][0])) - print(pvc.float_s32_to_float(state.adaption_controller_config.leakage_alpha)) + print(pvc.float_s32_to_float(state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha)) # print(pvc.float_s32_to_float(state.config_params.delta)) return output_py, pvc.int32_to_float(output_c) From 3e5f0a7e6eef1c4db8a5eeba2ced71671e434b8d Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 28 Apr 2022 11:32:37 +0100 Subject: [PATCH 08/92] testing fixed mu and leakage --- .../py_c_frame_compare/test_ic_frame.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index e1f237efe..56425f181 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -30,6 +30,7 @@ x_channel_delay = 180 #For python only, this is already compiled into the C lib input_file = "../../../examples/bare-metal/ic/input.wav" output_file = "output.wav" +disable_adapt_controller = True @pytest.fixture(params=[34]) def pre_test_stuff(request): @@ -46,7 +47,8 @@ def __init__(self): K = 1, lamda = 0.9995117188, gamma = 2.0, - leakage = 0.995, + #leakage = 0.995, + leakage = 1, y_channel_delay = x_channel_delay, remove_NQ = True, #use_noise_minimisation = False, @@ -54,19 +56,20 @@ def __init__(self): ) ic_test_lib.test_init() - self.input_delay_py = np.zeros(frame_advance + x_channel_delay) - def delay_y_samples(self, frame): - self.input_delay_py[0:x_channel_delay] = self.input_delay_py[frame_advance:frame_advance + x_channel_delay] - self.input_delay_py[x_channel_delay:frame_advance + x_channel_delay] = frame - return self.input_delay_py[:frame_advance] + if disable_adapt_controller: + state = ic_test_lib.test_get_state() + state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha.mant = int(1) + state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha.exp = int(0) + state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = pvc.float_to_uint8(0.0) + print('Leakage initialised to: ', pvc.float_s32_to_float(state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha)) + print('Enable set to: ', pvc.uint8_to_float(state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller)) def process_frame(self, frame): #we need to delay the y for python as not done in model #first copy the input data for C ver before we modfiy it frame_int = pvc.float_to_int32(frame) #now delay y samples for the python version (not done internally) - #####frame[0] = self.delay_y_samples(frame[0]) all_channels_output, Error_ap = self.ic.process_frame(frame) self.ic.adapt(Error_ap) @@ -78,17 +81,15 @@ def process_frame(self, frame): x_data = ffi.cast("int32_t *", ffi.from_buffer(frame_int[1].data)) output_c = np.zeros((240), dtype=np.int32) output_c_ptr = ffi.cast("int32_t *", ffi.from_buffer(output_c.data)) + ic_test_lib.test_filter(y_data, x_data, output_c_ptr) - state = ic_test_lib.test_get_state() - state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha.mant = int(1) - state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha.exp = int(0) - state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = pvc.float_to_uint8(0.0) + vad = int(0) ic_test_lib.test_adapt(vad, output_c_ptr) state = ic_test_lib.test_get_state() - print(pvc.float_s32_to_float(state.mu[0][0])) - print(pvc.float_s32_to_float(state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha)) + print('mu_c = ', pvc.float_s32_to_float(state.mu[0][0]), ', nu_py = ', self.ic.mu) + print('leakage_c = ', pvc.float_s32_to_float(state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha), ', leakage_py = ', self.ic.leakage) # print(pvc.float_s32_to_float(state.config_params.delta)) return output_py, pvc.int32_to_float(output_c) From 8cf349dbb8d57a40ec57bd57b3ca00ccd28b567d Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 28 Apr 2022 11:52:12 +0100 Subject: [PATCH 09/92] tetsing leakage values --- modules/lib_ic/src/ic_low_level.c | 7 +++++++ test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/lib_ic/src/ic_low_level.c b/modules/lib_ic/src/ic_low_level.c index 86cb2b7e0..240f6da06 100644 --- a/modules/lib_ic/src/ic_low_level.c +++ b/modules/lib_ic/src/ic_low_level.c @@ -9,6 +9,8 @@ #include "aec_api.h" #include "aec_priv.h" +#include + ///Delay y input w.r.t. x input void ic_delay_y_input(ic_state_t *state, @@ -251,10 +253,15 @@ void ic_adaption_controller(ic_state_t *state, uint8_t vad){ ic_adaption_controller_state_t *ad_state = &state->ic_adaption_controller_state; ic_adaption_controller_config_t *ad_config = &state->ic_adaption_controller_state.adaption_controller_config; + float leak = float_s32_to_float(state->ic_adaption_controller_state.adaption_controller_config.leakage_alpha); + printf("leak before if = %f ", leak); + if(!ad_config->enable_adaption_controller){ //skip this function if adaption controller not enabled return; } + printf("after if = %f ", leak); + const float_s32_t one = {1, 0}; const float_s32_t zero = {0, 0}; const float_s32_t delta = {1100, -40}; //1100 * 2**-40 = 0.000000001 (from stage_b.py) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 56425f181..6680206bc 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -88,7 +88,7 @@ def process_frame(self, frame): ic_test_lib.test_adapt(vad, output_c_ptr) state = ic_test_lib.test_get_state() - print('mu_c = ', pvc.float_s32_to_float(state.mu[0][0]), ', nu_py = ', self.ic.mu) + #print('mu_c = ', pvc.float_s32_to_float(state.mu[0][0]), ', nu_py = ', self.ic.mu) print('leakage_c = ', pvc.float_s32_to_float(state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha), ', leakage_py = ', self.ic.leakage) # print(pvc.float_s32_to_float(state.config_params.delta)) return output_py, pvc.int32_to_float(output_c) From 0a4e8fc6141d85d1bceb0e41ccb5ace58341345e Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 28 Apr 2022 15:29:55 +0100 Subject: [PATCH 10/92] test cleanup --- .../py_c_frame_compare/test_ic_frame.py | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 6680206bc..c977467f3 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -30,7 +30,6 @@ x_channel_delay = 180 #For python only, this is already compiled into the C lib input_file = "../../../examples/bare-metal/ic/input.wav" output_file = "output.wav" -disable_adapt_controller = True @pytest.fixture(params=[34]) def pre_test_stuff(request): @@ -40,42 +39,27 @@ def pre_test_stuff(request): class ic_comparison: def __init__(self): self.ic = IC.adaptive_interference_canceller(frame_advance, proc_frame_length, num_phases, - #0, mu = 0.36956599983386695, delta = 7.450580593454381e-09, #two_mic_stereo.json - #delay = 0, K = 1, lamda = 0.9995117188, gamma = 2.0, - #leakage = 0.995, - leakage = 1, + leakage = 0.995, y_channel_delay = x_channel_delay, remove_NQ = True, - #use_noise_minimisation = False, - #output_beamform_on_ch1 = True ) ic_test_lib.test_init() - if disable_adapt_controller: - state = ic_test_lib.test_get_state() - state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha.mant = int(1) - state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha.exp = int(0) - state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = pvc.float_to_uint8(0.0) - print('Leakage initialised to: ', pvc.float_s32_to_float(state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha)) - print('Enable set to: ', pvc.uint8_to_float(state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller)) def process_frame(self, frame): #we need to delay the y for python as not done in model #first copy the input data for C ver before we modfiy it frame_int = pvc.float_to_int32(frame) - #now delay y samples for the python version (not done internally) - all_channels_output, Error_ap = self.ic.process_frame(frame) + output_py, Error_ap = self.ic.process_frame(frame) self.ic.adapt(Error_ap) - output_py = all_channels_output[0,:] - #Grab a pointer to the data storage of the numpy arrays y_data = ffi.cast("int32_t *", ffi.from_buffer(frame_int[0].data)) x_data = ffi.cast("int32_t *", ffi.from_buffer(frame_int[1].data)) @@ -89,7 +73,7 @@ def process_frame(self, frame): state = ic_test_lib.test_get_state() #print('mu_c = ', pvc.float_s32_to_float(state.mu[0][0]), ', nu_py = ', self.ic.mu) - print('leakage_c = ', pvc.float_s32_to_float(state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha), ', leakage_py = ', self.ic.leakage) + #print('leakage_c = ', pvc.float_s32_to_float(state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha), ', leakage_py = ', self.ic.leakage) # print(pvc.float_s32_to_float(state.config_params.delta)) return output_py, pvc.int32_to_float(output_c) @@ -121,9 +105,3 @@ def test_frame_compare(pre_test_stuff): assert arith_closeness > 0.98 assert geo_closeness > 0.99 assert c_delay == 0 - - - - - - From 336b8eba9845ad7e9eeb65010dbbed00538f6eae Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 28 Apr 2022 15:38:38 +0100 Subject: [PATCH 11/92] more cleanup --- modules/lib_ic/src/ic_low_level.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/lib_ic/src/ic_low_level.c b/modules/lib_ic/src/ic_low_level.c index 240f6da06..ba5ed471e 100644 --- a/modules/lib_ic/src/ic_low_level.c +++ b/modules/lib_ic/src/ic_low_level.c @@ -9,9 +9,6 @@ #include "aec_api.h" #include "aec_priv.h" -#include - - ///Delay y input w.r.t. x input void ic_delay_y_input(ic_state_t *state, int32_t y_data[IC_FRAME_ADVANCE]){ @@ -253,15 +250,10 @@ void ic_adaption_controller(ic_state_t *state, uint8_t vad){ ic_adaption_controller_state_t *ad_state = &state->ic_adaption_controller_state; ic_adaption_controller_config_t *ad_config = &state->ic_adaption_controller_state.adaption_controller_config; - float leak = float_s32_to_float(state->ic_adaption_controller_state.adaption_controller_config.leakage_alpha); - printf("leak before if = %f ", leak); - if(!ad_config->enable_adaption_controller){ //skip this function if adaption controller not enabled return; } - printf("after if = %f ", leak); - const float_s32_t one = {1, 0}; const float_s32_t zero = {0, 0}; const float_s32_t delta = {1100, -40}; //1100 * 2**-40 = 0.000000001 (from stage_b.py) From 99c7f131f7f7668c98b6d27b40da40a948958bcc Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 13:47:52 +0100 Subject: [PATCH 12/92] adding adapt controller paramter --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index c977467f3..3e12c4793 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -47,6 +47,7 @@ def __init__(self): leakage = 0.995, y_channel_delay = x_channel_delay, remove_NQ = True, + adaptation_config = 'IC_ADAPTION_FORCE_ON' ) ic_test_lib.test_init() From 3c421abbad49a8b5770886681425b25392a54651 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 14:29:26 +0100 Subject: [PATCH 13/92] changing that calls to test_wav_ic --- .../lib_ic/characterise_c_py/characterise_c_py.py | 13 ++++++++----- test/lib_ic/characterise_c_py/test_json.json | 15 +++++++++++++++ test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 test/lib_ic/characterise_c_py/test_json.json diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 286c96956..4bad547f4 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -8,6 +8,7 @@ import os import shutil import tempfile +from json_utils import json_to_dict package_dir = os.path.dirname(os.path.abspath(__file__)) att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') @@ -101,10 +102,12 @@ def generate_test_audio(filename, audio_dir, max_freq, db, angle_theta, rt60, sa scipy.io.wavfile.write(file_path, SAMPLE_RATE, output) -def process_py(input_file, output_file, x_channel_delay, audio_dir="."): +def process_py(input_file, output_file, audio_dir="."): + config_file = os.path.join('test_json.json') + ic_params = json_to_dict(config_file) test_wav_ic.test_file(os.path.join(audio_dir, input_file), os.path.join(audio_dir, output_file), - PHASES, x_channel_delay, FRAME_ADVANCE, PROC_FRAME_LENGTH, verbose=False) + ic_params) def process_c(input_file, output_file, audio_dir="."): @@ -151,14 +154,14 @@ def get_attenuation(input_file, output_file, audio_dir="."): return attenuation -def get_attenuation_c_py(test_id, noise_band, noise_db, angle_theta, rt60, x_channel_delay): +def get_attenuation_c_py(test_id, noise_band, noise_db, angle_theta, rt60): input_file = "input_{}.wav".format(test_id) # Required by test_wav_suppression.xe output_file_py = "output_{}_py.wav".format(test_id) output_file_c = "output_{}_c.wav".format(test_id) audio_dir = test_id generate_test_audio(input_file, audio_dir, noise_band, noise_db, angle_theta, rt60) - process_py(input_file, output_file_py, x_channel_delay, audio_dir) + process_py(input_file, output_file_py, audio_dir) process_c(input_file, output_file_c, audio_dir) attenuation_py = get_attenuation(input_file, output_file_py, audio_dir) @@ -199,7 +202,7 @@ def main(): start_time = time.time() args = parse_arguments() angle_theta = args.angle * np.pi/180 - get_attenuation_c_py("test", args.noise_band, args.noise_level, angle_theta, args.rt60, args.ic_delay) + get_attenuation_c_py("test", args.noise_band, args.noise_level, angle_theta, args.rt60) print("--- {0:.2f} seconds ---".format(time.time() - start_time)) diff --git a/test/lib_ic/characterise_c_py/test_json.json b/test/lib_ic/characterise_c_py/test_json.json new file mode 100644 index 000000000..839ebc640 --- /dev/null +++ b/test/lib_ic/characterise_c_py/test_json.json @@ -0,0 +1,15 @@ +{ + "frame_advance": 240, + "proc_frame_length": 512, + "phases": 10, + "mu" : 0.36956599983386695, + "delta" : 7.450580593454381e-09, + "y_channel_delay": 180, + "K" : 1, + "lamda" : 0.9995117188, + "gamma" : 2.0, + "leakage" : 0.995, + "remove_NQ" : true, + "vnr_model" : "../py_vnr/model_output_0_0_2/trained_model.h5", + "adaption_config": "IC_ADAPTION_ON" +} \ No newline at end of file diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 3e12c4793..455ffe65d 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -47,7 +47,7 @@ def __init__(self): leakage = 0.995, y_channel_delay = x_channel_delay, remove_NQ = True, - adaptation_config = 'IC_ADAPTION_FORCE_ON' + adaption_config = 'IC_ADAPTION_FORCE_ON' ) ic_test_lib.test_init() From 1015721cfbf314955dcf7b03b8abe84c05871d17 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 14:36:24 +0100 Subject: [PATCH 14/92] fixing test_file call --- test/lib_ic/characterise_c_py/characterise_c_py.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 4bad547f4..b6c06c276 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -103,11 +103,11 @@ def generate_test_audio(filename, audio_dir, max_freq, db, angle_theta, rt60, sa def process_py(input_file, output_file, audio_dir="."): - config_file = os.path.join('test_json.json') - ic_params = json_to_dict(config_file) + config_file = 'test_json.json' + test_wav_ic.test_file(os.path.join(audio_dir, input_file), os.path.join(audio_dir, output_file), - ic_params) + config_file) def process_c(input_file, output_file, audio_dir="."): From 6fe557cb46ffa51e42a7aaef7185d87107bc0448 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 14:49:26 +0100 Subject: [PATCH 15/92] fixing issues --- test/lib_ic/characterise_c_py/test_json.json | 2 +- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/lib_ic/characterise_c_py/test_json.json b/test/lib_ic/characterise_c_py/test_json.json index 839ebc640..f34b07e79 100644 --- a/test/lib_ic/characterise_c_py/test_json.json +++ b/test/lib_ic/characterise_c_py/test_json.json @@ -10,6 +10,6 @@ "gamma" : 2.0, "leakage" : 0.995, "remove_NQ" : true, - "vnr_model" : "../py_vnr/model_output_0_0_2/trained_model.h5", + "vnr_model" : "../../../../py_vnr/model_output_0_0_2/trained_model.h5", "adaption_config": "IC_ADAPTION_ON" } \ No newline at end of file diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 455ffe65d..7ef540661 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -47,6 +47,7 @@ def __init__(self): leakage = 0.995, y_channel_delay = x_channel_delay, remove_NQ = True, + vnr_model = '../../../../py_vnr/model_output_0_0_2/trained_model.h5' adaption_config = 'IC_ADAPTION_FORCE_ON' ) From 6b00ca898bbefb21fb960adefd0ec5ca40136b3c Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 15:07:13 +0100 Subject: [PATCH 16/92] missig comma --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 7ef540661..6b31cc6a6 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -47,7 +47,7 @@ def __init__(self): leakage = 0.995, y_channel_delay = x_channel_delay, remove_NQ = True, - vnr_model = '../../../../py_vnr/model_output_0_0_2/trained_model.h5' + vnr_model = '../../../../py_vnr/model_output_0_0_2/trained_model.h5', adaption_config = 'IC_ADAPTION_FORCE_ON' ) From f5296ddbc21c2e2054efc3359ba4374df2852b9d Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 15:28:11 +0100 Subject: [PATCH 17/92] removing json library --- test/lib_ic/characterise_c_py/characterise_c_py.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index b6c06c276..681d2eb3c 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -8,7 +8,6 @@ import os import shutil import tempfile -from json_utils import json_to_dict package_dir = os.path.dirname(os.path.abspath(__file__)) att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') From 0a2c88aeb83474f250557c2c31979230e6314f15 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 15:29:59 +0100 Subject: [PATCH 18/92] moving stuff in Jenkins --- Jenkinsfile | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8c2fe9d1d..f35131480 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -235,14 +235,18 @@ pipeline { } } } - stage('IC Python C equivalence') { + stage('IC characterisation') { steps { - dir("${REPO}/test/lib_ic/py_c_frame_compare") { + dir("${REPO}/test/lib_ic/characterise_c_py") { viewEnv() { withVenv { - runPython("python build_ic_frame_proc.py") - sh "pytest -s --junitxml=pytest_result.xml" + //This test compares the suppression performance across angles between model and C implementation + //and fails if they differ significantly. It requires that the C implementation run with fixed mu + sh "pytest -s --junitxml=pytest_result.xml" //-n 2 fails often so run single threaded and also print result junit "pytest_result.xml" + //This script sweeps the y_delay value to find what the optimum suppression is across RT60 and angle. + //It's more of a model develpment tool than testing the implementation so not run. It take a few minutes. + // sh "python sweep_ic_delay.py" } } } @@ -349,6 +353,19 @@ pipeline { } } } + stage('IC Python C equivalence') { + steps { + dir("${REPO}/test/lib_ic/py_c_frame_compare") { + viewEnv() { + withVenv { + runPython("python build_ic_frame_proc.py") + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" + } + } + } + } + } stage('IC test profile') { steps { dir("${REPO}/test/lib_ic/test_ic_profile") { @@ -383,23 +400,6 @@ pipeline { } } } - stage('IC characterisation') { - steps { - dir("${REPO}/test/lib_ic/characterise_c_py") { - viewEnv() { - withVenv { - //This test compares the suppression performance across angles between model and C implementation - //and fails if they differ significantly. It requires that the C implementation run with fixed mu - sh "pytest -s --junitxml=pytest_result.xml" //-n 2 fails often so run single threaded and also print result - junit "pytest_result.xml" - //This script sweeps the y_delay value to find what the optimum suppression is across RT60 and angle. - //It's more of a model develpment tool than testing the implementation so not run. It take a few minutes. - // sh "python sweep_ic_delay.py" - } - } - } - } - } stage('Stage B tests') { steps { dir("${REPO}/test/stage_b") { From 92991ab2a69dab51576657256cbc08f20274f907 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 15:48:18 +0100 Subject: [PATCH 19/92] fixing function call --- test/lib_ic/characterise_c_py/get_polar_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/characterise_c_py/get_polar_response.py b/test/lib_ic/characterise_c_py/get_polar_response.py index 61b477481..3db47f969 100644 --- a/test/lib_ic/characterise_c_py/get_polar_response.py +++ b/test/lib_ic/characterise_c_py/get_polar_response.py @@ -29,7 +29,7 @@ def get_polar_response(test_id, angle_roi, step_size, noise_band, noise_db, angle_radians = angle * np.pi / 180 generate_test_audio(input_file, audio_dir, noise_band, noise_db, angle_radians, rt60) - process_py(input_file, output_file_py, x_channel_delay, audio_dir) + process_py(input_file, output_file_py, audio_dir) attenuation_py = get_attenuation(input_file, output_file_py, audio_dir) results_py.append(attenuation_py[-2]) From 1cf1b2fb2754c45b2c13b4342989cc1172e2740b Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 17:16:07 +0100 Subject: [PATCH 20/92] fixing paths --- test/lib_ic/characterise_c_py/characterise_c_py.py | 2 +- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 681d2eb3c..4dea97a75 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -11,7 +11,7 @@ package_dir = os.path.dirname(os.path.abspath(__file__)) att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic') +py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/python/') sys.path.append(att_path) sys.path.append(py_ic_path) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 6b31cc6a6..e9f3530b2 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -14,7 +14,7 @@ package_dir = os.path.dirname(os.path.abspath(__file__)) att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic') +py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/python/') pvc_path = os.path.join(package_dir, '../../shared/python') sys.path.append(att_path) From 25fec7377e738d5cf6b15d5c4aed4fc44a3a1fbc Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 5 May 2022 17:42:50 +0100 Subject: [PATCH 21/92] experimenting with paths --- requirements.txt | 1 + test/lib_ic/characterise_c_py/characterise_c_py.py | 4 ++-- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index fa80dc347..7091aacc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,3 +49,4 @@ jinja2 # setup.py file, e.g., '-e .' or '-e ./python' (without the quotes). -e ./../audio_test_tools/python -e ./../py_vnr +-e ./../py_ic/py_ic diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 4dea97a75..e08e9d407 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -11,7 +11,7 @@ package_dir = os.path.dirname(os.path.abspath(__file__)) att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/python/') +py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/') sys.path.append(att_path) sys.path.append(py_ic_path) @@ -29,7 +29,7 @@ try: import test_wav_ic except ModuleNotFoundError: - print(f"Please install lib_interference_canceller at root of project to support model testing") + print(f"Please install py_ic at root of project to support model testing") import xtagctl, xscope_fileio diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index e9f3530b2..e0da548b9 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -14,7 +14,7 @@ package_dir = os.path.dirname(os.path.abspath(__file__)) att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/python/') +py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/') pvc_path = os.path.join(package_dir, '../../shared/python') sys.path.append(att_path) From 27a88e7936dbeb5e12edd454d5cd8ffe8b195e11 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 6 May 2022 13:15:22 +0100 Subject: [PATCH 22/92] testing changes in py_ic --- test/lib_ic/characterise_c_py/characterise_c_py.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index e08e9d407..e6fad0fe1 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -10,11 +10,11 @@ import tempfile package_dir = os.path.dirname(os.path.abspath(__file__)) -att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/') +#att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') +#py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/') -sys.path.append(att_path) -sys.path.append(py_ic_path) +#sys.path.append(att_path) +#sys.path.append(py_ic_path) from audio_generation import get_band_limited_noise, write_data import subprocess @@ -27,7 +27,7 @@ import scipy try: - import test_wav_ic + import py_ic.test_wav_ic as twic except ModuleNotFoundError: print(f"Please install py_ic at root of project to support model testing") @@ -104,7 +104,8 @@ def generate_test_audio(filename, audio_dir, max_freq, db, angle_theta, rt60, sa def process_py(input_file, output_file, audio_dir="."): config_file = 'test_json.json' - test_wav_ic.test_file(os.path.join(audio_dir, input_file), + #test_wav_ic.test_file + twic.test_file(os.path.join(audio_dir, input_file), os.path.join(audio_dir, output_file), config_file) From cf3dd194f7fb9df8ab6c2185a6f798214aea1164 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 6 May 2022 13:40:18 +0100 Subject: [PATCH 23/92] testing import --- test/lib_ic/characterise_c_py/characterise_c_py.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index e6fad0fe1..385cc29dd 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -27,7 +27,7 @@ import scipy try: - import py_ic.test_wav_ic as twic + import test_wav_ic as twic except ModuleNotFoundError: print(f"Please install py_ic at root of project to support model testing") From 45156fe7be47a8aa7e5cd6e97935ed36870b611c Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 6 May 2022 14:00:55 +0100 Subject: [PATCH 24/92] fixing config value --- test/lib_ic/characterise_c_py/test_json.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/characterise_c_py/test_json.json b/test/lib_ic/characterise_c_py/test_json.json index f34b07e79..c8f78e7e9 100644 --- a/test/lib_ic/characterise_c_py/test_json.json +++ b/test/lib_ic/characterise_c_py/test_json.json @@ -11,5 +11,5 @@ "leakage" : 0.995, "remove_NQ" : true, "vnr_model" : "../../../../py_vnr/model_output_0_0_2/trained_model.h5", - "adaption_config": "IC_ADAPTION_ON" + "adaption_config": "IC_ADAPTION_FORCE_ON" } \ No newline at end of file From 4f99f0eab42196fcc0ebfd1c54c1f7c128fd8604 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 6 May 2022 14:23:29 +0100 Subject: [PATCH 25/92] small cleanup and rearanging stuff for Jenkins --- Jenkinsfile | 26 +++++++++---------- .../characterise_c_py/characterise_c_py.py | 12 +-------- .../py_c_frame_compare/test_ic_frame.py | 19 ++++++++------ 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 127447919..49b4c0e80 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -244,6 +244,19 @@ pipeline { } } } + stage('IC Python C equivalence') { + steps { + dir("${REPO}/test/lib_ic/py_c_frame_compare") { + viewEnv() { + withVenv { + runPython("python build_ic_frame_proc.py") + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" + } + } + } + } + } stage('IC characterisation') { steps { dir("${REPO}/test/lib_ic/characterise_c_py") { @@ -362,19 +375,6 @@ pipeline { } } } - stage('IC Python C equivalence') { - steps { - dir("${REPO}/test/lib_ic/py_c_frame_compare") { - viewEnv() { - withVenv { - runPython("python build_ic_frame_proc.py") - sh "pytest -s --junitxml=pytest_result.xml" - junit "pytest_result.xml" - } - } - } - } - } stage('IC test profile') { steps { dir("${REPO}/test/lib_ic/test_ic_profile") { diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 385cc29dd..b02ac6ef9 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -2,22 +2,12 @@ # This Software is subject to the terms of the XMOS Public Licence: Version 1. from __future__ import division from __future__ import print_function -from builtins import str from builtins import range -import sys import os import shutil import tempfile -package_dir = os.path.dirname(os.path.abspath(__file__)) -#att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -#py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/') - -#sys.path.append(att_path) -#sys.path.append(py_ic_path) - -from audio_generation import get_band_limited_noise, write_data -import subprocess +from audio_generation import get_band_limited_noise import time import numpy as np import scipy.io.wavfile as wavfile diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index e0da548b9..6159a5692 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -4,7 +4,6 @@ import numpy as np import scipy.io.wavfile import audio_wav_utils as awu -import ctypes import pytest import sys, os @@ -13,21 +12,25 @@ import ic_test_py.lib as ic_test_lib package_dir = os.path.dirname(os.path.abspath(__file__)) -att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/') +#att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') +#py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/') pvc_path = os.path.join(package_dir, '../../shared/python') -sys.path.append(att_path) -sys.path.append(py_ic_path) +#sys.path.append(att_path) +#sys.path.append(py_ic_path) sys.path.append(pvc_path) -import IC +try: + import IC +except ModuleNotFoundError: + print(f"Please install py_ic at root of project to support model testing") + import py_vs_c_utils as pvc proc_frame_length = 512 frame_advance = 240 num_phases = 10 -x_channel_delay = 180 #For python only, this is already compiled into the C lib +y_channel_delay = 180 #For python only, this is already compiled into the C lib input_file = "../../../examples/bare-metal/ic/input.wav" output_file = "output.wav" @@ -45,7 +48,7 @@ def __init__(self): lamda = 0.9995117188, gamma = 2.0, leakage = 0.995, - y_channel_delay = x_channel_delay, + y_channel_delay = y_channel_delay, remove_NQ = True, vnr_model = '../../../../py_vnr/model_output_0_0_2/trained_model.h5', adaption_config = 'IC_ADAPTION_FORCE_ON' From 92752eb6621ac7414af16a236d123d96bcbe1fc5 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 6 May 2022 14:51:56 +0100 Subject: [PATCH 26/92] changing num phases to 6 --- test/lib_ic/characterise_c_py/test_json.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/characterise_c_py/test_json.json b/test/lib_ic/characterise_c_py/test_json.json index c8f78e7e9..c2e5b3570 100644 --- a/test/lib_ic/characterise_c_py/test_json.json +++ b/test/lib_ic/characterise_c_py/test_json.json @@ -1,7 +1,7 @@ { "frame_advance": 240, "proc_frame_length": 512, - "phases": 10, + "phases": 6, "mu" : 0.36956599983386695, "delta" : 7.450580593454381e-09, "y_channel_delay": 180, From 39fd20e705949f50dd543b4c7dd2d150bde821aa Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 6 May 2022 15:29:44 +0100 Subject: [PATCH 27/92] small cleanup and fix a test_wav call --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 6 +----- test/lib_ic/test_ic_spec/test_ic_spec.py | 12 +++++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 6159a5692..f0283ad48 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -12,12 +12,8 @@ import ic_test_py.lib as ic_test_lib package_dir = os.path.dirname(os.path.abspath(__file__)) -#att_path = os.path.join(package_dir,'../../../audio_test_tools/python/') -#py_ic_path = os.path.join(package_dir,'../../../../py_ic/py_ic/') pvc_path = os.path.join(package_dir, '../../shared/python') -#sys.path.append(att_path) -#sys.path.append(py_ic_path) sys.path.append(pvc_path) try: import IC @@ -30,7 +26,7 @@ proc_frame_length = 512 frame_advance = 240 num_phases = 10 -y_channel_delay = 180 #For python only, this is already compiled into the C lib +y_channel_delay = 180 input_file = "../../../examples/bare-metal/ic/input.wav" output_file = "output.wav" diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 62c6478e3..999c0eec2 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -28,6 +28,7 @@ import subprocess import numpy as np import filters +from common_utils import json_to_dict input_folder = os.path.abspath("input_wavs") output_folder = os.path.abspath("output_wavs") @@ -134,12 +135,13 @@ def write_output(test_name, output, c_or_py): def process_py(input_data, test_name): - phases = 10 - x_channel_delay = 180 + file_length = input_data.shape[1] - output = test_wav_ic.test_data(input_data, 16000, file_length, phases, - x_channel_delay, frame_advance=frame_advance, - proc_frame_length=proc_frame_length, verbose=False) + config_file = '../characterise_c_py/test_json.json' + ic_parameters = json_to_dict(config_file) + ic_parameters['phases'] = 10 + + output = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) write_output(test_name, output, 'py') return output From 79f2045d94b778a902411e480fb495c703df02ee Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 6 May 2022 16:01:03 +0100 Subject: [PATCH 28/92] experimenting with test_wav output --- Jenkinsfile | 42 ++++++++++++------------ test/lib_ic/test_ic_spec/test_ic_spec.py | 20 +++++------ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 49b4c0e80..31e30c6e8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -257,6 +257,27 @@ pipeline { } } } + stage('IC test specification') { + steps { + dir("${REPO}/test/lib_ic/test_ic_spec") { + viewEnv() { + withVenv { + //This test compares the model and C implementation over a range of scenarious for: + //convergence_time, db_suppression, maximum noise added to input (to test for stability) + //and expected group delay. It will fail if these are not met. + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" + sh "python print_stats.py > ic_spec_summary.txt" + //This script generates a number of polar plots of attenuation vs null point angle vs freq + //It currently only uses the python model to do this. It takes about 40 mins for all plots + //and generates a series of IC_performance_xxxHz.svg files which could be archived + // sh "python plot_ic.py" + } + } + archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true + } + } + } stage('IC characterisation') { steps { dir("${REPO}/test/lib_ic/characterise_c_py") { @@ -388,27 +409,6 @@ pipeline { } } } - stage('IC test specification') { - steps { - dir("${REPO}/test/lib_ic/test_ic_spec") { - viewEnv() { - withVenv { - //This test compares the model and C implementation over a range of scenarious for: - //convergence_time, db_suppression, maximum noise added to input (to test for stability) - //and expected group delay. It will fail if these are not met. - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" - sh "python print_stats.py > ic_spec_summary.txt" - //This script generates a number of polar plots of attenuation vs null point angle vs freq - //It currently only uses the python model to do this. It takes about 40 mins for all plots - //and generates a series of IC_performance_xxxHz.svg files which could be archived - // sh "python plot_ic.py" - } - } - archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true - } - } - } stage('Stage B tests') { steps { dir("${REPO}/test/stage_b") { diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 999c0eec2..854e92cf4 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -10,15 +10,15 @@ import os import warnings -py_env = os.environ.copy() -sys.path += [os.path.join(os.environ['XMOS_ROOT'], dep) for dep in [ - "audio_test_tools/python", - "lib_interference_canceller/python", - "lib_vad/python", - "lib_audio_pipelines/python", - ] - ] -py_env['PYTHONPATH'] += ':' + ':'.join(sys.path) +#py_env = os.environ.copy() +#sys.path += [os.path.join(os.environ['XMOS_ROOT'], dep) for dep in [ +# "audio_test_tools/python", +# "lib_interference_canceller/python", +# "lib_vad/python", +# "lib_audio_pipelines/python", +# ] +# ] +#py_env['PYTHONPATH'] += ':' + ':'.join(sys.path) from scipy.signal import convolve import scipy.io.wavfile @@ -141,7 +141,7 @@ def process_py(input_data, test_name): ic_parameters = json_to_dict(config_file) ic_parameters['phases'] = 10 - output = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) + output, Mu, Input_vnr_pred, Output_vnr_pred, Control_flag = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) write_output(test_name, output, 'py') return output From 08b3d41fbcd36c5bfdb7b5d9f8e9e6e050c89e64 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 6 May 2022 16:19:09 +0100 Subject: [PATCH 29/92] array dimention fix --- test/lib_ic/test_ic_spec/test_ic_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 854e92cf4..3cf46f77e 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -215,7 +215,7 @@ def get_suppression_arr(input_audio, output_audio): for i in np.arange(0, num_frames*frame_advance, frame_advance): i = int(i) in_rms = rms(input_audio[0][i:i+frame_advance]) - out_rms = rms(output_audio[0][i:i+frame_advance]) + out_rms = rms(output_audio[i:i+frame_advance]) if in_rms == 0 or out_rms == 0: suppression_db = 0.0 elif i - frame_advance < ICSpec.expected_delay: From b5cca6f54bdba5cf844c745c95aa9a97d5a7732a Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 9 May 2022 14:15:53 +0100 Subject: [PATCH 30/92] changing delta for the char test --- Jenkinsfile | 102 +++++++++--------- .../{test_json.json => char_test.json} | 2 +- .../characterise_c_py/characterise_c_py.py | 2 +- test/lib_ic/test_ic_spec/spec_test.json | 15 +++ test/lib_ic/test_ic_spec/test_ic_spec.py | 3 +- 5 files changed, 69 insertions(+), 55 deletions(-) rename test/lib_ic/characterise_c_py/{test_json.json => char_test.json} (90%) create mode 100644 test/lib_ic/test_ic_spec/spec_test.json diff --git a/Jenkinsfile b/Jenkinsfile index 31e30c6e8..b12740793 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -164,6 +164,57 @@ pipeline { } } } + stage('IC Python C equivalence') { + steps { + dir("${REPO}/test/lib_ic/py_c_frame_compare") { + viewEnv() { + withVenv { + runPython("python build_ic_frame_proc.py") + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" + } + } + } + } + } + stage('IC characterisation') { + steps { + dir("${REPO}/test/lib_ic/characterise_c_py") { + viewEnv() { + withVenv { + //This test compares the suppression performance across angles between model and C implementation + //and fails if they differ significantly. It requires that the C implementation run with fixed mu + sh "pytest -s --junitxml=pytest_result.xml" //-n 2 fails often so run single threaded and also print result + junit "pytest_result.xml" + //This script sweeps the y_delay value to find what the optimum suppression is across RT60 and angle. + //It's more of a model develpment tool than testing the implementation so not run. It take a few minutes. + // sh "python sweep_ic_delay.py" + } + } + } + } + } + stage('IC test specification') { + steps { + dir("${REPO}/test/lib_ic/test_ic_spec") { + viewEnv() { + withVenv { + //This test compares the model and C implementation over a range of scenarious for: + //convergence_time, db_suppression, maximum noise added to input (to test for stability) + //and expected group delay. It will fail if these are not met. + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" + sh "python print_stats.py > ic_spec_summary.txt" + //This script generates a number of polar plots of attenuation vs null point angle vs freq + //It currently only uses the python model to do this. It takes about 40 mins for all plots + //and generates a series of IC_performance_xxxHz.svg files which could be archived + // sh "python plot_ic.py" + } + } + archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true + } + } + } stage('Examples') { steps { dir("${REPO}/examples/bare-metal/aec_1_thread") { @@ -244,57 +295,6 @@ pipeline { } } } - stage('IC Python C equivalence') { - steps { - dir("${REPO}/test/lib_ic/py_c_frame_compare") { - viewEnv() { - withVenv { - runPython("python build_ic_frame_proc.py") - sh "pytest -s --junitxml=pytest_result.xml" - junit "pytest_result.xml" - } - } - } - } - } - stage('IC test specification') { - steps { - dir("${REPO}/test/lib_ic/test_ic_spec") { - viewEnv() { - withVenv { - //This test compares the model and C implementation over a range of scenarious for: - //convergence_time, db_suppression, maximum noise added to input (to test for stability) - //and expected group delay. It will fail if these are not met. - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" - sh "python print_stats.py > ic_spec_summary.txt" - //This script generates a number of polar plots of attenuation vs null point angle vs freq - //It currently only uses the python model to do this. It takes about 40 mins for all plots - //and generates a series of IC_performance_xxxHz.svg files which could be archived - // sh "python plot_ic.py" - } - } - archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true - } - } - } - stage('IC characterisation') { - steps { - dir("${REPO}/test/lib_ic/characterise_c_py") { - viewEnv() { - withVenv { - //This test compares the suppression performance across angles between model and C implementation - //and fails if they differ significantly. It requires that the C implementation run with fixed mu - sh "pytest -s --junitxml=pytest_result.xml" //-n 2 fails often so run single threaded and also print result - junit "pytest_result.xml" - //This script sweeps the y_delay value to find what the optimum suppression is across RT60 and angle. - //It's more of a model develpment tool than testing the implementation so not run. It take a few minutes. - // sh "python sweep_ic_delay.py" - } - } - } - } - } stage('VNR test_wav_vnr') { steps { dir("${REPO}/test/lib_vnr/test_wav_vnr") { diff --git a/test/lib_ic/characterise_c_py/test_json.json b/test/lib_ic/characterise_c_py/char_test.json similarity index 90% rename from test/lib_ic/characterise_c_py/test_json.json rename to test/lib_ic/characterise_c_py/char_test.json index c2e5b3570..564792950 100644 --- a/test/lib_ic/characterise_c_py/test_json.json +++ b/test/lib_ic/characterise_c_py/char_test.json @@ -3,7 +3,7 @@ "proc_frame_length": 512, "phases": 6, "mu" : 0.36956599983386695, - "delta" : 7.450580593454381e-09, + "delta" : 0.0156249999963620211929, "y_channel_delay": 180, "K" : 1, "lamda" : 0.9995117188, diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index b02ac6ef9..1a10cb793 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -92,7 +92,7 @@ def generate_test_audio(filename, audio_dir, max_freq, db, angle_theta, rt60, sa def process_py(input_file, output_file, audio_dir="."): - config_file = 'test_json.json' + config_file = 'char_test.json' #test_wav_ic.test_file twic.test_file(os.path.join(audio_dir, input_file), diff --git a/test/lib_ic/test_ic_spec/spec_test.json b/test/lib_ic/test_ic_spec/spec_test.json new file mode 100644 index 000000000..acfe53a82 --- /dev/null +++ b/test/lib_ic/test_ic_spec/spec_test.json @@ -0,0 +1,15 @@ +{ + "frame_advance": 240, + "proc_frame_length": 512, + "phases": 10, + "mu" : 1.0, + "delta" : 7.450580593454381e-09, + "y_channel_delay": 180, + "K" : 1, + "lamda" : 0.9995117188, + "gamma" : 2.0, + "leakage" : 0.995, + "remove_NQ" : true, + "vnr_model" : "./../../py_vnr/model_output_0_0_2/trained_model.h5", + "adaption_config": "IC_ADAPTION_FORCE_ON" +} \ No newline at end of file diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 3cf46f77e..f3667c0ed 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -137,9 +137,8 @@ def write_output(test_name, output, c_or_py): def process_py(input_data, test_name): file_length = input_data.shape[1] - config_file = '../characterise_c_py/test_json.json' + config_file = 'spec_test.json' ic_parameters = json_to_dict(config_file) - ic_parameters['phases'] = 10 output, Mu, Input_vnr_pred, Output_vnr_pred, Control_flag = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) write_output(test_name, output, 'py') From a828013996a78bdce2c28b4669fdcf9fe6d75613 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 9 May 2022 16:16:37 +0100 Subject: [PATCH 31/92] addidng mu_control_system to the test --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index f0283ad48..6360c96eb 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -59,6 +59,8 @@ def process_frame(self, frame): frame_int = pvc.float_to_int32(frame) output_py, Error_ap = self.ic.process_frame(frame) + mu, input_vnr_pred, output_vnr_pred, control_flag = self.ic.mu_control_system(Error_ap) + print('mu = ', mu, ', in_vnr = ', input_vnr_pred, ', out_vnr = ', output_vnr_pred, 'flag = ', control_flag) self.ic.adapt(Error_ap) #Grab a pointer to the data storage of the numpy arrays From 4450184b204494ed0a4033729c005a74c43d5dc4 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 9 May 2022 16:49:11 +0100 Subject: [PATCH 32/92] adding phases fix --- test/lib_ic/characterise_c_py/char_test.json | 2 +- test/lib_ic/characterise_c_py/characterise_c_py.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/characterise_c_py/char_test.json b/test/lib_ic/characterise_c_py/char_test.json index 564792950..68245ea43 100644 --- a/test/lib_ic/characterise_c_py/char_test.json +++ b/test/lib_ic/characterise_c_py/char_test.json @@ -1,7 +1,7 @@ { "frame_advance": 240, "proc_frame_length": 512, - "phases": 6, + "phases": 10, "mu" : 0.36956599983386695, "delta" : 0.0156249999963620211929, "y_channel_delay": 180, diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 1a10cb793..419b4baa0 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -26,7 +26,7 @@ NOISE_FLOOR_dBFS = -63.0 SIGMA2_AWGN = ((10 ** (float(NOISE_FLOOR_dBFS)/20)) * np.iinfo(np.int32).max) ** 2 -PHASES = 6 +PHASES = 10 FRAME_ADVANCE = 240 PROC_FRAME_LENGTH = 512 From a5a7c96da555fd7d32fe1fb8ff3fdaa9c4ec82da Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 9 May 2022 17:38:11 +0100 Subject: [PATCH 33/92] removing second channel from attenuation calc --- test/lib_ic/characterise_c_py/characterise_c_py.py | 12 ++++-------- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 419b4baa0..6ab158bc1 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -26,10 +26,6 @@ NOISE_FLOOR_dBFS = -63.0 SIGMA2_AWGN = ((10 ** (float(NOISE_FLOOR_dBFS)/20)) * np.iinfo(np.int32).max) ** 2 -PHASES = 10 -FRAME_ADVANCE = 240 -PROC_FRAME_LENGTH = 512 - SAMPLE_RATE = 16000 SAMPLE_COUNT = 160080 @@ -94,7 +90,6 @@ def generate_test_audio(filename, audio_dir, max_freq, db, angle_theta, rt60, sa def process_py(input_file, output_file, audio_dir="."): config_file = 'char_test.json' - #test_wav_ic.test_file twic.test_file(os.path.join(audio_dir, input_file), os.path.join(audio_dir, output_file), config_file) @@ -134,9 +129,10 @@ def get_attenuation(input_file, output_file, audio_dir="."): for i in range(len(in_power_l) // SAMPLE_RATE): window_start = i*SAMPLE_RATE window_end = window_start + SAMPLE_RATE - av_in_power_l = np.mean(in_power_l[window_start:window_end]) - av_in_power_r = np.mean(in_power_r[window_start:window_end]) - av_in_power = (av_in_power_l + av_in_power_r) / 2 + #av_in_power_l = np.mean(in_power_l[window_start:window_end]) + #av_in_power_r = np.mean(in_power_r[window_start:window_end]) + #av_in_power = (av_in_power_l + av_in_power_r) / 2 + av_in_power = np.mean(in_power_l[window_start:window_end]) av_out_power = np.mean(out_power[window_start:window_end]) new_atten = 10 * np.log10(av_in_power / av_out_power) if av_out_power != 0 else 1000 diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 6360c96eb..f4bfe89ce 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -60,7 +60,7 @@ def process_frame(self, frame): output_py, Error_ap = self.ic.process_frame(frame) mu, input_vnr_pred, output_vnr_pred, control_flag = self.ic.mu_control_system(Error_ap) - print('mu = ', mu, ', in_vnr = ', input_vnr_pred, ', out_vnr = ', output_vnr_pred, 'flag = ', control_flag) + #print('mu = ', mu, ', in_vnr = ', input_vnr_pred, ', out_vnr = ', output_vnr_pred, 'flag = ', control_flag) self.ic.adapt(Error_ap) #Grab a pointer to the data storage of the numpy arrays From 7359bdb871619ee1b1bbd648a80570a7fe21a42c Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 10:00:40 +0100 Subject: [PATCH 34/92] trying to swap channels --- test/lib_ic/characterise_c_py/characterise_c_py.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 6ab158bc1..c839012ed 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -126,13 +126,13 @@ def get_attenuation(input_file, output_file, audio_dir="."): attenuation = [] - for i in range(len(in_power_l) // SAMPLE_RATE): + for i in range(len(in_power_r) // SAMPLE_RATE): window_start = i*SAMPLE_RATE window_end = window_start + SAMPLE_RATE #av_in_power_l = np.mean(in_power_l[window_start:window_end]) #av_in_power_r = np.mean(in_power_r[window_start:window_end]) #av_in_power = (av_in_power_l + av_in_power_r) / 2 - av_in_power = np.mean(in_power_l[window_start:window_end]) + av_in_power = np.mean(in_power_r[window_start:window_end]) av_out_power = np.mean(out_power[window_start:window_end]) new_atten = 10 * np.log10(av_in_power / av_out_power) if av_out_power != 0 else 1000 From 27f514cb16601fc9ceca7c0b192e14a16a8b5f2a Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 11:35:10 +0100 Subject: [PATCH 35/92] disabling jenkins tests and assertion --- Jenkinsfile | 162 +++++++++--------- .../characterise_c_py/test_polar_response.py | 2 +- 2 files changed, 83 insertions(+), 81 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b12740793..49fe0c33a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -202,16 +202,16 @@ pipeline { //This test compares the model and C implementation over a range of scenarious for: //convergence_time, db_suppression, maximum noise added to input (to test for stability) //and expected group delay. It will fail if these are not met. - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" - sh "python print_stats.py > ic_spec_summary.txt" + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" + //sh "python print_stats.py > ic_spec_summary.txt" //This script generates a number of polar plots of attenuation vs null point angle vs freq //It currently only uses the python model to do this. It takes about 40 mins for all plots //and generates a series of IC_performance_xxxHz.svg files which could be archived // sh "python plot_ic.py" } } - archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true + //archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true } } } @@ -220,76 +220,76 @@ pipeline { dir("${REPO}/examples/bare-metal/aec_1_thread") { viewEnv() { withVenv { - sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/aec_1_thread/bin/avona_example_bare_metal_aec_1_thread.xe --input ../shared_src/test_streams/aec_example_input.wav" + //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/aec_1_thread/bin/avona_example_bare_metal_aec_1_thread.xe --input ../shared_src/test_streams/aec_example_input.wav" } } } dir("${REPO}/examples/bare-metal/aec_2_threads") { viewEnv() { withVenv { - sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/aec_2_threads/bin/avona_example_bare_metal_aec_2_thread.xe --input ../shared_src/test_streams/aec_example_input.wav" + //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/aec_2_threads/bin/avona_example_bare_metal_aec_2_thread.xe --input ../shared_src/test_streams/aec_example_input.wav" // Make sure 1 thread and 2 threads output is bitexact - sh "diff output.wav ../aec_1_thread/output.wav" + //sh "diff output.wav ../aec_1_thread/output.wav" } } } dir("${REPO}/examples/bare-metal/ic") { viewEnv() { withVenv { - sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/ic/bin/avona_example_bare_metal_ic.xe" - sh "mv output.wav ic_example_output.wav" + //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/ic/bin/avona_example_bare_metal_ic.xe" + //sh "mv output.wav ic_example_output.wav" } } - archiveArtifacts artifacts: "ic_example_output.wav", fingerprint: true + //archiveArtifacts artifacts: "ic_example_output.wav", fingerprint: true } dir("${REPO}/examples/bare-metal/vad") { viewEnv() { withVenv { - sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/vad/bin/avona_example_bare_metal_vad.xe" + //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/vad/bin/avona_example_bare_metal_vad.xe" } } } dir("${REPO}/examples/bare-metal/pipeline_single_threaded") { viewEnv() { withVenv { - sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_single_threaded/bin/avona_example_bare_metal_pipeline_single_thread.xe --input ../shared_src/test_streams/pipeline_example_input.wav" + //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_single_threaded/bin/avona_example_bare_metal_pipeline_single_thread.xe --input ../shared_src/test_streams/pipeline_example_input.wav" } } } dir("${REPO}/examples/bare-metal/pipeline_multi_threaded") { viewEnv() { withVenv { - sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_multi_threaded/bin/avona_example_bare_metal_pipeline_multi_thread.xe --input ../shared_src/test_streams/pipeline_example_input.wav" + //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_multi_threaded/bin/avona_example_bare_metal_pipeline_multi_thread.xe --input ../shared_src/test_streams/pipeline_example_input.wav" // Make sure single thread and multi threads pipeline output is bitexact - sh "diff output.wav ../pipeline_single_threaded/output.wav" + //sh "diff output.wav ../pipeline_single_threaded/output.wav" } } } dir("${REPO}/examples/bare-metal/pipeline_alt_arch") { viewEnv() { withVenv { - sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_alt_arch/bin/avona_example_bare_metal_pipeline_alt_arch_st.xe --input ../shared_src/test_streams/pipeline_example_input.wav" - sh "mv output.wav output_st.wav" + //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_alt_arch/bin/avona_example_bare_metal_pipeline_alt_arch_st.xe --input ../shared_src/test_streams/pipeline_example_input.wav" + //sh "mv output.wav output_st.wav" - sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_alt_arch/bin/avona_example_bare_metal_pipeline_alt_arch_mt.xe --input ../shared_src/test_streams/pipeline_example_input.wav" - sh "mv output.wav output_mt.wav" - sh "diff output_st.wav output_mt.wav" + //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_alt_arch/bin/avona_example_bare_metal_pipeline_alt_arch_mt.xe --input ../shared_src/test_streams/pipeline_example_input.wav" + //sh "mv output.wav output_mt.wav" + //sh "diff output_st.wav output_mt.wav" } } } dir("${REPO}/examples/bare-metal/agc") { viewEnv() { withVenv { - sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/agc/bin/avona_example_bare_metal_agc.xe --input ../shared_src/test_streams/agc_example_input.wav" + //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/agc/bin/avona_example_bare_metal_agc.xe --input ../shared_src/test_streams/agc_example_input.wav" } } } dir("${REPO}/examples/bare-metal/vnr") { viewEnv() { withVenv { - sh "python host_app.py test_stream_1.wav vnr_out2.bin --run_with_xscope_fileio=1" //With xscope host in lib xscope_fileio - sh "python host_app.py test_stream_1.wav vnr_out1.bin" //With xscope host in python - sh "diff vnr_out1.bin vnr_out2.bin" + //sh "python host_app.py test_stream_1.wav vnr_out2.bin --run_with_xscope_fileio=1" //With xscope host in lib xscope_fileio + //sh "python host_app.py test_stream_1.wav vnr_out1.bin" //With xscope host in python + //sh "diff vnr_out1.bin vnr_out2.bin" } } } @@ -302,7 +302,7 @@ pipeline { withVenv { withMounts([["projects", "projects/hydra_audio", "hydra_audio_vnr_tests"]]) { withEnv(["hydra_audio_PATH=$hydra_audio_vnr_tests_PATH"]) { - sh "pytest -n 1 --junitxml=pytest_result.xml" + //sh "pytest -n 1 --junitxml=pytest_result.xml" } } } @@ -315,8 +315,8 @@ pipeline { dir("${REPO}/test/lib_vad/vad_unit_tests") { viewEnv() { withVenv { - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -327,8 +327,8 @@ pipeline { dir("${REPO}/test/lib_vad/compare_xc_c") { viewEnv() { withVenv { - sh "pytest -s --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -s --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -339,11 +339,11 @@ pipeline { dir("${REPO}/test/lib_vad/test_vad_profile") { viewEnv() { withVenv { - sh "pytest -s --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -s --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } - archiveArtifacts artifacts: "vad_profile_report.log", fingerprint: true + //archiveArtifacts artifacts: "vad_profile_report.log", fingerprint: true } } } @@ -352,8 +352,8 @@ pipeline { dir("${REPO}/test/lib_ns/test_ns_profile") { viewEnv() { withVenv { - sh "pytest -n 1 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 1 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -365,8 +365,8 @@ pipeline { copyArtifacts filter: '**/*.xe', fingerprintArtifacts: true, projectName: '../lib_noise_suppression/develop', selector: lastSuccessful() viewEnv() { withVenv { - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -377,8 +377,8 @@ pipeline { dir("${REPO}/test/lib_ns/ns_unit_tests") { viewEnv() { withVenv { - sh "pytest -n 1 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 1 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -389,8 +389,8 @@ pipeline { dir("${REPO}/test/lib_ic/ic_unit_tests") { viewEnv() { withVenv { - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -401,11 +401,11 @@ pipeline { dir("${REPO}/test/lib_ic/test_ic_profile") { viewEnv() { withVenv { - sh "pytest --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } - archiveArtifacts artifacts: "ic_prof.log", fingerprint: true + //archiveArtifacts artifacts: "ic_prof.log", fingerprint: true } } } @@ -416,9 +416,9 @@ pipeline { withVenv { withMounts([["projects", "projects/hydra_audio", "hydra_audio_stage_b_tests"]]) { withEnv(["hydra_audio_PATH=$hydra_audio_stage_b_tests_PATH"]) { - runPython("python build_c_code.py") - sh "pytest -s --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //runPython("python build_c_code.py") + //sh "pytest -s --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -431,8 +431,8 @@ pipeline { dir("${REPO}/test/lib_adec/de_unit_tests") { viewEnv() { withVenv { - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -447,9 +447,9 @@ pipeline { withEnv(["hydra_audio_PATH=$hydra_audio_test_de_PATH"]) { sh 'mkdir -p ./input_wavs/' sh 'mkdir -p ./output_files/' - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" - runPython("python print_stats.py") + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" + //runPython("python print_stats.py") } } } @@ -464,8 +464,8 @@ pipeline { withVenv { withMounts([["projects", "projects/hydra_audio", "hydra_audio_adec_tests"]]) { withEnv(["hydra_audio_PATH=$hydra_audio_adec_tests_PATH"]) { - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -480,8 +480,8 @@ pipeline { withVenv { withMounts([["projects", "projects/hydra_audio", "hydra_audio_adec_tests"]]) { withEnv(["hydra_audio_PATH=$hydra_audio_adec_tests_PATH"]) { - sh "pytest -n 1 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 1 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -497,8 +497,8 @@ pipeline { withMounts([["projects", "projects/hydra_audio", "hydra_audio_test_skype"]]) { withEnv(["hydra_audio_PATH=$hydra_audio_test_skype_PATH"]) { sh "./make_dirs.sh" - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -511,8 +511,8 @@ pipeline { dir("${REPO}/test/lib_aec/aec_unit_tests") { viewEnv() { withVenv { - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -529,17 +529,17 @@ pipeline { sh 'mv excluded_tests_quick.txt excluded_tests.txt' } } - sh "python generate_audio.py" - sh "pytest -n 2 --junitxml=pytest_result.xml test_process_audio.py" - sh "cp pytest_result.xml results_process.xml" + //sh "python generate_audio.py" + //sh "pytest -n 2 --junitxml=pytest_result.xml test_process_audio.py" + //sh "cp pytest_result.xml results_process.xml" catchError { - sh "pytest --junitxml=pytest_result.xml test_check_output.py" + //sh "pytest --junitxml=pytest_result.xml test_check_output.py" } - sh "cp pytest_result.xml results_check.xml" - sh "python parse_results.py" - sh "pytest --junitxml=pytest_results.xml test_evaluate_results.py" - sh "cp pytest_result.xml results_final.xml" - junit "results_final.xml" + //sh "cp pytest_result.xml results_check.xml" + //sh "python parse_results.py" + //sh "pytest --junitxml=pytest_results.xml test_evaluate_results.py" + //sh "cp pytest_result.xml results_final.xml" + //junit "results_final.xml" } } } @@ -550,8 +550,8 @@ pipeline { dir("${REPO}/test/lib_agc/test_process_frame") { viewEnv() { withVenv { - sh "pytest -n 2 --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -n 2 --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -562,8 +562,8 @@ pipeline { dir("${REPO}/test/test_hpf") { viewEnv() { withVenv { - sh "pytest --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -578,9 +578,9 @@ pipeline { withVenv { echo "PIPELINE_FULL_RUN set as " + env.PIPELINE_FULL_RUN //Note we have 2 xcore targets and we can run x86 threads too. But in case we have only xcore jobs in the config, limit to 4 so we don't timeout waiting for xtags - sh "pytest -n 4 --junitxml=pytest_result.xml -vv" + //sh "pytest -n 4 --junitxml=pytest_result.xml -vv" // sh "pytest -s --junitxml=pytest_result.xml" //Debug, run single threaded with STDIO captured - junit "pytest_result.xml" + //junit "pytest_result.xml" // Archive below (always section) even if fails } } @@ -593,16 +593,18 @@ pipeline { post { always { //AEC aretfacts - archiveArtifacts artifacts: "${REPO}/test/lib_adec/test_adec_profile/**/adec_prof*.log", fingerprint: true + //archiveArtifacts artifacts: "${REPO}/test/lib_adec/test_adec_profile/**/adec_prof*.log", fingerprint: true //NS artefacts - archiveArtifacts artifacts: "${REPO}/test/lib_ns/test_ns_profile/ns_prof.log", fingerprint: true + //archiveArtifacts artifacts: "${REPO}/test/lib_ns/test_ns_profile/ns_prof.log", fingerprint: true + //IC artefacts + archiveArtifacts artifacts: "${REPO}/test/lib_ic/characterise_c_py/*.wav", fingerprint true //VNR artifacts - archiveArtifacts artifacts: "${REPO}/test/lib_vnr/test_wav_vnr/*.png", fingerprint: true - archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/*.png", fingerprint: true - archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/vnr_prof.log", fingerprint: true + //archiveArtifacts artifacts: "${REPO}/test/lib_vnr/test_wav_vnr/*.png", fingerprint: true + //archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/*.png", fingerprint: true + //archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/vnr_prof.log", fingerprint: true //Pipelines tests - archiveArtifacts artifacts: "${REPO}/test/pipeline/results_*.csv", fingerprint: true - archiveArtifacts artifacts: "${REPO}/test/pipeline/keyword_input_*/*.wav", fingerprint: true + //archiveArtifacts artifacts: "${REPO}/test/pipeline/results_*.csv", fingerprint: true + //archiveArtifacts artifacts: "${REPO}/test/pipeline/keyword_input_*/*.wav", fingerprint: true } cleanup { cleanWs() diff --git a/test/lib_ic/characterise_c_py/test_polar_response.py b/test/lib_ic/characterise_c_py/test_polar_response.py index 7f930981c..7e8a5863f 100644 --- a/test/lib_ic/characterise_c_py/test_polar_response.py +++ b/test/lib_ic/characterise_c_py/test_polar_response.py @@ -39,4 +39,4 @@ def test_compare_polar_reponse(): run_c=True) for (i, atten_py, atten_c) in zip(angles, results[0], results[1]): print(f"Angle: {i}, PY {atten_py}, C {atten_c}") - assert abs(atten_py - atten_c) < 1, "Angle: {}, PY {}, C {}".format(i, atten_py, atten_c) + #assert abs(atten_py - atten_c) < 1, "Angle: {}, PY {}, C {}".format(i, atten_py, atten_c) From 60d9c0892029c2836b1f941cafc9ad929c26f066 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 11:37:37 +0100 Subject: [PATCH 36/92] typo --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 49fe0c33a..b7be0e5a7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -597,7 +597,7 @@ pipeline { //NS artefacts //archiveArtifacts artifacts: "${REPO}/test/lib_ns/test_ns_profile/ns_prof.log", fingerprint: true //IC artefacts - archiveArtifacts artifacts: "${REPO}/test/lib_ic/characterise_c_py/*.wav", fingerprint true + archiveArtifacts artifacts: "${REPO}/test/lib_ic/characterise_c_py/*.wav", fingerprint: true //VNR artifacts //archiveArtifacts artifacts: "${REPO}/test/lib_vnr/test_wav_vnr/*.png", fingerprint: true //archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/*.png", fingerprint: true From a9a626f803bc1f5eff2a1bdef1b17e56585d2438 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 12:02:33 +0100 Subject: [PATCH 37/92] fixing path --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b7be0e5a7..e4b8041b2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -597,7 +597,7 @@ pipeline { //NS artefacts //archiveArtifacts artifacts: "${REPO}/test/lib_ns/test_ns_profile/ns_prof.log", fingerprint: true //IC artefacts - archiveArtifacts artifacts: "${REPO}/test/lib_ic/characterise_c_py/*.wav", fingerprint: true + archiveArtifacts artifacts: "${REPO}/test/lib_ic/characterise_c_py/pytest_audio/*.wav", fingerprint: true //VNR artifacts //archiveArtifacts artifacts: "${REPO}/test/lib_vnr/test_wav_vnr/*.png", fingerprint: true //archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/*.png", fingerprint: true From 554aa264cc6cc86e638db06ea19899ab6e2d5082 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 13:53:34 +0100 Subject: [PATCH 38/92] mathcing delta to avona --- test/lib_ic/characterise_c_py/char_test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/characterise_c_py/char_test.json b/test/lib_ic/characterise_c_py/char_test.json index 68245ea43..c8f78e7e9 100644 --- a/test/lib_ic/characterise_c_py/char_test.json +++ b/test/lib_ic/characterise_c_py/char_test.json @@ -3,7 +3,7 @@ "proc_frame_length": 512, "phases": 10, "mu" : 0.36956599983386695, - "delta" : 0.0156249999963620211929, + "delta" : 7.450580593454381e-09, "y_channel_delay": 180, "K" : 1, "lamda" : 0.9995117188, From 6a4c429a1a219a71238398acc081fea1b39c466a Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 14:14:37 +0100 Subject: [PATCH 39/92] enabling nyqist in pyhton --- test/lib_ic/characterise_c_py/char_test.json | 2 +- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/characterise_c_py/char_test.json b/test/lib_ic/characterise_c_py/char_test.json index c8f78e7e9..62e48c3fd 100644 --- a/test/lib_ic/characterise_c_py/char_test.json +++ b/test/lib_ic/characterise_c_py/char_test.json @@ -9,7 +9,7 @@ "lamda" : 0.9995117188, "gamma" : 2.0, "leakage" : 0.995, - "remove_NQ" : true, + "remove_NQ" : false, "vnr_model" : "../../../../py_vnr/model_output_0_0_2/trained_model.h5", "adaption_config": "IC_ADAPTION_FORCE_ON" } \ No newline at end of file diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index f4bfe89ce..dd59315ed 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -45,7 +45,7 @@ def __init__(self): gamma = 2.0, leakage = 0.995, y_channel_delay = y_channel_delay, - remove_NQ = True, + remove_NQ = False, vnr_model = '../../../../py_vnr/model_output_0_0_2/trained_model.h5', adaption_config = 'IC_ADAPTION_FORCE_ON' ) From 66923699685b57103a21f556b744ac7aea1ccb2d Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 14:15:46 +0100 Subject: [PATCH 40/92] swapping channels back --- test/lib_ic/characterise_c_py/characterise_c_py.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index c839012ed..6ab158bc1 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -126,13 +126,13 @@ def get_attenuation(input_file, output_file, audio_dir="."): attenuation = [] - for i in range(len(in_power_r) // SAMPLE_RATE): + for i in range(len(in_power_l) // SAMPLE_RATE): window_start = i*SAMPLE_RATE window_end = window_start + SAMPLE_RATE #av_in_power_l = np.mean(in_power_l[window_start:window_end]) #av_in_power_r = np.mean(in_power_r[window_start:window_end]) #av_in_power = (av_in_power_l + av_in_power_r) / 2 - av_in_power = np.mean(in_power_r[window_start:window_end]) + av_in_power = np.mean(in_power_l[window_start:window_end]) av_out_power = np.mean(out_power[window_start:window_end]) new_atten = 10 * np.log10(av_in_power / av_out_power) if av_out_power != 0 else 1000 From cc661b83c2dfa63809fc5c6f0a15e2fbfaf723d2 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 14:43:56 +0100 Subject: [PATCH 41/92] small cleanup and debug print --- test/lib_ic/characterise_c_py/characterise_c_py.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 6ab158bc1..f3bba9a7b 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -120,19 +120,18 @@ def get_attenuation(input_file, output_file, audio_dir="."): out_wav_data, out_channel_count, out_file_length = awu.parse_audio(out_wav_file) # Calculate EWM of audio power in 1s window - in_power_l = np.power(in_wav_data[0, :], 2) - in_power_r = np.power(in_wav_data[1, :], 2) + print(input_file, ' has ', in_channel_count, ' channels and ', in_wav_data.shape, 'shape') + print(output_file, ' has ', out_channel_count, 'channels and ', out_wav_data.shape, 'shape') + in_power = np.power(in_wav_data[0, :], 2) out_power = np.power(out_wav_data[0, :], 2) attenuation = [] - for i in range(len(in_power_l) // SAMPLE_RATE): + for i in range(len(in_power) // SAMPLE_RATE): window_start = i*SAMPLE_RATE window_end = window_start + SAMPLE_RATE - #av_in_power_l = np.mean(in_power_l[window_start:window_end]) - #av_in_power_r = np.mean(in_power_r[window_start:window_end]) - #av_in_power = (av_in_power_l + av_in_power_r) / 2 - av_in_power = np.mean(in_power_l[window_start:window_end]) + + av_in_power = np.mean(in_power[window_start:window_end]) av_out_power = np.mean(out_power[window_start:window_end]) new_atten = 10 * np.log10(av_in_power / av_out_power) if av_out_power != 0 else 1000 From ef4123aa400a4800b686189514f5f9ff0123f8da Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 15:42:23 +0100 Subject: [PATCH 42/92] enabling a spec test --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e4b8041b2..a4fb6e9b1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -202,16 +202,16 @@ pipeline { //This test compares the model and C implementation over a range of scenarious for: //convergence_time, db_suppression, maximum noise added to input (to test for stability) //and expected group delay. It will fail if these are not met. - //sh "pytest -n 2 --junitxml=pytest_result.xml" + sh "pytest -s --junitxml=pytest_result.xml" //junit "pytest_result.xml" - //sh "python print_stats.py > ic_spec_summary.txt" + sh "python print_stats.py > ic_spec_summary.txt" //This script generates a number of polar plots of attenuation vs null point angle vs freq //It currently only uses the python model to do this. It takes about 40 mins for all plots //and generates a series of IC_performance_xxxHz.svg files which could be archived // sh "python plot_ic.py" } } - //archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true + archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true } } } From c2d0d640977f716e8a4d6719097d6f8b49e9adb5 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 16:12:35 +0100 Subject: [PATCH 43/92] disabling assertion --- test/lib_ic/test_ic_spec/test_ic_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index f3667c0ed..22a0b28f5 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -309,5 +309,5 @@ def test_all(test_input, model, record_property): print("{}: {}".format(test_case.name, model)) # Assert checks criteria = [converged, stable, delayed] - assert np.all(criteria), " and ".join([str(c) for c in criteria]) + #assert np.all(criteria), " and ".join([str(c) for c in criteria]) From 198e750084d927eb8bcf54a0f4913653397364f5 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 16:28:36 +0100 Subject: [PATCH 44/92] debug print and small fix --- test/lib_ic/test_ic_spec/test_ic_spec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 22a0b28f5..a7506f773 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -274,7 +274,8 @@ def check_delay(record_property, test_case, input_audio, output_audio): output_audio = np.asarray(output_audio, dtype=float) / np.iinfo(np.int32).max frame_in = input_audio[0][:frame_advance*2] - frame_out = output_audio[0][:frame_advance*2] + print(output_audio.shape) + frame_out = output_audio[0, :frame_advance*2] corr = scipy.signal.correlate(frame_in, frame_out, mode='same') delay = 240 - np.argmax(corr) From d9a3a74d3b6e4bc4535cba1f1a8c132224137133 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 10 May 2022 16:51:59 +0100 Subject: [PATCH 45/92] array indexing workaround --- test/lib_ic/characterise_c_py/characterise_c_py.py | 6 +++++- test/lib_ic/test_ic_spec/test_ic_spec.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index f3bba9a7b..7a5b597cd 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -123,7 +123,11 @@ def get_attenuation(input_file, output_file, audio_dir="."): print(input_file, ' has ', in_channel_count, ' channels and ', in_wav_data.shape, 'shape') print(output_file, ' has ', out_channel_count, 'channels and ', out_wav_data.shape, 'shape') in_power = np.power(in_wav_data[0, :], 2) - out_power = np.power(out_wav_data[0, :], 2) + if len(out_wav_data.shape) > 1: + out_power = np.power(out_wav_data[0, :], 2) + else: + out_power = np.power(out_wav_data, 2) + #out_power = np.power(out_wav_data[0, :], 2) attenuation = [] diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index a7506f773..9f033231b 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -275,7 +275,10 @@ def check_delay(record_property, test_case, input_audio, output_audio): frame_in = input_audio[0][:frame_advance*2] print(output_audio.shape) - frame_out = output_audio[0, :frame_advance*2] + if len(output_audio.shape) > 1: + frame_out = output_audio[0, :frame_advance*2] + else: + frame_out = output_audio[:frame_advance*2] corr = scipy.signal.correlate(frame_in, frame_out, mode='same') delay = 240 - np.argmax(corr) From 71e12250f7d34f49bfb10c42112c6b8680bb0d4e Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 11 May 2022 10:13:46 +0100 Subject: [PATCH 46/92] adding debug prints --- test/lib_ic/test_ic_spec/test_ic_spec.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 9f033231b..6f8485887 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -217,11 +217,15 @@ def get_suppression_arr(input_audio, output_audio): out_rms = rms(output_audio[i:i+frame_advance]) if in_rms == 0 or out_rms == 0: suppression_db = 0.0 + print('case 1 ', in_rms, ' ', out_rms) elif i - frame_advance < ICSpec.expected_delay: suppression_db = 0.0 + print('case 2 ', in_rms, ' ', out_rms, ' ', i) else: suppression_db = 20 * np.log10(in_rms / out_rms) + print('case 3 ', in_rms, ' ', out_rms, ' ', suppression_db) suppression_arr[int(i//frame_advance)] = suppression_db + print('supression_arr = ', suppression_arr, '\n') return suppression_arr @@ -274,7 +278,7 @@ def check_delay(record_property, test_case, input_audio, output_audio): output_audio = np.asarray(output_audio, dtype=float) / np.iinfo(np.int32).max frame_in = input_audio[0][:frame_advance*2] - print(output_audio.shape) + #print(output_audio.shape) if len(output_audio.shape) > 1: frame_out = output_audio[0, :frame_advance*2] else: @@ -298,6 +302,7 @@ def check_delay(record_property, test_case, input_audio, output_audio): @pytest.mark.parametrize('model', xe_files) def test_all(test_input, model, record_property): test_case, input_audio = test_input + print("\n{}: {}\n".format(test_case.name, model)) output_audio = process_audio(model, input_audio, test_case.get_test_name()) suppression_arr = get_suppression_arr(input_audio, output_audio) @@ -310,7 +315,6 @@ def test_all(test_input, model, record_property): stable = check_stability(record_property, test_case, suppression_arr) delayed = check_delay(record_property, test_case, input_audio, output_audio) - print("{}: {}".format(test_case.name, model)) # Assert checks criteria = [converged, stable, delayed] #assert np.all(criteria), " and ".join([str(c) for c in criteria]) From e00818e447b450526db7b369ccdc9ccfe89d9f6a Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 11 May 2022 11:03:38 +0100 Subject: [PATCH 47/92] trying workaround to avaoid nan values --- test/lib_ic/test_ic_spec/test_ic_spec.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 6f8485887..14e7bdfa4 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -214,7 +214,11 @@ def get_suppression_arr(input_audio, output_audio): for i in np.arange(0, num_frames*frame_advance, frame_advance): i = int(i) in_rms = rms(input_audio[0][i:i+frame_advance]) - out_rms = rms(output_audio[i:i+frame_advance]) + if len(output_audio.shape) > 1: + out_rms = rms(output_audio[0, i:i+frame_advance]) + else: + out_rms = rms(output_audio[i:i+frame_advance]) + #out_rms = rms(output_audio[i:i+frame_advance]) if in_rms == 0 or out_rms == 0: suppression_db = 0.0 print('case 1 ', in_rms, ' ', out_rms) From 022feb8566764a17f1ae1d0f88ab8f036539771f Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 11 May 2022 16:48:16 +0100 Subject: [PATCH 48/92] experimenting with parameters --- .../lib_ic/characterise_c_py/characterise_c_py.py | 2 +- test/lib_ic/characterise_c_py/test_test.json | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 test/lib_ic/characterise_c_py/test_test.json diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 7a5b597cd..0a14ce862 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -88,7 +88,7 @@ def generate_test_audio(filename, audio_dir, max_freq, db, angle_theta, rt60, sa def process_py(input_file, output_file, audio_dir="."): - config_file = 'char_test.json' + config_file = 'test_test.json' twic.test_file(os.path.join(audio_dir, input_file), os.path.join(audio_dir, output_file), diff --git a/test/lib_ic/characterise_c_py/test_test.json b/test/lib_ic/characterise_c_py/test_test.json new file mode 100644 index 000000000..1e49f20d7 --- /dev/null +++ b/test/lib_ic/characterise_c_py/test_test.json @@ -0,0 +1,15 @@ +{ + "frame_advance": 240, + "proc_frame_length": 512, + "phases": 10, + "mu" : 1.0, + "delta" : 0.015624999996362021, + "y_channel_delay": 180, + "K" : 1, + "lamda" : 0.99951171875, + "gamma" : 2.0, + "leakage" : 1.0, + "remove_NQ" : false, + "vnr_model" : "../../../../py_vnr/model_output/trained_model.h5", + "adaption_config": "IC_ADAPTION_FORCE_ON" +} \ No newline at end of file From 2377414613bf08e002226d3762627f406d4f5797 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 11 May 2022 17:13:29 +0100 Subject: [PATCH 49/92] removing debug prints and bringing back assertions --- test/lib_ic/characterise_c_py/characterise_c_py.py | 4 ++-- test/lib_ic/characterise_c_py/test_polar_response.py | 2 +- test/lib_ic/test_ic_spec/test_ic_spec.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 0a14ce862..71772529a 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -120,8 +120,8 @@ def get_attenuation(input_file, output_file, audio_dir="."): out_wav_data, out_channel_count, out_file_length = awu.parse_audio(out_wav_file) # Calculate EWM of audio power in 1s window - print(input_file, ' has ', in_channel_count, ' channels and ', in_wav_data.shape, 'shape') - print(output_file, ' has ', out_channel_count, 'channels and ', out_wav_data.shape, 'shape') + #print(input_file, ' has ', in_channel_count, ' channels and ', in_wav_data.shape, 'shape') + #print(output_file, ' has ', out_channel_count, 'channels and ', out_wav_data.shape, 'shape') in_power = np.power(in_wav_data[0, :], 2) if len(out_wav_data.shape) > 1: out_power = np.power(out_wav_data[0, :], 2) diff --git a/test/lib_ic/characterise_c_py/test_polar_response.py b/test/lib_ic/characterise_c_py/test_polar_response.py index 7e8a5863f..7f930981c 100644 --- a/test/lib_ic/characterise_c_py/test_polar_response.py +++ b/test/lib_ic/characterise_c_py/test_polar_response.py @@ -39,4 +39,4 @@ def test_compare_polar_reponse(): run_c=True) for (i, atten_py, atten_c) in zip(angles, results[0], results[1]): print(f"Angle: {i}, PY {atten_py}, C {atten_c}") - #assert abs(atten_py - atten_c) < 1, "Angle: {}, PY {}, C {}".format(i, atten_py, atten_c) + assert abs(atten_py - atten_c) < 1, "Angle: {}, PY {}, C {}".format(i, atten_py, atten_c) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 14e7bdfa4..dbc358f2d 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -221,15 +221,15 @@ def get_suppression_arr(input_audio, output_audio): #out_rms = rms(output_audio[i:i+frame_advance]) if in_rms == 0 or out_rms == 0: suppression_db = 0.0 - print('case 1 ', in_rms, ' ', out_rms) + #print('case 1 ', in_rms, ' ', out_rms) elif i - frame_advance < ICSpec.expected_delay: suppression_db = 0.0 - print('case 2 ', in_rms, ' ', out_rms, ' ', i) + #print('case 2 ', in_rms, ' ', out_rms, ' ', i) else: suppression_db = 20 * np.log10(in_rms / out_rms) - print('case 3 ', in_rms, ' ', out_rms, ' ', suppression_db) + #print('case 3 ', in_rms, ' ', out_rms, ' ', suppression_db) suppression_arr[int(i//frame_advance)] = suppression_db - print('supression_arr = ', suppression_arr, '\n') + #print('supression_arr = ', suppression_arr, '\n') return suppression_arr @@ -321,5 +321,5 @@ def test_all(test_input, model, record_property): # Assert checks criteria = [converged, stable, delayed] - #assert np.all(criteria), " and ".join([str(c) for c in criteria]) + assert np.all(criteria), " and ".join([str(c) for c in criteria]) From e2d6b7e04692dbd07e7568c40c863972609afa1a Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 11 May 2022 17:47:15 +0100 Subject: [PATCH 50/92] setting a different config for spec test --- test/lib_ic/test_ic_spec/test_ic_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index dbc358f2d..1155de139 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -137,7 +137,7 @@ def write_output(test_name, output, c_or_py): def process_py(input_data, test_name): file_length = input_data.shape[1] - config_file = 'spec_test.json' + config_file = '../characterise_c_py/test_test.json' ic_parameters = json_to_dict(config_file) output, Mu, Input_vnr_pred, Output_vnr_pred, Control_flag = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) From 800424a2f1e5a974db1347d6737e22831bc19dc4 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Wed, 11 May 2022 17:50:36 +0100 Subject: [PATCH 51/92] diasbling jenkins --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 35307bd5c..20fe9f276 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -315,7 +315,7 @@ pipeline { dir("${REPO}/test/lib_vnr/vnr_unit_tests") { viewEnv() { withVenv { - sh "pytest -n 2 --junitxml=pytest_result.xml" + //sh "pytest -n 2 --junitxml=pytest_result.xml" } } } From 8dbde3629a04c9a91814b4efd1f479e08376a5c2 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 12 May 2022 11:13:58 +0100 Subject: [PATCH 52/92] changing the config file --- test/lib_ic/characterise_c_py/char_test.json | 15 --------------- .../lib_ic/characterise_c_py/characterise_c_py.py | 2 +- .../{test_test.json => test_config.json} | 2 +- test/lib_ic/test_ic_spec/test_ic_spec.py | 2 +- 4 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 test/lib_ic/characterise_c_py/char_test.json rename test/lib_ic/characterise_c_py/{test_test.json => test_config.json} (92%) diff --git a/test/lib_ic/characterise_c_py/char_test.json b/test/lib_ic/characterise_c_py/char_test.json deleted file mode 100644 index 62e48c3fd..000000000 --- a/test/lib_ic/characterise_c_py/char_test.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "frame_advance": 240, - "proc_frame_length": 512, - "phases": 10, - "mu" : 0.36956599983386695, - "delta" : 7.450580593454381e-09, - "y_channel_delay": 180, - "K" : 1, - "lamda" : 0.9995117188, - "gamma" : 2.0, - "leakage" : 0.995, - "remove_NQ" : false, - "vnr_model" : "../../../../py_vnr/model_output_0_0_2/trained_model.h5", - "adaption_config": "IC_ADAPTION_FORCE_ON" -} \ No newline at end of file diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 71772529a..76dce837c 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -88,7 +88,7 @@ def generate_test_audio(filename, audio_dir, max_freq, db, angle_theta, rt60, sa def process_py(input_file, output_file, audio_dir="."): - config_file = 'test_test.json' + config_file = 'test_config.json' twic.test_file(os.path.join(audio_dir, input_file), os.path.join(audio_dir, output_file), diff --git a/test/lib_ic/characterise_c_py/test_test.json b/test/lib_ic/characterise_c_py/test_config.json similarity index 92% rename from test/lib_ic/characterise_c_py/test_test.json rename to test/lib_ic/characterise_c_py/test_config.json index 1e49f20d7..da875e99a 100644 --- a/test/lib_ic/characterise_c_py/test_test.json +++ b/test/lib_ic/characterise_c_py/test_config.json @@ -2,7 +2,7 @@ "frame_advance": 240, "proc_frame_length": 512, "phases": 10, - "mu" : 1.0, + "mu" : 0.3695650300753292, "delta" : 0.015624999996362021, "y_channel_delay": 180, "K" : 1, diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 1155de139..43067d469 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -137,7 +137,7 @@ def write_output(test_name, output, c_or_py): def process_py(input_data, test_name): file_length = input_data.shape[1] - config_file = '../characterise_c_py/test_test.json' + config_file = '../characterise_c_py/test_config.json' ic_parameters = json_to_dict(config_file) output, Mu, Input_vnr_pred, Output_vnr_pred, Control_flag = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) From 434e85051ff61409b485547765c3c8dd7ba9dbb8 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 12 May 2022 11:33:23 +0100 Subject: [PATCH 53/92] adapting to the changed py_ic api --- test/lib_ic/characterise_c_py/test_config.json | 2 +- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- test/lib_ic/test_ic_spec/spec_test.json | 15 --------------- 3 files changed, 2 insertions(+), 17 deletions(-) delete mode 100644 test/lib_ic/test_ic_spec/spec_test.json diff --git a/test/lib_ic/characterise_c_py/test_config.json b/test/lib_ic/characterise_c_py/test_config.json index da875e99a..3d85eca53 100644 --- a/test/lib_ic/characterise_c_py/test_config.json +++ b/test/lib_ic/characterise_c_py/test_config.json @@ -12,4 +12,4 @@ "remove_NQ" : false, "vnr_model" : "../../../../py_vnr/model_output/trained_model.h5", "adaption_config": "IC_ADAPTION_FORCE_ON" -} \ No newline at end of file +} diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index dd59315ed..77b31d1b3 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -59,7 +59,7 @@ def process_frame(self, frame): frame_int = pvc.float_to_int32(frame) output_py, Error_ap = self.ic.process_frame(frame) - mu, input_vnr_pred, output_vnr_pred, control_flag = self.ic.mu_control_system(Error_ap) + mu, control_flag = self.ic.mu_control_system() #print('mu = ', mu, ', in_vnr = ', input_vnr_pred, ', out_vnr = ', output_vnr_pred, 'flag = ', control_flag) self.ic.adapt(Error_ap) diff --git a/test/lib_ic/test_ic_spec/spec_test.json b/test/lib_ic/test_ic_spec/spec_test.json deleted file mode 100644 index acfe53a82..000000000 --- a/test/lib_ic/test_ic_spec/spec_test.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "frame_advance": 240, - "proc_frame_length": 512, - "phases": 10, - "mu" : 1.0, - "delta" : 7.450580593454381e-09, - "y_channel_delay": 180, - "K" : 1, - "lamda" : 0.9995117188, - "gamma" : 2.0, - "leakage" : 0.995, - "remove_NQ" : true, - "vnr_model" : "./../../py_vnr/model_output_0_0_2/trained_model.h5", - "adaption_config": "IC_ADAPTION_FORCE_ON" -} \ No newline at end of file From bff8f510f3fe03bcb50a1392f093781e41b4777e Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 12 May 2022 14:41:25 +0100 Subject: [PATCH 54/92] adding error array print --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 77b31d1b3..6b81474be 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -49,7 +49,7 @@ def __init__(self): vnr_model = '../../../../py_vnr/model_output_0_0_2/trained_model.h5', adaption_config = 'IC_ADAPTION_FORCE_ON' ) - + self.error_ap = None ic_test_lib.test_init() @@ -59,6 +59,7 @@ def process_frame(self, frame): frame_int = pvc.float_to_int32(frame) output_py, Error_ap = self.ic.process_frame(frame) + self.error_ap = np.fft.irfft(Error_ap) mu, control_flag = self.ic.mu_control_system() #print('mu = ', mu, ', in_vnr = ', input_vnr_pred, ', out_vnr = ', output_vnr_pred, 'flag = ', control_flag) self.ic.adapt(Error_ap) @@ -74,7 +75,7 @@ def process_frame(self, frame): vad = int(0) ic_test_lib.test_adapt(vad, output_c_ptr) - state = ic_test_lib.test_get_state() + #state = ic_test_lib.test_get_state() #print('mu_c = ', pvc.float_s32_to_float(state.mu[0][0]), ', nu_py = ', self.ic.mu) #print('leakage_c = ', pvc.float_s32_to_float(state.ic_adaption_controller_state.adaption_controller_config.leakage_alpha), ', leakage_py = ', self.ic.leakage) # print(pvc.float_s32_to_float(state.config_params.delta)) @@ -100,6 +101,15 @@ def test_frame_compare(pre_test_stuff): output_wav_data[0, frame_start: frame_start + frame_advance] = output_py output_wav_data[1, frame_start: frame_start + frame_advance] = output_c + + if frame_start < 240: + state = ic_test_lib.test_get_state() + c_error_int32 = np.asarray(state.Error, dtype = np.int32) + c_error_exp = state.error_bfp.exp + c_error = np.power(c_error_int32, c_error_exp) + print(c_error[:10]) + print(icc.error_ap[:10]) + #Write a copy of the output for post analysis if needed scipy.io.wavfile.write(output_file, input_rate, pvc.float_to_int32(output_wav_data.T)) From 44a2f08274d3fc24bb16419e6a7d828f16571491 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 12 May 2022 15:13:19 +0100 Subject: [PATCH 55/92] removing int32 cast --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 6b81474be..fe0e3c3c4 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -104,7 +104,7 @@ def test_frame_compare(pre_test_stuff): if frame_start < 240: state = ic_test_lib.test_get_state() - c_error_int32 = np.asarray(state.Error, dtype = np.int32) + c_error_int32 = np.asarray(state.Error) c_error_exp = state.error_bfp.exp c_error = np.power(c_error_int32, c_error_exp) print(c_error[:10]) From ae68cd9643996ba9641aac5f977e04b70d308ccb Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 12 May 2022 15:38:52 +0100 Subject: [PATCH 56/92] trying the other way to get error array --- test/lib_ic/py_c_frame_compare/ic_test.c | 1 + test/lib_ic/py_c_frame_compare/test_ic_frame.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/ic_test.c b/test/lib_ic/py_c_frame_compare/ic_test.c index 754fc24f3..496f0aad7 100644 --- a/test/lib_ic/py_c_frame_compare/ic_test.c +++ b/test/lib_ic/py_c_frame_compare/ic_test.c @@ -14,6 +14,7 @@ void test_init(void){ } ic_state_t test_get_state(void){ + bfp_s32_use_exponent(&ic_state.error_bfp, -31); return ic_state; } diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index fe0e3c3c4..f2f1c1706 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -105,8 +105,9 @@ def test_frame_compare(pre_test_stuff): if frame_start < 240: state = ic_test_lib.test_get_state() c_error_int32 = np.asarray(state.Error) - c_error_exp = state.error_bfp.exp - c_error = np.power(c_error_int32, c_error_exp) + #c_error_exp = state.error_bfp.exp + #c_error = np.power(c_error_int32, c_error_exp) + c_error = pvc.int32_to_float(c_error_int32) print(c_error[:10]) print(icc.error_ap[:10]) From 19ca4269839b91b2185ff1e28909d39c085bbcb8 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 12 May 2022 16:05:23 +0100 Subject: [PATCH 57/92] changing ic types a bit to check if python likes it --- modules/lib_ic/api/ic_state.h | 2 +- modules/lib_ic/src/ic.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/lib_ic/api/ic_state.h b/modules/lib_ic/api/ic_state.h index e748676c9..ed6d0cf02 100644 --- a/modules/lib_ic/api/ic_state.h +++ b/modules/lib_ic/api/ic_state.h @@ -168,7 +168,7 @@ typedef struct { bfp_s32_t error_bfp[IC_Y_CHANNELS]; /** Storage for Error and error mantissas. Note IFFT is done in-place * so the Error storage is reused for error. */ - complex_s32_t DWORD_ALIGNED Error[IC_Y_CHANNELS][IC_FD_FRAME_LENGTH]; + int32_t DWORD_ALIGNED Error[IC_Y_CHANNELS][IC_FRAME_LENGTH + FFT_PADDING]; /** BFP array pointing to the frequency domain estimate of transfer function. */ bfp_complex_s32_t H_hat_bfp[IC_Y_CHANNELS][IC_X_CHANNELS*IC_FILTER_PHASES]; diff --git a/modules/lib_ic/src/ic.c b/modules/lib_ic/src/ic.c index b89afd5b8..6972b4925 100644 --- a/modules/lib_ic/src/ic.c +++ b/modules/lib_ic/src/ic.c @@ -69,8 +69,8 @@ void ic_init(ic_state_t *state){ } //initialise Error for(unsigned ch=0; chError_bfp[ch], state->Error[ch], zero_exp, IC_FD_FRAME_LENGTH, 0); - bfp_s32_init(&state->error_bfp[ch], (int32_t *)state->Error[ch], zero_exp, IC_FRAME_LENGTH, 0); + bfp_complex_s32_init(&state->Error_bfp[ch], (complex_s32_t * )state->Error[ch], zero_exp, IC_FD_FRAME_LENGTH, 0); + bfp_s32_init(&state->error_bfp[ch], state->Error[ch], zero_exp, IC_FRAME_LENGTH, 0); } //Initiaise Y_hat for(unsigned ch=0; ch Date: Thu, 12 May 2022 16:21:10 +0100 Subject: [PATCH 58/92] trying int32 cast --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index f2f1c1706..4dd546c9e 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -104,10 +104,10 @@ def test_frame_compare(pre_test_stuff): if frame_start < 240: state = ic_test_lib.test_get_state() - c_error_int32 = np.asarray(state.Error) + c_error_int32 = np.asarray(state.Error, dtype=np.int32) #c_error_exp = state.error_bfp.exp #c_error = np.power(c_error_int32, c_error_exp) - c_error = pvc.int32_to_float(c_error_int32) + c_error = pvc.int32_to_float(c_error_int32[0, :]) print(c_error[:10]) print(icc.error_ap[:10]) From eb99f906e8364347947019c4ba677940fa5566f3 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 12 May 2022 16:39:22 +0100 Subject: [PATCH 59/92] testing python --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 4dd546c9e..f4f46b5c5 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -104,11 +104,11 @@ def test_frame_compare(pre_test_stuff): if frame_start < 240: state = ic_test_lib.test_get_state() - c_error_int32 = np.asarray(state.Error, dtype=np.int32) + #c_error_int32 = np.asarray(state.Error, dtype=np.int32) #c_error_exp = state.error_bfp.exp #c_error = np.power(c_error_int32, c_error_exp) - c_error = pvc.int32_to_float(c_error_int32[0, :]) - print(c_error[:10]) + c_error = pvc.int32_to_float(state.Error[0, 5]) + print(c_error) print(icc.error_ap[:10]) From 8e24ca3245039c5d05246c0967153fa03c0f6d62 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 12 May 2022 16:52:36 +0100 Subject: [PATCH 60/92] testing python --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index f4f46b5c5..39b379427 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -107,7 +107,7 @@ def test_frame_compare(pre_test_stuff): #c_error_int32 = np.asarray(state.Error, dtype=np.int32) #c_error_exp = state.error_bfp.exp #c_error = np.power(c_error_int32, c_error_exp) - c_error = pvc.int32_to_float(state.Error[0, 5]) + c_error = pvc.int32_to_float(state.Error[0][5]) print(c_error) print(icc.error_ap[:10]) From a6597e11fe02d46f84b8ab73da691f32438b862c Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 13 May 2022 10:47:23 +0100 Subject: [PATCH 61/92] indexing fix --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 39b379427..3d3c9a123 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -109,7 +109,7 @@ def test_frame_compare(pre_test_stuff): #c_error = np.power(c_error_int32, c_error_exp) c_error = pvc.int32_to_float(state.Error[0][5]) print(c_error) - print(icc.error_ap[:10]) + print(icc.error_ap[0][:10]) #Write a copy of the output for post analysis if needed From fbd5e83e4a9db06c5fe39294117b1b3be825805a Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 13 May 2022 11:03:39 +0100 Subject: [PATCH 62/92] trying to print part of the array --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 3d3c9a123..9f8bbfd04 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -107,7 +107,7 @@ def test_frame_compare(pre_test_stuff): #c_error_int32 = np.asarray(state.Error, dtype=np.int32) #c_error_exp = state.error_bfp.exp #c_error = np.power(c_error_int32, c_error_exp) - c_error = pvc.int32_to_float(state.Error[0][5]) + c_error = pvc.int32_to_float(state.Error[0][:10]) print(c_error) print(icc.error_ap[0][:10]) From 69ee62f4e6d3fbde1e6af694a954c0949c7ecaee Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 13 May 2022 11:40:36 +0100 Subject: [PATCH 63/92] workaround --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 9f8bbfd04..f8a30adc7 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -107,7 +107,7 @@ def test_frame_compare(pre_test_stuff): #c_error_int32 = np.asarray(state.Error, dtype=np.int32) #c_error_exp = state.error_bfp.exp #c_error = np.power(c_error_int32, c_error_exp) - c_error = pvc.int32_to_float(state.Error[0][:10]) + c_error = pvc.int32_to_float(state.Error[0][0:10]) print(c_error) print(icc.error_ap[0][:10]) From 95098b96e5181333ecd1ba8de8c3c8cc934ede56 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 13 May 2022 13:34:17 +0100 Subject: [PATCH 64/92] trying diffrent print --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index f8a30adc7..7cff3bdcb 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -107,9 +107,10 @@ def test_frame_compare(pre_test_stuff): #c_error_int32 = np.asarray(state.Error, dtype=np.int32) #c_error_exp = state.error_bfp.exp #c_error = np.power(c_error_int32, c_error_exp) - c_error = pvc.int32_to_float(state.Error[0][0:10]) - print(c_error) - print(icc.error_ap[0][:10]) + for i in range(0, 10, 1): + c_error = pvc.int32_to_float(state.Error[0][i]) + print('C: ', c_error, ', PY: ', icc.error_ap[0][i]) + #print(icc.error_ap[0][:10]) #Write a copy of the output for post analysis if needed From abe23aaed90d6ff08e19b33921525fcf00b15040 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 13 May 2022 14:17:24 +0100 Subject: [PATCH 65/92] adding H_hat print --- modules/lib_ic/api/ic_state.h | 2 +- modules/lib_ic/src/ic.c | 2 +- test/lib_ic/py_c_frame_compare/ic_test.c | 3 +++ test/lib_ic/py_c_frame_compare/test_ic_frame.py | 8 ++++++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/modules/lib_ic/api/ic_state.h b/modules/lib_ic/api/ic_state.h index ed6d0cf02..2b61e3d98 100644 --- a/modules/lib_ic/api/ic_state.h +++ b/modules/lib_ic/api/ic_state.h @@ -173,7 +173,7 @@ typedef struct { /** BFP array pointing to the frequency domain estimate of transfer function. */ bfp_complex_s32_t H_hat_bfp[IC_Y_CHANNELS][IC_X_CHANNELS*IC_FILTER_PHASES]; /** Storage for H_hat mantissas. */ - complex_s32_t DWORD_ALIGNED H_hat[IC_Y_CHANNELS][IC_FILTER_PHASES*IC_X_CHANNELS][IC_FD_FRAME_LENGTH]; + int32_t DWORD_ALIGNED H_hat[IC_Y_CHANNELS][IC_FILTER_PHASES*IC_X_CHANNELS][IC_FRAME_LENGTH + FFT_PADDING]; /** BFP array pointing to the frequency domain X input history used for calculating normalisation. */ bfp_complex_s32_t X_fifo_bfp[IC_X_CHANNELS][IC_FILTER_PHASES]; diff --git a/modules/lib_ic/src/ic.c b/modules/lib_ic/src/ic.c index 6972b4925..c0ee8d7f5 100644 --- a/modules/lib_ic/src/ic.c +++ b/modules/lib_ic/src/ic.c @@ -57,7 +57,7 @@ void ic_init(ic_state_t *state){ //H_hat for(unsigned ch=0; chH_hat_bfp[ch][ph], state->H_hat[ch][ph], zero_exp, IC_FD_FRAME_LENGTH, 0); + bfp_complex_s32_init(&state->H_hat_bfp[ch][ph], (complex_s32_t *)state->H_hat[ch][ph], zero_exp, IC_FD_FRAME_LENGTH, 0); } } //X_fifo diff --git a/test/lib_ic/py_c_frame_compare/ic_test.c b/test/lib_ic/py_c_frame_compare/ic_test.c index 496f0aad7..266a9267c 100644 --- a/test/lib_ic/py_c_frame_compare/ic_test.c +++ b/test/lib_ic/py_c_frame_compare/ic_test.c @@ -15,6 +15,9 @@ void test_init(void){ ic_state_t test_get_state(void){ bfp_s32_use_exponent(&ic_state.error_bfp, -31); + for(int v = 0; v < IC_FILTER_PHASES; v++){ + bfp_complex_s32_use_exponent(&ic_state.H_hat_bfp[0][v]); + } return ic_state; } diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 7cff3bdcb..dbf42f640 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -107,9 +107,17 @@ def test_frame_compare(pre_test_stuff): #c_error_int32 = np.asarray(state.Error, dtype=np.int32) #c_error_exp = state.error_bfp.exp #c_error = np.power(c_error_int32, c_error_exp) + print('error_ap:') for i in range(0, 10, 1): c_error = pvc.int32_to_float(state.Error[0][i]) print('C: ', c_error, ', PY: ', icc.error_ap[0][i]) + print('H_hat:') + for i in range(0, 10, 1): + c_H_hat = pvc.int32_to_float(state.H_hat[0][0][i]) + py_H_hat = icc.ic.H[0][int(i/2)].real + if (i / 2) == (int(i / 2)): + py_H_hat = icc.ic.H[0][int(i/2)].imag + print('C: ', c_H_hat, ', PY: ', py_H_hat) #print(icc.error_ap[0][:10]) From 3c5563556bcdf68a2457422f4a5708512467f07a Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 13 May 2022 14:28:59 +0100 Subject: [PATCH 66/92] no exp fix --- test/lib_ic/py_c_frame_compare/ic_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/ic_test.c b/test/lib_ic/py_c_frame_compare/ic_test.c index 266a9267c..aa7502a11 100644 --- a/test/lib_ic/py_c_frame_compare/ic_test.c +++ b/test/lib_ic/py_c_frame_compare/ic_test.c @@ -16,7 +16,7 @@ void test_init(void){ ic_state_t test_get_state(void){ bfp_s32_use_exponent(&ic_state.error_bfp, -31); for(int v = 0; v < IC_FILTER_PHASES; v++){ - bfp_complex_s32_use_exponent(&ic_state.H_hat_bfp[0][v]); + bfp_complex_s32_use_exponent(&ic_state.H_hat_bfp[0][v], -31); } return ic_state; } From 91d21fa8000243bd20ba5b38e4111924c8204642 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 13 May 2022 15:11:42 +0100 Subject: [PATCH 67/92] adding sigma_xx to print --- test/lib_ic/py_c_frame_compare/ic_test.c | 3 ++- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/ic_test.c b/test/lib_ic/py_c_frame_compare/ic_test.c index aa7502a11..b4c30fe5a 100644 --- a/test/lib_ic/py_c_frame_compare/ic_test.c +++ b/test/lib_ic/py_c_frame_compare/ic_test.c @@ -14,10 +14,11 @@ void test_init(void){ } ic_state_t test_get_state(void){ - bfp_s32_use_exponent(&ic_state.error_bfp, -31); + bfp_s32_use_exponent(&ic_state.error_bfp[0], -31); for(int v = 0; v < IC_FILTER_PHASES; v++){ bfp_complex_s32_use_exponent(&ic_state.H_hat_bfp[0][v], -31); } + bfp_s32_use_exponent(&ic_state.sigma_XX_bfp[0], -31); return ic_state; } diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index dbf42f640..202bb2c2a 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -104,9 +104,7 @@ def test_frame_compare(pre_test_stuff): if frame_start < 240: state = ic_test_lib.test_get_state() - #c_error_int32 = np.asarray(state.Error, dtype=np.int32) - #c_error_exp = state.error_bfp.exp - #c_error = np.power(c_error_int32, c_error_exp) + print('error_ap:') for i in range(0, 10, 1): c_error = pvc.int32_to_float(state.Error[0][i]) @@ -114,11 +112,18 @@ def test_frame_compare(pre_test_stuff): print('H_hat:') for i in range(0, 10, 1): c_H_hat = pvc.int32_to_float(state.H_hat[0][0][i]) - py_H_hat = icc.ic.H[0][int(i/2)].real - if (i / 2) == (int(i / 2)): + py_H_hat = 0 + if (i % 2) == 0: + py_H_hat = icc.ic.H[0][int(i/2)].real + else: py_H_hat = icc.ic.H[0][int(i/2)].imag print('C: ', c_H_hat, ', PY: ', py_H_hat) - #print(icc.error_ap[0][:10]) + print('sigma_xx:') + for i in range(0, 10, 1): + c_sigma_xx = pvc.int32_to_float(state.sigma_xx[0][i]) + print('C: ', c_sigma_xx, ', PY: ', icc.ic.sigma_xx[i]) + + #Write a copy of the output for post analysis if needed From 0c732054de5a2b68ced644eb9187930406209cfa Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 13 May 2022 15:27:05 +0100 Subject: [PATCH 68/92] typo --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 202bb2c2a..e3cfa0310 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -120,7 +120,7 @@ def test_frame_compare(pre_test_stuff): print('C: ', c_H_hat, ', PY: ', py_H_hat) print('sigma_xx:') for i in range(0, 10, 1): - c_sigma_xx = pvc.int32_to_float(state.sigma_xx[0][i]) + c_sigma_xx = pvc.int32_to_float(state.sigma_XX[0][i]) print('C: ', c_sigma_xx, ', PY: ', icc.ic.sigma_xx[i]) From 44096b4986fd2cf4d62228e5c55c5d316159be3b Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 13 May 2022 16:25:18 +0100 Subject: [PATCH 69/92] adding X_energy print and some changes to py_ic --- test/lib_ic/py_c_frame_compare/ic_test.c | 1 + test/lib_ic/py_c_frame_compare/test_ic_frame.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/test/lib_ic/py_c_frame_compare/ic_test.c b/test/lib_ic/py_c_frame_compare/ic_test.c index b4c30fe5a..9b37e83b5 100644 --- a/test/lib_ic/py_c_frame_compare/ic_test.c +++ b/test/lib_ic/py_c_frame_compare/ic_test.c @@ -19,6 +19,7 @@ ic_state_t test_get_state(void){ bfp_complex_s32_use_exponent(&ic_state.H_hat_bfp[0][v], -31); } bfp_s32_use_exponent(&ic_state.sigma_XX_bfp[0], -31); + bfp_s32_use_exponent(&ic_state.X_energy_bfp[0], -31); return ic_state; } diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index e3cfa0310..149a84e93 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -122,6 +122,10 @@ def test_frame_compare(pre_test_stuff): for i in range(0, 10, 1): c_sigma_xx = pvc.int32_to_float(state.sigma_XX[0][i]) print('C: ', c_sigma_xx, ', PY: ', icc.ic.sigma_xx[i]) + print('X_energy:') + for i in range(0, 10, 1): + c_X_energy = pvc.int32_to_float(state.X_energy[0][i]) + print('C: ', c_X_energy, ', PY: ', icc.ic.X_energy[i]) From 91aa099345d67525eb0685a322ccd7be5b1aab26 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 16 May 2022 15:49:05 +0100 Subject: [PATCH 70/92] adding test for X_energy --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 149a84e93..030ded395 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -123,9 +123,12 @@ def test_frame_compare(pre_test_stuff): c_sigma_xx = pvc.int32_to_float(state.sigma_XX[0][i]) print('C: ', c_sigma_xx, ', PY: ', icc.ic.sigma_xx[i]) print('X_energy:') - for i in range(0, 10, 1): + for i in range(0, (proc_frame_length * 2) + 1, 1): c_X_energy = pvc.int32_to_float(state.X_energy[0][i]) - print('C: ', c_X_energy, ', PY: ', icc.ic.X_energy[i]) + rtol = np.ldexp(1, -15) + if not np.isclose(c_X_energy, icc.ic.X_energy[i], rtol = rtol): + print('C: ', c_X_energy, ', PY: ', icc.ic.X_energy[i]) + print('TEST FAILED at i = ', i) From 1697f3fe525525542f83c24b16c9601802173f87 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 16 May 2022 16:39:37 +0100 Subject: [PATCH 71/92] fixing range typo --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 030ded395..243569b4b 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -123,7 +123,7 @@ def test_frame_compare(pre_test_stuff): c_sigma_xx = pvc.int32_to_float(state.sigma_XX[0][i]) print('C: ', c_sigma_xx, ', PY: ', icc.ic.sigma_xx[i]) print('X_energy:') - for i in range(0, (proc_frame_length * 2) + 1, 1): + for i in range(0, (proc_frame_length / 2) + 1, 1): c_X_energy = pvc.int32_to_float(state.X_energy[0][i]) rtol = np.ldexp(1, -15) if not np.isclose(c_X_energy, icc.ic.X_energy[i], rtol = rtol): From 69e09351680503bd0a582285e76c102ed6792ed1 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 16 May 2022 17:07:36 +0100 Subject: [PATCH 72/92] adding H_hat to the test --- .../py_c_frame_compare/test_ic_frame.py | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 243569b4b..550b9d1cf 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -24,6 +24,7 @@ proc_frame_length = 512 +fd_length = proc_frame_length // 2 + 1 frame_advance = 240 num_phases = 10 y_channel_delay = 180 @@ -104,32 +105,40 @@ def test_frame_compare(pre_test_stuff): if frame_start < 240: state = ic_test_lib.test_get_state() - - print('error_ap:') - for i in range(0, 10, 1): - c_error = pvc.int32_to_float(state.Error[0][i]) - print('C: ', c_error, ', PY: ', icc.error_ap[0][i]) + print('H_hat:') - for i in range(0, 10, 1): - c_H_hat = pvc.int32_to_float(state.H_hat[0][0][i]) - py_H_hat = 0 - if (i % 2) == 0: - py_H_hat = icc.ic.H[0][int(i/2)].real - else: - py_H_hat = icc.ic.H[0][int(i/2)].imag - print('C: ', c_H_hat, ', PY: ', py_H_hat) + for ph in range(num_phases): + for i in range(proc_frame_length + 2): + c_H_hat = pvc.int32_to_float(state.H_hat[0][ph][i]) + py_H_hat = 0 + if (i % 2) == 0: + py_H_hat = icc.ic.H[ph][int(i/2)].real + else: + py_H_hat = icc.ic.H[ph][int(i/2)].imag + rtol = np.ldexp(1, -15) + if not np.isclose(c_H_hat, py_H_hat, rtol = rtol): + print('C: ', c_H_hat, ', PY: ', py_H_hat) + print('TEST FAILED at i = ', i) + print('sigma_xx:') - for i in range(0, 10, 1): + for i in range(10): c_sigma_xx = pvc.int32_to_float(state.sigma_XX[0][i]) print('C: ', c_sigma_xx, ', PY: ', icc.ic.sigma_xx[i]) + print('X_energy:') - for i in range(0, (proc_frame_length / 2) + 1, 1): + for i in range(fd_length): c_X_energy = pvc.int32_to_float(state.X_energy[0][i]) + py_X_energy = icc.ic.X_energy[i] rtol = np.ldexp(1, -15) - if not np.isclose(c_X_energy, icc.ic.X_energy[i], rtol = rtol): - print('C: ', c_X_energy, ', PY: ', icc.ic.X_energy[i]) + if not np.isclose(c_X_energy, py_X_energy, rtol = rtol): + print('C: ', c_X_energy, ', PY: ', py_X_energy) print('TEST FAILED at i = ', i) + print('error_ap:') + for i in range(10): + c_error = pvc.int32_to_float(state.Error[0][i]) + print('C: ', c_error, ', PY: ', icc.error_ap[0][i]) + From b792fda6f80d5d3d76fdccb265add507521a0e74 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 16 May 2022 17:42:22 +0100 Subject: [PATCH 73/92] adding Y_hat check --- modules/lib_ic/api/ic_state.h | 2 +- modules/lib_ic/src/ic.c | 2 +- test/lib_ic/py_c_frame_compare/ic_test.c | 1 + .../py_c_frame_compare/test_ic_frame.py | 39 +++++++++++-------- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/modules/lib_ic/api/ic_state.h b/modules/lib_ic/api/ic_state.h index 2b61e3d98..16098c6d5 100644 --- a/modules/lib_ic/api/ic_state.h +++ b/modules/lib_ic/api/ic_state.h @@ -160,7 +160,7 @@ typedef struct { /** BFP array pointing to the estimated frequency domain Y signal. */ bfp_complex_s32_t Y_hat_bfp[IC_Y_CHANNELS]; /** Storage for Y_hat mantissas. */ - complex_s32_t DWORD_ALIGNED Y_hat[IC_Y_CHANNELS][IC_FD_FRAME_LENGTH]; + int32_t DWORD_ALIGNED Y_hat[IC_Y_CHANNELS][IC_FRAME_LENGTH + FFT_PADDING]; /** BFP array pointing to the frequency domain Error output. */ bfp_complex_s32_t Error_bfp[IC_Y_CHANNELS]; diff --git a/modules/lib_ic/src/ic.c b/modules/lib_ic/src/ic.c index c0ee8d7f5..530c839fa 100644 --- a/modules/lib_ic/src/ic.c +++ b/modules/lib_ic/src/ic.c @@ -74,7 +74,7 @@ void ic_init(ic_state_t *state){ } //Initiaise Y_hat for(unsigned ch=0; chY_hat_bfp[ch], state->Y_hat[ch], zero_exp, IC_FD_FRAME_LENGTH, 0); + bfp_complex_s32_init(&state->Y_hat_bfp[ch], (complex_s32_t * )state->Y_hat[ch], zero_exp, IC_FD_FRAME_LENGTH, 0); } //X_energy for(unsigned ch=0; ch Date: Tue, 17 May 2022 10:35:50 +0100 Subject: [PATCH 74/92] trying to access complex_s32_t from python --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index df87668a4..bce7c4b07 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -129,6 +129,9 @@ def test_frame_compare(pre_test_stuff): print('C: ', c_H_hat, ', PY: ', py_H_hat) print('TEST FAILED at ph = ', ph, ', i = ', i) + print('X_fifo test:') + print('re = ', pvc.int32_to_float(state.X_fifo[0][0][56].re), ', im = ', pvc.int32_to_float(state.X_fifo[0][0][56].im)) + print('Y_hat:') for i in range(10): c_Y_hat = pvc.int32_to_float(state.Y_hat[0][i]) From b17d32f3f9dd20201cc62aa69edc3084e1c31e3b Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 17 May 2022 11:00:48 +0100 Subject: [PATCH 75/92] changing back to complex_s32_t --- modules/lib_ic/api/ic_state.h | 4 ++-- modules/lib_ic/src/ic.c | 4 ++-- test/lib_ic/py_c_frame_compare/ic_test.c | 2 +- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 10 +++++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/modules/lib_ic/api/ic_state.h b/modules/lib_ic/api/ic_state.h index 16098c6d5..8dc176fef 100644 --- a/modules/lib_ic/api/ic_state.h +++ b/modules/lib_ic/api/ic_state.h @@ -160,7 +160,7 @@ typedef struct { /** BFP array pointing to the estimated frequency domain Y signal. */ bfp_complex_s32_t Y_hat_bfp[IC_Y_CHANNELS]; /** Storage for Y_hat mantissas. */ - int32_t DWORD_ALIGNED Y_hat[IC_Y_CHANNELS][IC_FRAME_LENGTH + FFT_PADDING]; + complex_s32_t DWORD_ALIGNED Y_hat[IC_Y_CHANNELS][IC_FRAME_LENGTH + FFT_PADDING]; /** BFP array pointing to the frequency domain Error output. */ bfp_complex_s32_t Error_bfp[IC_Y_CHANNELS]; @@ -173,7 +173,7 @@ typedef struct { /** BFP array pointing to the frequency domain estimate of transfer function. */ bfp_complex_s32_t H_hat_bfp[IC_Y_CHANNELS][IC_X_CHANNELS*IC_FILTER_PHASES]; /** Storage for H_hat mantissas. */ - int32_t DWORD_ALIGNED H_hat[IC_Y_CHANNELS][IC_FILTER_PHASES*IC_X_CHANNELS][IC_FRAME_LENGTH + FFT_PADDING]; + complex_s32_t DWORD_ALIGNED H_hat[IC_Y_CHANNELS][IC_FILTER_PHASES*IC_X_CHANNELS][IC_FD_FRAME_LENGTH]; /** BFP array pointing to the frequency domain X input history used for calculating normalisation. */ bfp_complex_s32_t X_fifo_bfp[IC_X_CHANNELS][IC_FILTER_PHASES]; diff --git a/modules/lib_ic/src/ic.c b/modules/lib_ic/src/ic.c index 530c839fa..6972b4925 100644 --- a/modules/lib_ic/src/ic.c +++ b/modules/lib_ic/src/ic.c @@ -57,7 +57,7 @@ void ic_init(ic_state_t *state){ //H_hat for(unsigned ch=0; chH_hat_bfp[ch][ph], (complex_s32_t *)state->H_hat[ch][ph], zero_exp, IC_FD_FRAME_LENGTH, 0); + bfp_complex_s32_init(&state->H_hat_bfp[ch][ph], state->H_hat[ch][ph], zero_exp, IC_FD_FRAME_LENGTH, 0); } } //X_fifo @@ -74,7 +74,7 @@ void ic_init(ic_state_t *state){ } //Initiaise Y_hat for(unsigned ch=0; chY_hat_bfp[ch], (complex_s32_t * )state->Y_hat[ch], zero_exp, IC_FD_FRAME_LENGTH, 0); + bfp_complex_s32_init(&state->Y_hat_bfp[ch], state->Y_hat[ch], zero_exp, IC_FD_FRAME_LENGTH, 0); } //X_energy for(unsigned ch=0; ch Date: Tue, 17 May 2022 13:32:09 +0100 Subject: [PATCH 76/92] fixing indexing --- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 830844b96..4bff460f8 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -121,10 +121,10 @@ def test_frame_compare(pre_test_stuff): c_H_hat = 0 py_H_hat = 0 if (i % 2) == 0: - c_H_hat = pvc.int32_to_float(state.H_hat[0][ph][i].re) + c_H_hat = pvc.int32_to_float(state.H_hat[0][ph][i // 2].re) py_H_hat = icc.ic.H[ph][i // 2].real else: - c_H_hat = pvc.int32_to_float(state.H_hat[0][ph][i].im) + c_H_hat = pvc.int32_to_float(state.H_hat[0][ph][i // 2].im) py_H_hat = icc.ic.H[ph][i // 2].imag rtol = np.ldexp(1, -13) if not np.isclose(c_H_hat, py_H_hat, rtol = rtol): @@ -139,10 +139,10 @@ def test_frame_compare(pre_test_stuff): c_Y_hat = 0 py_Y_hat = 0 if (i % 2) == 0: - c_Y_hat = pvc.int32_to_float(state.Y_hat[0][i].re) + c_Y_hat = pvc.int32_to_float(state.Y_hat[0][i // 2].re) py_Y_hat = icc.ic.Y_hat[0][i // 2].real else: - c_Y_hat = pvc.int32_to_float(state.Y_hat[0][i].im) + c_Y_hat = pvc.int32_to_float(state.Y_hat[0][i // 2].im) py_Y_hat = icc.ic.Y_hat[0][i // 2].imag print('C: ', c_Y_hat, ', PY: ', py_Y_hat) From 56c336dd83a891b5a914908a605a04de67408c01 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 17 May 2022 14:12:56 +0100 Subject: [PATCH 77/92] disabling jenkins --- Jenkinsfile | 17 ++++++++++------- modules/lib_ic/api/ic_state.h | 2 +- test/lib_ic/py_c_frame_compare/test_ic_frame.py | 7 ++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 75a9147f4..02a0ad6f9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,7 +56,8 @@ pipeline { withVenv { sh "cmake --version" sh 'cmake -S.. -DPython3_FIND_VIRTUALENV="ONLY" -DTEST_WAV_ADEC_BUILD_CONFIG="1 2 2 10 5" -DAVONA_BUILD_TESTS=ON' - sh "make -j8" + //sh "make -j8" + sh "make -C test/lib_ic/" } } } @@ -77,11 +78,11 @@ pipeline { } } dir("${REPO}") { - stash name: 'cmake_build_x86_examples', includes: 'build/**/avona_example_bare_metal_*' + //stash name: 'cmake_build_x86_examples', includes: 'build/**/avona_example_bare_metal_*' //We are archveing the x86 version. Be careful - these have the same file name as the xcore versions but the linker should warn at least in this case - stash name: 'cmake_build_x86_libs', includes: 'build/**/*.a' - archiveArtifacts artifacts: "build/**/avona_example_bare_metal_*", fingerprint: true - stash name: 'vnr_py_c_feature_compare', includes: 'test/lib_vnr/py_c_feature_compare/build/**' + //stash name: 'cmake_build_x86_libs', includes: 'build/**/*.a' + //archiveArtifacts artifacts: "build/**/avona_example_bare_metal_*", fingerprint: true + //stash name: 'vnr_py_c_feature_compare', includes: 'test/lib_vnr/py_c_feature_compare/build/**' stash name: 'py_c_frame_compare', includes: 'test/lib_ic/py_c_frame_compare/build/**' } // Now do xcore files @@ -97,7 +98,8 @@ pipeline { sh 'cmake -S.. -DCMAKE_TOOLCHAIN_FILE=../xmos_cmake_toolchain/xs3a.cmake -DPython3_VIRTUALENV_FIND="ONLY" -DTEST_SPEEDUP_FACTOR=4 -DAVONA_BUILD_TESTS=ON' } } - sh "make -j8" + //sh "make -j8" + sh "make -C test/lib_ic/" } } } @@ -147,7 +149,8 @@ pipeline { sh "cmake --version" sh 'cmake -S.. -DPython3_FIND_VIRTUALENV="ONLY" -DTEST_WAV_ADEC_BUILD_CONFIG="1 2 2 10 5" -DAVONA_BUILD_TESTS=ON' // sh "make -j8" - sh "make VERBOSE=1" + //sh "make VERBOSE=1" + sh "make -C test/lib_ic/" //We need to put this here because it is not fetched until we build sh "pip install -e avona_deps/xscope_fileio" diff --git a/modules/lib_ic/api/ic_state.h b/modules/lib_ic/api/ic_state.h index 8dc176fef..ed6d0cf02 100644 --- a/modules/lib_ic/api/ic_state.h +++ b/modules/lib_ic/api/ic_state.h @@ -160,7 +160,7 @@ typedef struct { /** BFP array pointing to the estimated frequency domain Y signal. */ bfp_complex_s32_t Y_hat_bfp[IC_Y_CHANNELS]; /** Storage for Y_hat mantissas. */ - complex_s32_t DWORD_ALIGNED Y_hat[IC_Y_CHANNELS][IC_FRAME_LENGTH + FFT_PADDING]; + complex_s32_t DWORD_ALIGNED Y_hat[IC_Y_CHANNELS][IC_FD_FRAME_LENGTH]; /** BFP array pointing to the frequency domain Error output. */ bfp_complex_s32_t Error_bfp[IC_Y_CHANNELS]; diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 4bff460f8..f2c8f8bdd 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -126,16 +126,13 @@ def test_frame_compare(pre_test_stuff): else: c_H_hat = pvc.int32_to_float(state.H_hat[0][ph][i // 2].im) py_H_hat = icc.ic.H[ph][i // 2].imag - rtol = np.ldexp(1, -13) + rtol = np.ldexp(1, -12) if not np.isclose(c_H_hat, py_H_hat, rtol = rtol): print('C: ', c_H_hat, ', PY: ', py_H_hat) print('TEST FAILED at ph = ', ph, ', i = ', i) - - print('X_fifo test:') - print('re = ', pvc.int32_to_float(state.X_fifo[0][0][56].re), ', im = ', pvc.int32_to_float(state.X_fifo[0][0][56].im)) print('Y_hat:') - for i in range(10): + for i in range(proc_frame_length + 2): c_Y_hat = 0 py_Y_hat = 0 if (i % 2) == 0: From e79bc0f2f31147f00ce5616deadb0a525f844f57 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 17 May 2022 14:23:01 +0100 Subject: [PATCH 78/92] fixing jenkins --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 02a0ad6f9..a1a2385ce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,7 +73,7 @@ pipeline { dir("${REPO}/test/lib_vnr/py_c_feature_compare") { viewEnv() { withVenv { - runPython("python build_vnr_feature_extraction.py") + //runPython("python build_vnr_feature_extraction.py") } } } From 4cdfebaa31a7883a4e340157d4c60f8497b54cff Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 23 May 2022 08:49:38 +0100 Subject: [PATCH 79/92] saving everything for later --- modules/lib_aec/src/aec_impl.c | 1 + modules/lib_aec/src/aec_priv_impl.c | 27 +++- modules/lib_ic/src/ic.c | 1 + test/lib_ic/py_c_frame_compare/ic_test.c | 64 +++++++- .../py_c_frame_compare/test_ic_frame.py | 153 ++++++++++++------ 5 files changed, 185 insertions(+), 61 deletions(-) diff --git a/modules/lib_aec/src/aec_impl.c b/modules/lib_aec/src/aec_impl.c index f7bc9774a..d0ff8c2e7 100644 --- a/modules/lib_aec/src/aec_impl.c +++ b/modules/lib_aec/src/aec_impl.c @@ -140,6 +140,7 @@ void aec_forward_fft( bfp_complex_s32_t *temp = bfp_fft_forward_mono(input); memcpy(output, temp, sizeof(bfp_complex_s32_t)); + output->hr = bfp_complex_s32_headroom(output); bfp_fft_unpack_mono(output); input->length = len; } diff --git a/modules/lib_aec/src/aec_priv_impl.c b/modules/lib_aec/src/aec_priv_impl.c index f55837250..add49430c 100644 --- a/modules/lib_aec/src/aec_priv_impl.c +++ b/modules/lib_aec/src/aec_priv_impl.c @@ -808,6 +808,24 @@ void aec_priv_calc_inverse( #if 1 //82204 cycles. 2 x-channels, single thread, but get rids of voice_toolbox dependency on vtb_inv_X_energy_asm (36323 cycles) bfp_s32_inverse(input, input); + /*bfp_s32_t chunk[4]; //257 + bfp_s32_init(&chunk[0], input->data, input->exp, 64, 1); + bfp_s32_init(&chunk[1], &input->data[64], input->exp, 64, 1); + bfp_s32_init(&chunk[2], &input->data[128], input->exp, 64, 1); + bfp_s32_init(&chunk[3], &input->data[192], input->exp, 65, 1); + bfp_s32_inverse(&chunk[0], &chunk[0]); + bfp_s32_inverse(&chunk[1], &chunk[1]); + bfp_s32_inverse(&chunk[2], &chunk[2]); + bfp_s32_inverse(&chunk[3], &chunk[3]); + exponent_t exp = (chunk[0].exp > chunk[1].exp) ? chunk[0].exp: chunk[1].exp; + exp = (exp > chunk[2].exp) ? exp: chunk[2].exp; + exp = (exp > chunk[3].exp) ? exp: chunk[3].exp; + bfp_s32_use_exponent(&chunk[0], exp); + bfp_s32_use_exponent(&chunk[1], exp); + bfp_s32_use_exponent(&chunk[2], exp); + bfp_s32_use_exponent(&chunk[3], exp); + input->exp = exp; + input->hr = bfp_s32_headroom(input);*/ #else //36323 cycles. 2 x-channels, single thread int32_t min_element = xs3_vect_s32_min( @@ -858,7 +876,12 @@ void aec_priv_calc_inv_X_energy_denom( } bfp_s32_convolve_same(inv_X_energy_denom, &norm_denom, &taps_q30[0], 5, PAD_MODE_REFLECT); - + float_s32_t min = bfp_s32_min(inv_X_energy_denom); + float_s32_t max = bfp_s32_max(inv_X_energy_denom); + headroom_t hd = inv_X_energy_denom->hr; + headroom_t act = bfp_s32_headroom(inv_X_energy_denom); + //inv_X_energy_denom->hr = (hd == act)? hd : act; + //printf(" %d %d ||",max.mant, max.exp); bfp_s32_add_scalar(inv_X_energy_denom, inv_X_energy_denom, delta); } else @@ -890,7 +913,9 @@ void aec_priv_calc_inv_X_energy_denom( //Option 2 (1528 cycles for the bfp_s32_min() call. Haven't profiled when min.mant == 0 is true float_s32_t min = bfp_s32_min(inv_X_energy_denom); + //printf(" %d %d |||",min.mant, min.exp); if(min.mant == 0) { + //printf("divide by zero occured!\n"); /** The presence of delta even when it's zero in bfp_s32_add_scalar(inv_X_energy_denom, X_energy, delta); above * ensures that bfp_s32_max(inv_X_energy_denom) always has a headroom of 1, making sure that t is not right shifted as part * of bfp_s32_add_scalar() making t.mant 0*/ diff --git a/modules/lib_ic/src/ic.c b/modules/lib_ic/src/ic.c index 6972b4925..3b25ede27 100644 --- a/modules/lib_ic/src/ic.c +++ b/modules/lib_ic/src/ic.c @@ -18,6 +18,7 @@ static void ic_init_config(ic_config_params_t *config){ config->ema_alpha_q30 = Q1_30(IC_INIT_EMA_ALPHA); config->bypass = 0; config->delta = double_to_float_s32(IC_INIT_DELTA); + //config->delta = {1073741823, -57}; } diff --git a/test/lib_ic/py_c_frame_compare/ic_test.c b/test/lib_ic/py_c_frame_compare/ic_test.c index 51d74a0d4..e01de9499 100644 --- a/test/lib_ic/py_c_frame_compare/ic_test.c +++ b/test/lib_ic/py_c_frame_compare/ic_test.c @@ -3,9 +3,58 @@ ic_state_t ic_state; +static inline int32_t ashr32(int32_t x, right_shift_t shr) +{ + printf("yo"); + if(shr >= 0){ + printf("ooooooo\n"); + return x >> shr;} + printf("ba\n"); + int64_t tmp = ((int64_t)x) << -shr; + + if(tmp > INT32_MAX) return INT32_MAX; + else if(tmp < INT32_MIN) return INT32_MIN; + else return tmp; +} + +float_s32_t add( + const float_s32_t x, + const float_s32_t y) +{ + float_s32_t res; + + const headroom_t x_hr = HR_S32(x.mant); + const headroom_t y_hr = HR_S32(y.mant); + printf("hr1 = %d, hr2 = %d\n", x_hr, y_hr); + const exponent_t x_min_exp = x.exp - x_hr; + const exponent_t y_min_exp = y.exp - y_hr; + printf("exp1 = %d, exp2 = %d\n", x_min_exp, y_min_exp); + res.exp = MAX(x_min_exp, y_min_exp) + 1; + printf("nex_exp = %d\n", res.exp); + const right_shift_t x_shr = res.exp - x.exp; + right_shift_t y_shr = res.exp - y.exp; + y_shr = 32; + printf("shr1 = %d, shr2 = %d\n", x_shr, y_shr); + int32_t x_int = ashr32(x.mant, x_shr); + int32_t y_int = ashr32(y.mant, y_shr); + printf("%d + %d\n", x_int, y_int); + res.mant = x_int + y_int; + printf("\n"); + return res; +} void test_init(void){ ic_init(&ic_state); + float_s32_t test1 = {277038593, -25}; + float_s32_t test2 = {233768088, -24}; + float_s32_t delta = {2147483646, -58}; + test1 = add(test1, delta);bfp_s32_add_scalar() + test2 = add(test2, delta); + printf("\ntest1 %d %d\n", test1.mant, test1.exp); + printf("test2 %d %d\n", test2.mant, test2.exp); + + int32_t a = 0x7fffffff; + printf("%d, %d\n", a, a>>32); //Custom setup for testing // ic_state.mu[0][0] = double_to_float_s32(0.0); ic_state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = 0; @@ -14,13 +63,14 @@ void test_init(void){ } ic_state_t test_get_state(void){ - bfp_s32_use_exponent(&ic_state.error_bfp[0], -31); - for(int v = 0; v < IC_FILTER_PHASES; v++){ - bfp_complex_s32_use_exponent(&ic_state.H_hat_bfp[0][v], -31); - } - bfp_s32_use_exponent(&ic_state.sigma_XX_bfp[0], -31); - bfp_s32_use_exponent(&ic_state.X_energy_bfp[0], -31); - bfp_complex_s32_use_exponent(&ic_state.Y_hat_bfp[0], -31); + //bfp_s32_use_exponent(&ic_state.error_bfp[0], -31); + //for(int v = 0; v < IC_FILTER_PHASES; v++){ + //bfp_complex_s32_use_exponent(&ic_state.H_hat_bfp[0][v], -31); + //} + //bfp_s32_use_exponent(&ic_state.sigma_XX_bfp[0], -31); + //bfp_s32_use_exponent(&ic_state.X_energy_bfp[0], -31); + //bfp_s32_use_exponent(&ic_state.inv_X_energy_bfp[0], -31); + //bfp_complex_s32_use_exponent(&ic_state.Y_hat_bfp[0], -31); return ic_state; } diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index f2c8f8bdd..68186c980 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -28,7 +28,8 @@ frame_advance = 240 num_phases = 10 y_channel_delay = 180 -input_file = "../../../examples/bare-metal/ic/input.wav" +input_file = 'pytest_audio_0.wav' +#input_file = "../../../examples/bare-metal/ic/input.wav" output_file = "output.wav" @pytest.fixture(params=[34]) @@ -50,20 +51,16 @@ def __init__(self): vnr_model = '../../../../py_vnr/model_output_0_0_2/trained_model.h5', adaption_config = 'IC_ADAPTION_FORCE_ON' ) - self.error_ap = None ic_test_lib.test_init() - def process_frame(self, frame): #we need to delay the y for python as not done in model #first copy the input data for C ver before we modfiy it frame_int = pvc.float_to_int32(frame) - - output_py, Error_ap = self.ic.process_frame(frame) - self.error_ap = np.fft.irfft(Error_ap) + output_py, self.Error_ap = self.ic.process_frame(frame) mu, control_flag = self.ic.mu_control_system() #print('mu = ', mu, ', in_vnr = ', input_vnr_pred, ', out_vnr = ', output_vnr_pred, 'flag = ', control_flag) - self.ic.adapt(Error_ap) + self.ic.adapt(self.Error_ap) #Grab a pointer to the data storage of the numpy arrays y_data = ffi.cast("int32_t *", ffi.from_buffer(frame_int[0].data)) @@ -82,49 +79,68 @@ def process_frame(self, frame): # print(pvc.float_s32_to_float(state.config_params.delta)) return output_py, pvc.int32_to_float(output_c) - -def test_frame_compare(pre_test_stuff): - icc = ic_comparison() - - input_rate, input_wav_file = scipy.io.wavfile.read(input_file, 'r') - input_wav_data, input_channel_count, file_length = awu.parse_audio(input_wav_file) - delays = np.zeros(input_channel_count) #we do delay of y channel in process_frame above and in C rather than awu.get_frame - - output_wav_data = np.zeros((2, file_length)) - - for frame_start in range(0, file_length-proc_frame_length*2, frame_advance): - input_frame = awu.get_frame(input_wav_data, frame_start, frame_advance, delays)[0:2,:] - - if False: - print ('# ' + str(frame_start // frame_advance)) - - output_py, output_c = icc.process_frame(input_frame) - - output_wav_data[0, frame_start: frame_start + frame_advance] = output_py - output_wav_data[1, frame_start: frame_start + frame_advance] = output_c - - if frame_start < 240: - state = ic_test_lib.test_get_state() +def check_filter_components(icc, frame_start): + state = ic_test_lib.test_get_state() + + if frame_start < 1200: + print('frame = ', frame_start // 240) + print('Y:') + exp = state.Y_bfp[0].exp + for i in range(proc_frame_length + 2): + c_Y = np.array(state.y[0][i]).astype(np.float64) * (2 ** exp) + py_Y = 0 + if (i % 2) == 0: + py_Y = icc.ic.Y_data[0][i // 2].real + else: + py_Y = icc.ic.Y_data[0][i // 2].imag + rtol = np.ldexp(1, -13) + if not np.isclose(c_Y, py_Y, rtol = rtol): + print('C: ', c_Y, ', PY: ', py_Y) + print('TEST FAILED at i = ', i) - print('X_energy:') - for i in range(fd_length): - c_X_energy = pvc.int32_to_float(state.X_energy[0][i]) - py_X_energy = icc.ic.X_energy[i] - rtol = np.ldexp(1, -15) - if not np.isclose(c_X_energy, py_X_energy, rtol = rtol): - print('C: ', c_X_energy, ', PY: ', py_X_energy) - print('TEST FAILED at i = ', i) + print('X_energy:') + exp = state.X_energy_bfp[0].exp + for i in range(fd_length): + c_X_energy = np.array(state.X_energy[0][i]).astype(np.float64) * (2 ** exp) + py_X_energy = icc.ic.X_energy[i] + rtol = np.ldexp(1, -20) + #if not np.isclose(c_X_energy, py_X_energy, rtol = rtol): + #print('C: ', c_X_energy, ', PY: ', py_X_energy) + #print('TEST FAILED at i = ', i) + + print('Inverse X energy:') + exp = state.inv_X_energy_bfp[0].exp + print('exp = ', exp) + for i in range(fd_length): + c_inv_X_energy = np.array(state.inv_X_energy[0][i]).astype(np.float64) * (2 ** exp) + py_inv_X_energy = icc.ic.inv_X_energy[0][i] + rtol = np.ldexp(1, -10) + #if not np.isclose(c_inv_X_energy, py_inv_X_energy, rtol = rtol): + #print('C: ', c_inv_X_energy, ', PY: ', py_inv_X_energy) + #print('TEST FAILED at i = ', i) + + print('sigma_xx:') + exp = state.sigma_XX_bfp[0].exp + for i in range(fd_length): + c_sigma_xx = np.array(state.sigma_XX[0][i]).astype(np.float64) * (2 ** exp) + py_sigma_xx = icc.ic.sigma_xx[i] + rtol = np.ldexp(1, -19) + if not np.isclose(c_sigma_xx, py_sigma_xx, rtol = rtol): + print('C: ', c_sigma_xx, ', PY: ', py_sigma_xx) + print('TEST FAILED at i = ', i) + if frame_start < 240: print('H_hat:') for ph in range(num_phases): + exp = state.H_hat_bfp[0][ph].exp for i in range(proc_frame_length + 2): c_H_hat = 0 py_H_hat = 0 if (i % 2) == 0: - c_H_hat = pvc.int32_to_float(state.H_hat[0][ph][i // 2].re) + c_H_hat = np.array(state.H_hat[0][ph][i // 2].re).astype(np.float64) * (2 ** exp) py_H_hat = icc.ic.H[ph][i // 2].real else: - c_H_hat = pvc.int32_to_float(state.H_hat[0][ph][i // 2].im) + c_H_hat = np.array(state.H_hat[0][ph][i // 2].im).astype(np.float64) * (2 ** exp) py_H_hat = icc.ic.H[ph][i // 2].imag rtol = np.ldexp(1, -12) if not np.isclose(c_H_hat, py_H_hat, rtol = rtol): @@ -132,31 +148,62 @@ def test_frame_compare(pre_test_stuff): print('TEST FAILED at ph = ', ph, ', i = ', i) print('Y_hat:') + exp = state.Y_hat_bfp[0].exp for i in range(proc_frame_length + 2): c_Y_hat = 0 py_Y_hat = 0 if (i % 2) == 0: - c_Y_hat = pvc.int32_to_float(state.Y_hat[0][i // 2].re) + c_Y_hat = np.array(state.Y_hat[0][i // 2].re).astype(np.float64) * (2 ** exp) py_Y_hat = icc.ic.Y_hat[0][i // 2].real else: - c_Y_hat = pvc.int32_to_float(state.Y_hat[0][i // 2].im) + c_Y_hat = np.array(state.Y_hat[0][i // 2].im).astype(np.float64) * (2 ** exp) py_Y_hat = icc.ic.Y_hat[0][i // 2].imag - print('C: ', c_Y_hat, ', PY: ', py_Y_hat) - - print('sigma_xx:') - for i in range(10): - c_sigma_xx = pvc.int32_to_float(state.sigma_XX[0][i]) - print('C: ', c_sigma_xx, ', PY: ', icc.ic.sigma_xx[i]) + rtol = np.ldexp(1, -15) + if not np.isclose(c_Y_hat, py_Y_hat, rtol = rtol): + print('C: ', c_Y_hat, ', PY: ', py_Y_hat) + print('TEST FAILED at i = ', i) print('error_ap:') - for i in range(10): - c_error = pvc.int32_to_float(state.Error[0][i]) - print('C: ', c_error, ', PY: ', icc.error_ap[0][i]) + exp = state.error_bfp[0].exp + for i in range(proc_frame_length + 2): + c_error = np.array(state.Error[0][i]).astype(np.float64) * (2 ** exp) + py_error = 0 + if (i % 2) == 0: + py_error = icc.Error_ap[0][i // 2].real + else: + py_error = icc.Error_ap[0][i // 2].imag + rtol = np.ldexp(1, -23) + if not np.isclose(c_error, py_error, rtol = rtol): + print('C: ', c_error, ', PY: ', py_error) + #print('TEST FAILED at i = ', i) + +def test_frame_compare(pre_test_stuff): + icc = ic_comparison() + + input_rate, input_wav_file = scipy.io.wavfile.read(input_file, 'r') + input_wav_data, input_channel_count, file_length = awu.parse_audio(input_wav_file) + delays = np.zeros(input_channel_count) #we do delay of y channel in process_frame above and in C rather than awu.get_frame + + output_wav_data = np.zeros((2, file_length)) + + #for frame_start in range(0, file_length-proc_frame_length*2, frame_advance): + for frame_start in range(0, 2400, frame_advance): + input_frame = awu.get_frame(input_wav_data, frame_start, frame_advance, delays)[0:2,:] + + if False: + print ('# ' + str(frame_start // frame_advance)) + + output_py, output_c = icc.process_frame(input_frame) + + output_wav_data[0, frame_start: frame_start + frame_advance] = output_py + output_wav_data[1, frame_start: frame_start + frame_advance] = output_c + + check_filter_components(icc, frame_start) #Write a copy of the output for post analysis if needed scipy.io.wavfile.write(output_file, input_rate, pvc.float_to_int32(output_wav_data.T)) arith_closeness, geo_closeness, c_delay, peak2ave = pvc.pcm_closeness_metric(output_file) - assert arith_closeness > 0.98 - assert geo_closeness > 0.99 + #assert arith_closeness > 0.98 + #assert geo_closeness > 0.99 assert c_delay == 0 From ba8cd78902063862242cff3ac48a202f28da99cb Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 24 May 2022 11:15:35 +0100 Subject: [PATCH 80/92] addidng a workaround bfp function --- modules/lib_aec/src/aec_priv_impl.c | 36 ++++++++++++++++--- test/lib_ic/py_c_frame_compare/ic_test.c | 4 +-- .../py_c_frame_compare/test_ic_frame.py | 14 ++++---- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/modules/lib_aec/src/aec_priv_impl.c b/modules/lib_aec/src/aec_priv_impl.c index add49430c..dba057845 100644 --- a/modules/lib_aec/src/aec_priv_impl.c +++ b/modules/lib_aec/src/aec_priv_impl.c @@ -843,6 +843,30 @@ void aec_priv_calc_inverse( #endif } + +void bfp_new_add_scalar( + bfp_s32_t* a, + const bfp_s32_t* b, + const float_s32_t c) +{ +#if (XS3_BFP_DEBUG_CHECK_LENGTHS) // See xs3_math_conf.h + assert(b->length == a->length); + assert(b->length != 0); +#endif + + right_shift_t b_shr, c_shr; + + xs3_vect_s32_add_scalar_prepare(&a->exp, &b_shr, &c_shr, b->exp, c.exp, + b->hr, HR_S32(c.mant)); + + int32_t cc = 0; + if (c_shr < 32) + cc = (c_shr >= 0)? (c.mant >> c_shr) : (c.mant << -c_shr); + + a->hr = xs3_vect_s32_add_scalar(a->data, b->data, cc, b->length, + b_shr); +} + void aec_priv_calc_inv_X_energy_denom( bfp_s32_t *inv_X_energy_denom, const bfp_s32_t *X_energy, @@ -876,13 +900,15 @@ void aec_priv_calc_inv_X_energy_denom( } bfp_s32_convolve_same(inv_X_energy_denom, &norm_denom, &taps_q30[0], 5, PAD_MODE_REFLECT); - float_s32_t min = bfp_s32_min(inv_X_energy_denom); - float_s32_t max = bfp_s32_max(inv_X_energy_denom); - headroom_t hd = inv_X_energy_denom->hr; - headroom_t act = bfp_s32_headroom(inv_X_energy_denom); + //float_s32_t min = bfp_s32_min(inv_X_energy_denom); + //float_s32_t max = bfp_s32_max(inv_X_energy_denom); + //headroom_t hd = inv_X_energy_denom->hr; + //headroom_t act = bfp_s32_headroom(inv_X_energy_denom); //inv_X_energy_denom->hr = (hd == act)? hd : act; //printf(" %d %d ||",max.mant, max.exp); - bfp_s32_add_scalar(inv_X_energy_denom, inv_X_energy_denom, delta); + //bfp_s32_add_scalar(inv_X_energy_denom, inv_X_energy_denom, delta); + bfp_new_add_scalar(inv_X_energy_denom, inv_X_energy_denom, delta); + } else { diff --git a/test/lib_ic/py_c_frame_compare/ic_test.c b/test/lib_ic/py_c_frame_compare/ic_test.c index e01de9499..ba8e03a78 100644 --- a/test/lib_ic/py_c_frame_compare/ic_test.c +++ b/test/lib_ic/py_c_frame_compare/ic_test.c @@ -45,7 +45,7 @@ float_s32_t add( void test_init(void){ ic_init(&ic_state); - float_s32_t test1 = {277038593, -25}; + /*float_s32_t test1 = {277038593, -25}; float_s32_t test2 = {233768088, -24}; float_s32_t delta = {2147483646, -58}; test1 = add(test1, delta);bfp_s32_add_scalar() @@ -54,7 +54,7 @@ void test_init(void){ printf("test2 %d %d\n", test2.mant, test2.exp); int32_t a = 0x7fffffff; - printf("%d, %d\n", a, a>>32); + printf("%d, %d\n", a, a>>32);*/ //Custom setup for testing // ic_state.mu[0][0] = double_to_float_s32(0.0); ic_state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = 0; diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 68186c980..a7586489b 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -104,9 +104,9 @@ def check_filter_components(icc, frame_start): c_X_energy = np.array(state.X_energy[0][i]).astype(np.float64) * (2 ** exp) py_X_energy = icc.ic.X_energy[i] rtol = np.ldexp(1, -20) - #if not np.isclose(c_X_energy, py_X_energy, rtol = rtol): - #print('C: ', c_X_energy, ', PY: ', py_X_energy) - #print('TEST FAILED at i = ', i) + if not np.isclose(c_X_energy, py_X_energy, rtol = rtol): + print('C: ', c_X_energy, ', PY: ', py_X_energy) + print('TEST FAILED at i = ', i) print('Inverse X energy:') exp = state.inv_X_energy_bfp[0].exp @@ -115,9 +115,9 @@ def check_filter_components(icc, frame_start): c_inv_X_energy = np.array(state.inv_X_energy[0][i]).astype(np.float64) * (2 ** exp) py_inv_X_energy = icc.ic.inv_X_energy[0][i] rtol = np.ldexp(1, -10) - #if not np.isclose(c_inv_X_energy, py_inv_X_energy, rtol = rtol): - #print('C: ', c_inv_X_energy, ', PY: ', py_inv_X_energy) - #print('TEST FAILED at i = ', i) + if not np.isclose(c_inv_X_energy, py_inv_X_energy, rtol = rtol): + print('C: ', c_inv_X_energy, ', PY: ', py_inv_X_energy) + print('TEST FAILED at i = ', i) print('sigma_xx:') exp = state.sigma_XX_bfp[0].exp @@ -175,7 +175,7 @@ def check_filter_components(icc, frame_start): rtol = np.ldexp(1, -23) if not np.isclose(c_error, py_error, rtol = rtol): print('C: ', c_error, ', PY: ', py_error) - #print('TEST FAILED at i = ', i) + print('TEST FAILED at i = ', i) def test_frame_compare(pre_test_stuff): icc = ic_comparison() From f11e9d9959fffd03a5955ad4376b22ee8966242c Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Tue, 24 May 2022 11:32:12 +0100 Subject: [PATCH 81/92] cleanup --- modules/lib_aec/src/aec_priv_impl.c | 28 +-------- modules/lib_ic/src/ic.c | 1 - test/lib_ic/py_c_frame_compare/ic_test.c | 59 +------------------ .../py_c_frame_compare/test_ic_frame.py | 11 ++-- 4 files changed, 8 insertions(+), 91 deletions(-) diff --git a/modules/lib_aec/src/aec_priv_impl.c b/modules/lib_aec/src/aec_priv_impl.c index dba057845..9f37a9d9a 100644 --- a/modules/lib_aec/src/aec_priv_impl.c +++ b/modules/lib_aec/src/aec_priv_impl.c @@ -808,24 +808,6 @@ void aec_priv_calc_inverse( #if 1 //82204 cycles. 2 x-channels, single thread, but get rids of voice_toolbox dependency on vtb_inv_X_energy_asm (36323 cycles) bfp_s32_inverse(input, input); - /*bfp_s32_t chunk[4]; //257 - bfp_s32_init(&chunk[0], input->data, input->exp, 64, 1); - bfp_s32_init(&chunk[1], &input->data[64], input->exp, 64, 1); - bfp_s32_init(&chunk[2], &input->data[128], input->exp, 64, 1); - bfp_s32_init(&chunk[3], &input->data[192], input->exp, 65, 1); - bfp_s32_inverse(&chunk[0], &chunk[0]); - bfp_s32_inverse(&chunk[1], &chunk[1]); - bfp_s32_inverse(&chunk[2], &chunk[2]); - bfp_s32_inverse(&chunk[3], &chunk[3]); - exponent_t exp = (chunk[0].exp > chunk[1].exp) ? chunk[0].exp: chunk[1].exp; - exp = (exp > chunk[2].exp) ? exp: chunk[2].exp; - exp = (exp > chunk[3].exp) ? exp: chunk[3].exp; - bfp_s32_use_exponent(&chunk[0], exp); - bfp_s32_use_exponent(&chunk[1], exp); - bfp_s32_use_exponent(&chunk[2], exp); - bfp_s32_use_exponent(&chunk[3], exp); - input->exp = exp; - input->hr = bfp_s32_headroom(input);*/ #else //36323 cycles. 2 x-channels, single thread int32_t min_element = xs3_vect_s32_min( @@ -900,12 +882,7 @@ void aec_priv_calc_inv_X_energy_denom( } bfp_s32_convolve_same(inv_X_energy_denom, &norm_denom, &taps_q30[0], 5, PAD_MODE_REFLECT); - //float_s32_t min = bfp_s32_min(inv_X_energy_denom); - //float_s32_t max = bfp_s32_max(inv_X_energy_denom); - //headroom_t hd = inv_X_energy_denom->hr; - //headroom_t act = bfp_s32_headroom(inv_X_energy_denom); - //inv_X_energy_denom->hr = (hd == act)? hd : act; - //printf(" %d %d ||",max.mant, max.exp); + //bfp_s32_add_scalar(inv_X_energy_denom, inv_X_energy_denom, delta); bfp_new_add_scalar(inv_X_energy_denom, inv_X_energy_denom, delta); @@ -939,9 +916,8 @@ void aec_priv_calc_inv_X_energy_denom( //Option 2 (1528 cycles for the bfp_s32_min() call. Haven't profiled when min.mant == 0 is true float_s32_t min = bfp_s32_min(inv_X_energy_denom); - //printf(" %d %d |||",min.mant, min.exp); + if(min.mant == 0) { - //printf("divide by zero occured!\n"); /** The presence of delta even when it's zero in bfp_s32_add_scalar(inv_X_energy_denom, X_energy, delta); above * ensures that bfp_s32_max(inv_X_energy_denom) always has a headroom of 1, making sure that t is not right shifted as part * of bfp_s32_add_scalar() making t.mant 0*/ diff --git a/modules/lib_ic/src/ic.c b/modules/lib_ic/src/ic.c index 3b25ede27..6972b4925 100644 --- a/modules/lib_ic/src/ic.c +++ b/modules/lib_ic/src/ic.c @@ -18,7 +18,6 @@ static void ic_init_config(ic_config_params_t *config){ config->ema_alpha_q30 = Q1_30(IC_INIT_EMA_ALPHA); config->bypass = 0; config->delta = double_to_float_s32(IC_INIT_DELTA); - //config->delta = {1073741823, -57}; } diff --git a/test/lib_ic/py_c_frame_compare/ic_test.c b/test/lib_ic/py_c_frame_compare/ic_test.c index ba8e03a78..66734fb34 100644 --- a/test/lib_ic/py_c_frame_compare/ic_test.c +++ b/test/lib_ic/py_c_frame_compare/ic_test.c @@ -3,58 +3,8 @@ ic_state_t ic_state; -static inline int32_t ashr32(int32_t x, right_shift_t shr) -{ - printf("yo"); - if(shr >= 0){ - printf("ooooooo\n"); - return x >> shr;} - printf("ba\n"); - int64_t tmp = ((int64_t)x) << -shr; - - if(tmp > INT32_MAX) return INT32_MAX; - else if(tmp < INT32_MIN) return INT32_MIN; - else return tmp; -} - -float_s32_t add( - const float_s32_t x, - const float_s32_t y) -{ - float_s32_t res; - - const headroom_t x_hr = HR_S32(x.mant); - const headroom_t y_hr = HR_S32(y.mant); - printf("hr1 = %d, hr2 = %d\n", x_hr, y_hr); - const exponent_t x_min_exp = x.exp - x_hr; - const exponent_t y_min_exp = y.exp - y_hr; - printf("exp1 = %d, exp2 = %d\n", x_min_exp, y_min_exp); - res.exp = MAX(x_min_exp, y_min_exp) + 1; - printf("nex_exp = %d\n", res.exp); - const right_shift_t x_shr = res.exp - x.exp; - right_shift_t y_shr = res.exp - y.exp; - y_shr = 32; - printf("shr1 = %d, shr2 = %d\n", x_shr, y_shr); - int32_t x_int = ashr32(x.mant, x_shr); - int32_t y_int = ashr32(y.mant, y_shr); - printf("%d + %d\n", x_int, y_int); - res.mant = x_int + y_int; - printf("\n"); - return res; -} - void test_init(void){ ic_init(&ic_state); - /*float_s32_t test1 = {277038593, -25}; - float_s32_t test2 = {233768088, -24}; - float_s32_t delta = {2147483646, -58}; - test1 = add(test1, delta);bfp_s32_add_scalar() - test2 = add(test2, delta); - printf("\ntest1 %d %d\n", test1.mant, test1.exp); - printf("test2 %d %d\n", test2.mant, test2.exp); - - int32_t a = 0x7fffffff; - printf("%d, %d\n", a, a>>32);*/ //Custom setup for testing // ic_state.mu[0][0] = double_to_float_s32(0.0); ic_state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = 0; @@ -63,14 +13,7 @@ void test_init(void){ } ic_state_t test_get_state(void){ - //bfp_s32_use_exponent(&ic_state.error_bfp[0], -31); - //for(int v = 0; v < IC_FILTER_PHASES; v++){ - //bfp_complex_s32_use_exponent(&ic_state.H_hat_bfp[0][v], -31); - //} - //bfp_s32_use_exponent(&ic_state.sigma_XX_bfp[0], -31); - //bfp_s32_use_exponent(&ic_state.X_energy_bfp[0], -31); - //bfp_s32_use_exponent(&ic_state.inv_X_energy_bfp[0], -31); - //bfp_complex_s32_use_exponent(&ic_state.Y_hat_bfp[0], -31); + return ic_state; } diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index a7586489b..95d1c21b6 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -28,8 +28,8 @@ frame_advance = 240 num_phases = 10 y_channel_delay = 180 -input_file = 'pytest_audio_0.wav' -#input_file = "../../../examples/bare-metal/ic/input.wav" +#input_file = 'pytest_audio_360.wav' +input_file = "../../../examples/bare-metal/ic/input.wav" output_file = "output.wav" @pytest.fixture(params=[34]) @@ -186,8 +186,7 @@ def test_frame_compare(pre_test_stuff): output_wav_data = np.zeros((2, file_length)) - #for frame_start in range(0, file_length-proc_frame_length*2, frame_advance): - for frame_start in range(0, 2400, frame_advance): + for frame_start in range(0, file_length-proc_frame_length*2, frame_advance): input_frame = awu.get_frame(input_wav_data, frame_start, frame_advance, delays)[0:2,:] if False: @@ -204,6 +203,6 @@ def test_frame_compare(pre_test_stuff): scipy.io.wavfile.write(output_file, input_rate, pvc.float_to_int32(output_wav_data.T)) arith_closeness, geo_closeness, c_delay, peak2ave = pvc.pcm_closeness_metric(output_file) - #assert arith_closeness > 0.98 - #assert geo_closeness > 0.99 + assert arith_closeness > 0.98 + assert geo_closeness > 0.99 assert c_delay == 0 From 4b492d3ce409f8861cb3b0fc16af9d9f22a3b719 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 9 Jun 2022 09:41:24 +0100 Subject: [PATCH 82/92] changing stage_b test to use py_ic and lib_vad instead of lib_ap --- Jenkinsfile | 34 ++--- modules/lib_ic/src/ic_low_level.c | 42 +++++- .../characterise_c_py/characterise_c_py.py | 3 +- .../py_c_frame_compare/test_ic_frame.py | 51 +++---- test/lib_ic/test_ic_spec/test_ic_spec.py | 3 +- test/shared/config/new_ic_conf.json | 15 ++ test/stage_b/test_stage_b_frame.py | 130 +++++++++++------- 7 files changed, 177 insertions(+), 101 deletions(-) create mode 100644 test/shared/config/new_ic_conf.json diff --git a/Jenkinsfile b/Jenkinsfile index 900a72318..afe19edf6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -227,6 +227,23 @@ pipeline { } } } + stage('Stage B tests') { + steps { + dir("${REPO}/test/stage_b") { + viewEnv() { + withVenv { + withMounts([["projects", "projects/hydra_audio", "hydra_audio_stage_b_tests"]]) { + withEnv(["hydra_audio_PATH=$hydra_audio_stage_b_tests_PATH"]) { + runPython("python build_c_code.py") + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" + } + } + } + } + } + } + } stage('Examples') { steps { dir("${REPO}/examples/bare-metal/aec_1_thread") { @@ -458,23 +475,6 @@ pipeline { } } } - stage('Stage B tests') { - steps { - dir("${REPO}/test/stage_b") { - viewEnv() { - withVenv { - withMounts([["projects", "projects/hydra_audio", "hydra_audio_stage_b_tests"]]) { - withEnv(["hydra_audio_PATH=$hydra_audio_stage_b_tests_PATH"]) { - //runPython("python build_c_code.py") - //sh "pytest -s --junitxml=pytest_result.xml" - //junit "pytest_result.xml" - } - } - } - } - } - } - } stage('ADEC de_unit_tests') { steps { dir("${REPO}/test/lib_adec/de_unit_tests") { diff --git a/modules/lib_ic/src/ic_low_level.c b/modules/lib_ic/src/ic_low_level.c index b7b835695..0cdc722b6 100644 --- a/modules/lib_ic/src/ic_low_level.c +++ b/modules/lib_ic/src/ic_low_level.c @@ -245,6 +245,43 @@ void ic_filter_adapt(ic_state_t *state){ aec_priv_filter_adapt(state->H_hat_bfp[y_ch], state->X_fifo_1d_bfp, T_ptr, IC_X_CHANNELS, IC_FILTER_PHASES); } +static inline int32_t ashr32(int32_t x, right_shift_t shr) +{ + if(shr >= 0) + return x >> shr; + + int64_t tmp = ((int64_t)x) << -shr; + + if(tmp > INT32_MAX) return INT32_MAX; + else if(tmp < INT32_MIN) return INT32_MIN; + else return tmp; +} + +float_s32_t float_s32_add_fix( + const float_s32_t x, + const float_s32_t y) +{ + float_s32_t res; + + const headroom_t x_hr = HR_S32(x.mant); + const headroom_t y_hr = HR_S32(y.mant); + + const exponent_t x_min_exp = x.exp - x_hr; + const exponent_t y_min_exp = y.exp - y_hr; + + res.exp = MAX(x_min_exp, y_min_exp) + 1; + + const right_shift_t x_shr = res.exp - x.exp; + const right_shift_t y_shr = res.exp - y.exp; + + int32_t x_mant = (x_shr >= 32) ? 0 : ashr32(x.mant, x_shr); + int32_t y_mant = (y_shr >= 32) ? 0 : ashr32(y.mant, y_shr); + + res.mant = x_mant + y_mant; + + return res; +} + //Python port void ic_adaption_controller(ic_state_t *state, uint8_t vad){ ic_adaption_controller_state_t *ad_state = &state->ic_adaption_controller_state; @@ -271,9 +308,8 @@ void ic_adaption_controller(ic_state_t *state, uint8_t vad){ //noise_mu = 1.0 - self.smoothed_voice_chance float_s32_t noise_mu = float_s32_sub(one, ad_state->smoothed_voice_chance); - //noise_mu = noise_mu * min(1.0, np.sqrt(np.sqrt(self.output_energy/(self.input_energy + 0.000000001)))) - float_s32_t input_plus_delta = float_s32_add(ad_state->input_energy_slow, delta); + float_s32_t input_plus_delta = float_s32_add_fix(ad_state->input_energy_slow, delta); float_s32_t ratio = float_s32_div(ad_state->output_energy_slow, input_plus_delta); ratio = float_s32_sqrt(ratio); ratio = float_s32_sqrt(ratio); @@ -290,7 +326,7 @@ void ic_adaption_controller(ic_state_t *state, uint8_t vad){ } //fast_ratio = self.output_energy0 / (self.input_energy0 + 0.000000001) - input_plus_delta = float_s32_add(ad_state->input_energy_fast, delta); + input_plus_delta = float_s32_add_fix(ad_state->input_energy_fast, delta); float_s32_t fast_ratio = float_s32_div(ad_state->output_energy_fast, input_plus_delta); // if fast_ratio > 1.0: diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 76dce837c..3a4a970cb 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -88,7 +88,8 @@ def generate_test_audio(filename, audio_dir, max_freq, db, angle_theta, rt60, sa def process_py(input_file, output_file, audio_dir="."): - config_file = 'test_config.json' + #config_file = 'test_config.json' + config_file = '../../shared/config/new_ic_conf.json' twic.test_file(os.path.join(audio_dir, input_file), os.path.join(audio_dir, output_file), diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 95d1c21b6..1f1234846 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -6,6 +6,7 @@ import audio_wav_utils as awu import pytest import sys, os +import json from build import ic_test_py from ic_test_py import ffi @@ -27,10 +28,10 @@ fd_length = proc_frame_length // 2 + 1 frame_advance = 240 num_phases = 10 -y_channel_delay = 180 #input_file = 'pytest_audio_360.wav' input_file = "../../../examples/bare-metal/ic/input.wav" output_file = "output.wav" +ap_config_file = "../../shared/config/new_ic_conf.json" @pytest.fixture(params=[34]) def pre_test_stuff(request): @@ -39,18 +40,10 @@ def pre_test_stuff(request): class ic_comparison: def __init__(self): - self.ic = IC.adaptive_interference_canceller(frame_advance, proc_frame_length, num_phases, - mu = 0.36956599983386695, - delta = 7.450580593454381e-09, #two_mic_stereo.json - K = 1, - lamda = 0.9995117188, - gamma = 2.0, - leakage = 0.995, - y_channel_delay = y_channel_delay, - remove_NQ = False, - vnr_model = '../../../../py_vnr/model_output_0_0_2/trained_model.h5', - adaption_config = 'IC_ADAPTION_FORCE_ON' - ) + conf = pvc.json_to_dict(ap_config_file) + + self.ic = IC.adaptive_interference_canceller(**conf) + ic_test_lib.test_init() def process_frame(self, frame): @@ -82,9 +75,9 @@ def process_frame(self, frame): def check_filter_components(icc, frame_start): state = ic_test_lib.test_get_state() - if frame_start < 1200: - print('frame = ', frame_start // 240) - print('Y:') + if frame_start < 12000: + #print('frame = ', frame_start // 240) + #print('Y:') exp = state.Y_bfp[0].exp for i in range(proc_frame_length + 2): c_Y = np.array(state.y[0][i]).astype(np.float64) * (2 ** exp) @@ -96,9 +89,8 @@ def check_filter_components(icc, frame_start): rtol = np.ldexp(1, -13) if not np.isclose(c_Y, py_Y, rtol = rtol): print('C: ', c_Y, ', PY: ', py_Y) - print('TEST FAILED at i = ', i) - print('X_energy:') + #print('X_energy:') exp = state.X_energy_bfp[0].exp for i in range(fd_length): c_X_energy = np.array(state.X_energy[0][i]).astype(np.float64) * (2 ** exp) @@ -106,20 +98,18 @@ def check_filter_components(icc, frame_start): rtol = np.ldexp(1, -20) if not np.isclose(c_X_energy, py_X_energy, rtol = rtol): print('C: ', c_X_energy, ', PY: ', py_X_energy) - print('TEST FAILED at i = ', i) - print('Inverse X energy:') + #print('Inverse X energy:') exp = state.inv_X_energy_bfp[0].exp - print('exp = ', exp) for i in range(fd_length): c_inv_X_energy = np.array(state.inv_X_energy[0][i]).astype(np.float64) * (2 ** exp) py_inv_X_energy = icc.ic.inv_X_energy[0][i] rtol = np.ldexp(1, -10) if not np.isclose(c_inv_X_energy, py_inv_X_energy, rtol = rtol): + print('exp = ', exp) print('C: ', c_inv_X_energy, ', PY: ', py_inv_X_energy) - print('TEST FAILED at i = ', i) - print('sigma_xx:') + #print('sigma_xx:') exp = state.sigma_XX_bfp[0].exp for i in range(fd_length): c_sigma_xx = np.array(state.sigma_XX[0][i]).astype(np.float64) * (2 ** exp) @@ -127,10 +117,9 @@ def check_filter_components(icc, frame_start): rtol = np.ldexp(1, -19) if not np.isclose(c_sigma_xx, py_sigma_xx, rtol = rtol): print('C: ', c_sigma_xx, ', PY: ', py_sigma_xx) - print('TEST FAILED at i = ', i) if frame_start < 240: - print('H_hat:') + #print('H_hat:') for ph in range(num_phases): exp = state.H_hat_bfp[0][ph].exp for i in range(proc_frame_length + 2): @@ -145,9 +134,8 @@ def check_filter_components(icc, frame_start): rtol = np.ldexp(1, -12) if not np.isclose(c_H_hat, py_H_hat, rtol = rtol): print('C: ', c_H_hat, ', PY: ', py_H_hat) - print('TEST FAILED at ph = ', ph, ', i = ', i) - print('Y_hat:') + #print('Y_hat:') exp = state.Y_hat_bfp[0].exp for i in range(proc_frame_length + 2): c_Y_hat = 0 @@ -161,9 +149,9 @@ def check_filter_components(icc, frame_start): rtol = np.ldexp(1, -15) if not np.isclose(c_Y_hat, py_Y_hat, rtol = rtol): print('C: ', c_Y_hat, ', PY: ', py_Y_hat) - print('TEST FAILED at i = ', i) - print('error_ap:') + + #print('error_ap:') exp = state.error_bfp[0].exp for i in range(proc_frame_length + 2): c_error = np.array(state.Error[0][i]).astype(np.float64) * (2 ** exp) @@ -175,19 +163,18 @@ def check_filter_components(icc, frame_start): rtol = np.ldexp(1, -23) if not np.isclose(c_error, py_error, rtol = rtol): print('C: ', c_error, ', PY: ', py_error) - print('TEST FAILED at i = ', i) + def test_frame_compare(pre_test_stuff): icc = ic_comparison() input_rate, input_wav_file = scipy.io.wavfile.read(input_file, 'r') input_wav_data, input_channel_count, file_length = awu.parse_audio(input_wav_file) - delays = np.zeros(input_channel_count) #we do delay of y channel in process_frame above and in C rather than awu.get_frame output_wav_data = np.zeros((2, file_length)) for frame_start in range(0, file_length-proc_frame_length*2, frame_advance): - input_frame = awu.get_frame(input_wav_data, frame_start, frame_advance, delays)[0:2,:] + input_frame = awu.get_frame(input_wav_data, frame_start, frame_advance)[0:2,:] if False: print ('# ' + str(frame_start // frame_advance)) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 43067d469..88c66d18d 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -137,7 +137,8 @@ def write_output(test_name, output, c_or_py): def process_py(input_data, test_name): file_length = input_data.shape[1] - config_file = '../characterise_c_py/test_config.json' + #config_file = '../characterise_c_py/test_config.json' + config_file = '../../shared/new_ic_conf.json' ic_parameters = json_to_dict(config_file) output, Mu, Input_vnr_pred, Output_vnr_pred, Control_flag = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) diff --git a/test/shared/config/new_ic_conf.json b/test/shared/config/new_ic_conf.json new file mode 100644 index 000000000..3adbaed82 --- /dev/null +++ b/test/shared/config/new_ic_conf.json @@ -0,0 +1,15 @@ +{ + "frame_advance": 240, + "proc_frame_length": 512, + "phases": 10, + "mu" : 0.36956599983386695, + "delta" : 7.450580593454381e-09, + "y_channel_delay": 180, + "K" : 1, + "lamda" : 0.9995117188, + "gamma" : 2.0, + "leakage" : 0.995, + "remove_NQ" : false, + "vnr_model" : "../../../../py_vnr/model_output/trained_model.h5", + "adaption_config": "IC_ADAPTION_FORCE_ON" +} diff --git a/test/stage_b/test_stage_b_frame.py b/test/stage_b/test_stage_b_frame.py index a947c2967..b9f2d7a75 100644 --- a/test/stage_b/test_stage_b_frame.py +++ b/test/stage_b/test_stage_b_frame.py @@ -14,25 +14,22 @@ import ic_vad_test_py.lib as ic_vad_test_lib package_dir = os.path.dirname(os.path.abspath(__file__)) -att_path = os.path.join(package_dir,'../../audio_test_tools/python/') -py_ic_path = os.path.join(package_dir,'../../../lib_interference_canceller/python') -py_stage_b_path = os.path.join(package_dir,'../../../lib_audio_pipelines/python') py_vad_path = os.path.join(package_dir,'../../../lib_vad/python') pvc_path = os.path.join(package_dir, '../shared/python') hydra_audio_base_dir = os.path.expanduser("~/hydra_audio/") -sys.path.append(att_path) -sys.path.append(py_stage_b_path) -sys.path.append(py_ic_path) sys.path.append(py_vad_path) sys.path.append(pvc_path) -from ap_stage_b import ap_stage_b +import IC +import vad import py_vs_c_utils as pvc -ap_config_file = "../shared/config/two_mic_stereo_prev_arch.json" +ap_config_file = "../shared/config/new_ic_conf.json" input_file = "input.wav" output_file = "output.wav" +frames_print = 15 + @pytest.fixture(params=[ap_config_file]) def test_config(request): ap_conf = pvc.json_to_dict(request.param) @@ -46,22 +43,29 @@ def test_config(request): cmd = f"sox {test_track} -c 2 {input_file} trim 120 30" subprocess.run(cmd.split()) - return ap_conf['ap_stage_b_conf'] + return ap_conf class stage_b_comparison: - def __init__(self, stage_b_conf): + def __init__(self, ic_conf): + + self.frame_advance = ic_conf["frame_advance"] + self.proc_frame_length = ic_conf["proc_frame_length"] + self.num_phases = ic_conf["phases"] + + self.ic = IC.adaptive_interference_canceller(**ic_conf) + self.vad = vad.vad() - self.frame_advance = stage_b_conf["ic_conf"]["frame_advance"] - self.proc_frame_length = stage_b_conf["ic_conf"]["proc_frame_length"] - self.num_phases = stage_b_conf["ic_conf"]["phases"] - self.passthrough_channel_count = stage_b_conf["ic_conf"]["passthrough_channel_count"] + self.smoothed_voice_chance = 1.0 + self.voice_chance_alpha = 0.99 - #override to match C version - stage_b_conf["ic_conf"]["use_noise_minimisation"] = False - print(stage_b_conf) + self.input_energy = 0.0 + self.output_energy = 0.0 + self.energy_alpha = 0.999 - self.sb = ap_stage_b(self.frame_advance, stage_b_conf["ic_conf"], self.passthrough_channel_count, mic_shift=0, mic_saturate=0) + self.input_energy0 = 0.0 + self.output_energy0 = 0.0 + self.energy_alpha0 = 0.98 ic_vad_test_lib.test_init() @@ -70,19 +74,42 @@ def __init__(self, stage_b_conf): self.py_vad = None self.c_vad = None + def adaption_controller(self, vad_result, index): + + self.smoothed_voice_chance = self.voice_chance_alpha*self.smoothed_voice_chance + self.smoothed_voice_chance = max(self.smoothed_voice_chance, vad_result) + + noise_mu = 1.0 - self.smoothed_voice_chance + noise_mu = noise_mu * min(1.0, np.sqrt(np.sqrt(self.output_energy/(self.input_energy + 0.000000001)))) + + fast_ratio = self.output_energy0 / (self.input_energy0 + 0.000000001) + if (index < frames_print) and False: + print(f"py_noise_mu: {noise_mu}") - def process_frame(self, frame): + if fast_ratio > 1.0: + self.ic.set_leakage(0.995) + self.ic.set_mu(0.0) + else: + self.ic.set_leakage (1.0) + self.ic.set_mu(noise_mu) + + def process_frame(self, frame, index): #we need to delay the y for python as not done in model #first copy the input data for C ver before we modfiy it frame_int = pvc.float_to_int32(frame) - - class stage_a_md: - vad = 0 #Run a frame through python - output_py, metadata = self.sb.process_frame(frame, stage_a_md) - py_vad = metadata.vad_result - self.py_vad = py_vad + output_py, Error = self.ic.process_frame(frame) + + self.input_energy = self.energy_alpha*self.input_energy + (1.0 - self.energy_alpha) * self.ic.e_in + self.input_energy0 = self.energy_alpha0*self.input_energy0 + (1.0 - self.energy_alpha0) * self.ic.e_in + self.output_energy = self.energy_alpha*self.output_energy + (1.0 - self.energy_alpha) * self.ic.e_out + self.output_energy0 = self.energy_alpha0*self.output_energy0 + (1.0 - self.energy_alpha0) * self.ic.e_out + + py_vad = self.vad.run(output_py) + self.adaption_controller(py_vad, index) + #self.py_vad = py_vad + self.ic.adapt(Error) #Grab a pointer to the data storage of the numpy arrays y_data = ffi.cast("int32_t *", ffi.from_buffer(frame_int[0].data)) @@ -92,49 +119,57 @@ class stage_a_md: ic_vad_test_lib.test_filter(y_data, x_data, output_c_ptr) c_vad = pvc.uint8_to_float(ic_vad_test_lib.test_vad(output_c_ptr)) self.c_vad = c_vad.copy() - print(f"1py_vad: {py_vad:.4f}, c_vad: {c_vad:.4f}") + if (index < frames_print) and False: + print(f"1py_vad: {py_vad:.4f}, c_vad: {c_vad:.4f}") #note we override c_vad to match py_vad for comparison c_vad = pvc.float_to_uint8(np.array(py_vad)) - print(f"2py_vad: {py_vad:.4f}, c_vad: {c_vad:.4f}") + #c_vad = int(0) # dummy + if (index < frames_print) and False: + print(f"2py_vad: {py_vad:.4f}, c_vad: {c_vad:.4f}") ic_vad_test_lib.test_adapt(c_vad, output_c_ptr) ic_state = ic_vad_test_lib.test_get_ic_state() self.ic_state = ic_state - print(f"py_mu: {self.sb.ifc.mu}, c_mu: {pvc.float_s32_to_float(ic_state.mu[0][0])}") + if (index < frames_print) and False: + print(f"py_smooth_vc: {self.smoothed_voice_chance}, c_smooth_vc: {pvc.float_s32_to_float(ic_state.ic_adaption_controller_state.smoothed_voice_chance)}") + print(f"py_mu: {self.ic.mu}, c_mu: {pvc.float_s32_to_float(ic_state.mu[0][0])}") # print(pvc.float_s32_to_float(state.config_params.delta)) cies = pvc.float_s32_to_float(ic_state.ic_adaption_controller_state.input_energy_slow) coes = pvc.float_s32_to_float(ic_state.ic_adaption_controller_state.output_energy_slow) cief = pvc.float_s32_to_float(ic_state.ic_adaption_controller_state.input_energy_fast) coef = pvc.float_s32_to_float(ic_state.ic_adaption_controller_state.output_energy_fast) - print(f"c - ies: {cies} oes: {coes} ief: {cief} oef: {coef}") - print(f"p - ies: {self.sb.input_energy} oes: {self.sb.output_energy} ief: {self.sb.input_energy0} oef: {self.sb.output_energy0}") - return output_py[0], pvc.int32_to_float(output_c) - + if (index < frames_print) and False: + print(f"c - ies: {cies} oes: {coes} ief: {cief} oef: {coef}") + print(f"p - ies: {self.input_energy} oes: {self.output_energy} ief: {self.input_energy0} oef: {self.output_energy0}") + if (index < frames_print) and False: + print('-') + return output_py, pvc.int32_to_float(output_c) def test_frame_compare(test_config): sbc = stage_b_comparison(test_config) - frame_advance = test_config["ic_conf"]["frame_advance"] - proc_frame_length = test_config["ic_conf"]["proc_frame_length"] + frame_advance = test_config["frame_advance"] + proc_frame_length = test_config["proc_frame_length"] input_rate, input_wav_file = scipy.io.wavfile.read(input_file, 'r') input_wav_data, input_channel_count, file_length = awu.parse_audio(input_wav_file) - delays = np.zeros(input_channel_count) #we do delay of y channel in process_frame above and in C rather than awu.get_frame output_wav_data = np.zeros((2, file_length)) mu_log = np.zeros((file_length//frame_advance, 2)) vad_log = np.zeros((file_length//frame_advance, 2)) + + #for frame_start in range(0, frame_advance * 15, frame_advance): for frame_start in range(0, file_length-proc_frame_length*2, frame_advance): - input_frame = awu.get_frame(input_wav_data, frame_start, frame_advance, delays)[0:2,:] + input_frame = awu.get_frame(input_wav_data, frame_start, frame_advance)[0:2,:] if False: print ('# ' + str(frame_start // frame_advance)) - output_py, output_c = sbc.process_frame(input_frame) - mu_log[frame_start//frame_advance, :] = np.array([sbc.sb.ifc.mu, pvc.float_s32_to_float(sbc.ic_state.mu[0][0])]) + output_py, output_c = sbc.process_frame(input_frame, frame_start // frame_advance) + mu_log[frame_start//frame_advance, :] = np.array([sbc.ic.mu, pvc.float_s32_to_float(sbc.ic_state.mu[0][0])]) vad_log[frame_start//frame_advance, :] = np.array([sbc.py_vad, sbc.c_vad]) output_wav_data[0, frame_start: frame_start + frame_advance] = output_py @@ -157,7 +192,8 @@ def test_frame_compare(test_config): def test_adaption_controller(test_config): stage_b_conf = test_config #instantiate and init a stage B instance - sb = ap_stage_b(stage_b_conf["ic_conf"]["frame_advance"], stage_b_conf["ic_conf"],stage_b_conf["ic_conf"]["passthrough_channel_count"], mic_shift=0, mic_saturate=0) + #sb = ap_stage_b(stage_b_conf["ic_conf"]["frame_advance"], stage_b_conf["ic_conf"],stage_b_conf["ic_conf"]["passthrough_channel_count"], mic_shift=0, mic_saturate=0) + sbc = stage_b_comparison(test_config) #Init the avona instance ic_vad_test_lib.test_init() @@ -178,14 +214,14 @@ def test_adaption_controller(test_config): for vad, in_s, out_s, in_f, out_f in zip(vad_vects, in_energy_vects_slow, out_energy_vects_slow, in_energy_vects_fast, out_energy_vects_fast): - sb.input_energy = in_s - sb.output_energy = out_s - sb.input_energy0 = in_f - sb.output_energy0 = out_f - - sb.adaption_controller(vad) - py_mu = sb.ifc.mu - py_svc = sb.smoothed_voice_chance + sbc.input_energy = in_s + sbc.output_energy = out_s + sbc.input_energy0 = in_f + sbc.output_energy0 = out_f + + sbc.adaption_controller(vad, 1000) + py_mu = sbc.ic.mu + py_svc = sbc.smoothed_voice_chance c_vad = pvc.float_to_uint8(vad) ic_vad_test_lib.test_set_ic_energies(in_s, out_s, in_f, out_f) From 6f58408fe23825535e12a0377d7dd48dfcee6b13 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 9 Jun 2022 10:04:03 +0100 Subject: [PATCH 83/92] moving config files and testing diffrent params --- test/lib_ic/characterise_c_py/characterise_c_py.py | 4 ++-- test/lib_ic/test_ic_spec/test_ic_spec.py | 2 +- .../test_config.json => shared/config/ic_conf_big_delta.json} | 0 .../{new_ic_conf.json => ic_conf_no_adapt_control.json} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename test/{lib_ic/characterise_c_py/test_config.json => shared/config/ic_conf_big_delta.json} (100%) rename test/shared/config/{new_ic_conf.json => ic_conf_no_adapt_control.json} (100%) diff --git a/test/lib_ic/characterise_c_py/characterise_c_py.py b/test/lib_ic/characterise_c_py/characterise_c_py.py index 3a4a970cb..2881d65ee 100644 --- a/test/lib_ic/characterise_c_py/characterise_c_py.py +++ b/test/lib_ic/characterise_c_py/characterise_c_py.py @@ -88,8 +88,8 @@ def generate_test_audio(filename, audio_dir, max_freq, db, angle_theta, rt60, sa def process_py(input_file, output_file, audio_dir="."): - #config_file = 'test_config.json' - config_file = '../../shared/config/new_ic_conf.json' + + config_file = '../../shared/config/ic_conf_big_delta.json' twic.test_file(os.path.join(audio_dir, input_file), os.path.join(audio_dir, output_file), diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index 88c66d18d..bc7c58eac 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -138,7 +138,7 @@ def process_py(input_data, test_name): file_length = input_data.shape[1] #config_file = '../characterise_c_py/test_config.json' - config_file = '../../shared/new_ic_conf.json' + config_file = '../../shared/ic_conf_no_adapt_control.json' ic_parameters = json_to_dict(config_file) output, Mu, Input_vnr_pred, Output_vnr_pred, Control_flag = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) diff --git a/test/lib_ic/characterise_c_py/test_config.json b/test/shared/config/ic_conf_big_delta.json similarity index 100% rename from test/lib_ic/characterise_c_py/test_config.json rename to test/shared/config/ic_conf_big_delta.json diff --git a/test/shared/config/new_ic_conf.json b/test/shared/config/ic_conf_no_adapt_control.json similarity index 100% rename from test/shared/config/new_ic_conf.json rename to test/shared/config/ic_conf_no_adapt_control.json From 3f3e47aaa83b7a2d464afa5566e93e14372559b3 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 9 Jun 2022 10:23:39 +0100 Subject: [PATCH 84/92] fixing some paths --- Jenkinsfile | 2 ++ test/lib_ic/py_c_frame_compare/test_ic_frame.py | 2 +- test/stage_b/test_stage_b_frame.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index afe19edf6..df099ccb3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -151,6 +151,8 @@ pipeline { // sh "make -j8" //sh "make VERBOSE=1" sh "make -C test/lib_ic/" + sh "make -C modules/lib_vad/" + sh "make -C modules/lib_vnr/" //We need to put this here because it is not fetched until we build sh "pip install -e avona_deps/xscope_fileio" diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 1f1234846..45afc13e0 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -31,7 +31,7 @@ #input_file = 'pytest_audio_360.wav' input_file = "../../../examples/bare-metal/ic/input.wav" output_file = "output.wav" -ap_config_file = "../../shared/config/new_ic_conf.json" +ap_config_file = "../../shared/config/ic_conf_no_adapt_control.json" @pytest.fixture(params=[34]) def pre_test_stuff(request): diff --git a/test/stage_b/test_stage_b_frame.py b/test/stage_b/test_stage_b_frame.py index b9f2d7a75..0dace81d3 100644 --- a/test/stage_b/test_stage_b_frame.py +++ b/test/stage_b/test_stage_b_frame.py @@ -24,7 +24,7 @@ import vad import py_vs_c_utils as pvc -ap_config_file = "../shared/config/new_ic_conf.json" +ap_config_file = "../shared/config/ic_conf_no_adapt_control.json" input_file = "input.wav" output_file = "output.wav" From 038997c2ae727989ca9108c6b42f90d3112535a0 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 9 Jun 2022 10:49:18 +0100 Subject: [PATCH 85/92] stage_b should pass without adapt controller --- test/lib_ic/test_ic_spec/test_ic_spec.py | 4 ++-- test/stage_b/ic_vad_test.c | 2 +- test/stage_b/test_stage_b_frame.py | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index bc7c58eac..fac33cd29 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -137,8 +137,8 @@ def write_output(test_name, output, c_or_py): def process_py(input_data, test_name): file_length = input_data.shape[1] - #config_file = '../characterise_c_py/test_config.json' - config_file = '../../shared/ic_conf_no_adapt_control.json' + + config_file = '../../shared/config/ic_conf_no_adapt_control.json' ic_parameters = json_to_dict(config_file) output, Mu, Input_vnr_pred, Output_vnr_pred, Control_flag = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) diff --git a/test/stage_b/ic_vad_test.c b/test/stage_b/ic_vad_test.c index 88e11ef35..1e6328044 100644 --- a/test/stage_b/ic_vad_test.c +++ b/test/stage_b/ic_vad_test.c @@ -10,7 +10,7 @@ void test_init(void){ vad_init(&vad_state); ic_init(&ic_state); //Custom setup for testing - ic_state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = 1; + ic_state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = 0; // ic_state.config_params.delta = double_to_float_s32(0.0156); } diff --git a/test/stage_b/test_stage_b_frame.py b/test/stage_b/test_stage_b_frame.py index 0dace81d3..7c076a12c 100644 --- a/test/stage_b/test_stage_b_frame.py +++ b/test/stage_b/test_stage_b_frame.py @@ -107,7 +107,7 @@ def process_frame(self, frame, index): self.output_energy0 = self.energy_alpha0*self.output_energy0 + (1.0 - self.energy_alpha0) * self.ic.e_out py_vad = self.vad.run(output_py) - self.adaption_controller(py_vad, index) + #self.adaption_controller(py_vad, index) #self.py_vad = py_vad self.ic.adapt(Error) @@ -123,8 +123,8 @@ def process_frame(self, frame, index): print(f"1py_vad: {py_vad:.4f}, c_vad: {c_vad:.4f}") #note we override c_vad to match py_vad for comparison - c_vad = pvc.float_to_uint8(np.array(py_vad)) - #c_vad = int(0) # dummy + #c_vad = pvc.float_to_uint8(np.array(py_vad)) + c_vad = int(0) # dummy if (index < frames_print) and False: print(f"2py_vad: {py_vad:.4f}, c_vad: {c_vad:.4f}") ic_vad_test_lib.test_adapt(c_vad, output_c_ptr) @@ -143,6 +143,7 @@ def process_frame(self, frame, index): if (index < frames_print) and False: print(f"c - ies: {cies} oes: {coes} ief: {cief} oef: {coef}") print(f"p - ies: {self.input_energy} oes: {self.output_energy} ief: {self.input_energy0} oef: {self.output_energy0}") + print(f"py_mu: {self.ic.mu}, c_mu: {pvc.float_s32_to_float(ic_state.mu[0][0])}") if (index < frames_print) and False: print('-') return output_py, pvc.int32_to_float(output_c) From 71b3d35982ba933647090c1860ee5991fcec951f Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 9 Jun 2022 11:08:07 +0100 Subject: [PATCH 86/92] changing config for spec test --- test/lib_ic/test_ic_spec/test_ic_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib_ic/test_ic_spec/test_ic_spec.py b/test/lib_ic/test_ic_spec/test_ic_spec.py index fac33cd29..f5238dc1a 100644 --- a/test/lib_ic/test_ic_spec/test_ic_spec.py +++ b/test/lib_ic/test_ic_spec/test_ic_spec.py @@ -138,7 +138,7 @@ def process_py(input_data, test_name): file_length = input_data.shape[1] - config_file = '../../shared/config/ic_conf_no_adapt_control.json' + config_file = '../../shared/config/ic_conf_big_delta.json' ic_parameters = json_to_dict(config_file) output, Mu, Input_vnr_pred, Output_vnr_pred, Control_flag = test_wav_ic.test_data(input_data, 16000, file_length, ic_parameters, verbose=False) From ff0a9311c21f4eb4ca62a4954d9434b798d77368 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 9 Jun 2022 12:07:41 +0100 Subject: [PATCH 87/92] enabling adaptaion controller in stage_b test, making adapt_control delta smaller --- Jenkinsfile | 10 +++++----- modules/lib_ic/src/ic_low_level.c | 3 ++- test/stage_b/ic_vad_test.c | 2 +- test/stage_b/test_stage_b_frame.py | 24 ++++++++++++++---------- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index df099ccb3..8b49e06f6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -358,8 +358,8 @@ pipeline { viewEnv() { withVenv { runPython("python build_vnr_feature_extraction.py") - sh "pytest -s --junitxml=pytest_result.xml" - junit "pytest_result.xml" + //sh "pytest -s --junitxml=pytest_result.xml" + //junit "pytest_result.xml" } } } @@ -444,8 +444,8 @@ pipeline { dir("${REPO}/test/lib_ic/ic_unit_tests") { viewEnv() { withVenv { - //sh "pytest -n 2 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -471,7 +471,7 @@ pipeline { withVenv { //This is a unit test for ic_calc_vnr_pred function. //It compares the avona output with py_ic model output - sh "pytest -n1 --junitxml=pytest_result.xml" + //sh "pytest -n1 --junitxml=pytest_result.xml" } } } diff --git a/modules/lib_ic/src/ic_low_level.c b/modules/lib_ic/src/ic_low_level.c index 0cdc722b6..b71d9b3b4 100644 --- a/modules/lib_ic/src/ic_low_level.c +++ b/modules/lib_ic/src/ic_low_level.c @@ -293,7 +293,8 @@ void ic_adaption_controller(ic_state_t *state, uint8_t vad){ const float_s32_t one = {1, 0}; const float_s32_t zero = {0, 0}; - const float_s32_t delta = {1100, -40}; //1100 * 2**-40 = 0.000000001 (from stage_b.py) + //const float_s32_t delta = {1100, -40}; //1100 * 2**-40 = 0.000000001 (from stage_b.py) + const float_s32_t delta = {352, -45}; //352 * 2**-45 ~ 0.00000000001 exponent_t q0_8_exp = -8; float_s32_t vad_float = {vad, q0_8_exp}; //convert to float between 0 and 0.99609375 diff --git a/test/stage_b/ic_vad_test.c b/test/stage_b/ic_vad_test.c index 1e6328044..88e11ef35 100644 --- a/test/stage_b/ic_vad_test.c +++ b/test/stage_b/ic_vad_test.c @@ -10,7 +10,7 @@ void test_init(void){ vad_init(&vad_state); ic_init(&ic_state); //Custom setup for testing - ic_state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = 0; + ic_state.ic_adaption_controller_state.adaption_controller_config.enable_adaption_controller = 1; // ic_state.config_params.delta = double_to_float_s32(0.0156); } diff --git a/test/stage_b/test_stage_b_frame.py b/test/stage_b/test_stage_b_frame.py index 7c076a12c..a40fe9009 100644 --- a/test/stage_b/test_stage_b_frame.py +++ b/test/stage_b/test_stage_b_frame.py @@ -75,14 +75,15 @@ def __init__(self, ic_conf): self.c_vad = None def adaption_controller(self, vad_result, index): + delta = 0.00000000001 self.smoothed_voice_chance = self.voice_chance_alpha*self.smoothed_voice_chance self.smoothed_voice_chance = max(self.smoothed_voice_chance, vad_result) noise_mu = 1.0 - self.smoothed_voice_chance - noise_mu = noise_mu * min(1.0, np.sqrt(np.sqrt(self.output_energy/(self.input_energy + 0.000000001)))) + noise_mu = noise_mu * min(1.0, np.sqrt(np.sqrt(self.output_energy/(self.input_energy + delta)))) - fast_ratio = self.output_energy0 / (self.input_energy0 + 0.000000001) + fast_ratio = self.output_energy0 / (self.input_energy0 + delta) if (index < frames_print) and False: print(f"py_noise_mu: {noise_mu}") @@ -101,13 +102,17 @@ def process_frame(self, frame, index): #Run a frame through python output_py, Error = self.ic.process_frame(frame) + if (index < frames_print) and False: + print(f"PY e_in: {self.ic.e_in}") + print(f"PY e_err_out: {self.ic.e_err_out}") + self.input_energy = self.energy_alpha*self.input_energy + (1.0 - self.energy_alpha) * self.ic.e_in self.input_energy0 = self.energy_alpha0*self.input_energy0 + (1.0 - self.energy_alpha0) * self.ic.e_in - self.output_energy = self.energy_alpha*self.output_energy + (1.0 - self.energy_alpha) * self.ic.e_out - self.output_energy0 = self.energy_alpha0*self.output_energy0 + (1.0 - self.energy_alpha0) * self.ic.e_out + self.output_energy = self.energy_alpha*self.output_energy + (1.0 - self.energy_alpha) * self.ic.e_err_out + self.output_energy0 = self.energy_alpha0*self.output_energy0 + (1.0 - self.energy_alpha0) * self.ic.e_err_out py_vad = self.vad.run(output_py) - #self.adaption_controller(py_vad, index) + self.adaption_controller(py_vad, index) #self.py_vad = py_vad self.ic.adapt(Error) @@ -123,8 +128,8 @@ def process_frame(self, frame, index): print(f"1py_vad: {py_vad:.4f}, c_vad: {c_vad:.4f}") #note we override c_vad to match py_vad for comparison - #c_vad = pvc.float_to_uint8(np.array(py_vad)) - c_vad = int(0) # dummy + c_vad = pvc.float_to_uint8(np.array(py_vad)) + #c_vad = int(0) # dummy if (index < frames_print) and False: print(f"2py_vad: {py_vad:.4f}, c_vad: {c_vad:.4f}") ic_vad_test_lib.test_adapt(c_vad, output_c_ptr) @@ -140,11 +145,11 @@ def process_frame(self, frame, index): coes = pvc.float_s32_to_float(ic_state.ic_adaption_controller_state.output_energy_slow) cief = pvc.float_s32_to_float(ic_state.ic_adaption_controller_state.input_energy_fast) coef = pvc.float_s32_to_float(ic_state.ic_adaption_controller_state.output_energy_fast) - if (index < frames_print) and False: + if (index < frames_print) and True: print(f"c - ies: {cies} oes: {coes} ief: {cief} oef: {coef}") print(f"p - ies: {self.input_energy} oes: {self.output_energy} ief: {self.input_energy0} oef: {self.output_energy0}") print(f"py_mu: {self.ic.mu}, c_mu: {pvc.float_s32_to_float(ic_state.mu[0][0])}") - if (index < frames_print) and False: + if (index < frames_print) and True: print('-') return output_py, pvc.int32_to_float(output_c) @@ -161,7 +166,6 @@ def test_frame_compare(test_config): mu_log = np.zeros((file_length//frame_advance, 2)) vad_log = np.zeros((file_length//frame_advance, 2)) - #for frame_start in range(0, frame_advance * 15, frame_advance): for frame_start in range(0, file_length-proc_frame_length*2, frame_advance): input_frame = awu.get_frame(input_wav_data, frame_start, frame_advance)[0:2,:] From f325cf6e7473d200697cfde9cfedb4d63c1cd108 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 9 Jun 2022 14:53:04 +0100 Subject: [PATCH 88/92] bringing back all Jenkins jobs and develop view file --- Jenkinsfile | 314 +++++++++++++++++++++++++--------------------------- 1 file changed, 153 insertions(+), 161 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fcf7465c1..84d2bd88a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { } environment { REPO = 'sw_avona' - VIEW = 'sw_avona_ic_test' + VIEW = getViewName(REPO) FULL_TEST = """${(params.FULL_TEST_OVERRIDE || env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'main' @@ -56,8 +56,7 @@ pipeline { withVenv { sh "cmake --version" sh 'cmake -S.. -DPython3_FIND_VIRTUALENV="ONLY" -DTEST_WAV_ADEC_BUILD_CONFIG="1 2 2 10 5" -DAVONA_BUILD_TESTS=ON' - //sh "make -j8" - sh "make -C test/lib_ic/" + sh "make -j8" } } } @@ -73,16 +72,16 @@ pipeline { dir("${REPO}/test/lib_vnr/py_c_feature_compare") { viewEnv() { withVenv { - //runPython("python build_vnr_feature_extraction.py") + runPython("python build_vnr_feature_extraction.py") } } } dir("${REPO}") { - //stash name: 'cmake_build_x86_examples', includes: 'build/**/avona_example_bare_metal_*' + stash name: 'cmake_build_x86_examples', includes: 'build/**/avona_example_bare_metal_*' //We are archveing the x86 version. Be careful - these have the same file name as the xcore versions but the linker should warn at least in this case - //stash name: 'cmake_build_x86_libs', includes: 'build/**/*.a' - //archiveArtifacts artifacts: "build/**/avona_example_bare_metal_*", fingerprint: true - //stash name: 'vnr_py_c_feature_compare', includes: 'test/lib_vnr/py_c_feature_compare/build/**' + stash name: 'cmake_build_x86_libs', includes: 'build/**/*.a' + archiveArtifacts artifacts: "build/**/avona_example_bare_metal_*", fingerprint: true + stash name: 'vnr_py_c_feature_compare', includes: 'test/lib_vnr/py_c_feature_compare/build/**' stash name: 'py_c_frame_compare', includes: 'test/lib_ic/py_c_frame_compare/build/**' } // Now do xcore files @@ -98,8 +97,7 @@ pipeline { sh 'cmake -S.. -DCMAKE_TOOLCHAIN_FILE=../xmos_cmake_toolchain/xs3a.cmake -DPython3_VIRTUALENV_FIND="ONLY" -DTEST_SPEEDUP_FACTOR=4 -DAVONA_BUILD_TESTS=ON' } } - //sh "make -j8" - sh "make -C test/lib_ic/" + sh "make -j8" } } } @@ -148,11 +146,7 @@ pipeline { withVenv { sh "cmake --version" sh 'cmake -S.. -DPython3_FIND_VIRTUALENV="ONLY" -DTEST_WAV_ADEC_BUILD_CONFIG="1 2 2 10 5" -DAVONA_BUILD_TESTS=ON' - // sh "make -j8" - //sh "make VERBOSE=1" - sh "make -C test/lib_ic/" - sh "make -C modules/lib_vad/" - sh "make -C modules/lib_vnr/" + sh "make -j8" //We need to put this here because it is not fetched until we build sh "pip install -e avona_deps/xscope_fileio" @@ -178,149 +172,81 @@ pipeline { } } } - stage('IC Python C equivalence') { - steps { - dir("${REPO}/test/lib_ic/py_c_frame_compare") { - viewEnv() { - withVenv { - runPython("python build_ic_frame_proc.py") - sh "pytest -s --junitxml=pytest_result.xml" - junit "pytest_result.xml" - } - } - } - } - } - stage('IC characterisation') { - steps { - dir("${REPO}/test/lib_ic/characterise_c_py") { - viewEnv() { - withVenv { - //This test compares the suppression performance across angles between model and C implementation - //and fails if they differ significantly. It requires that the C implementation run with fixed mu - sh "pytest -s --junitxml=pytest_result.xml" //-n 2 fails often so run single threaded and also print result - junit "pytest_result.xml" - //This script sweeps the y_delay value to find what the optimum suppression is across RT60 and angle. - //It's more of a model develpment tool than testing the implementation so not run. It take a few minutes. - // sh "python sweep_ic_delay.py" - } - } - } - } - } - stage('IC test specification') { - steps { - dir("${REPO}/test/lib_ic/test_ic_spec") { - viewEnv() { - withVenv { - //This test compares the model and C implementation over a range of scenarious for: - //convergence_time, db_suppression, maximum noise added to input (to test for stability) - //and expected group delay. It will fail if these are not met. - sh "pytest -s --junitxml=pytest_result.xml" - //junit "pytest_result.xml" - sh "python print_stats.py > ic_spec_summary.txt" - //This script generates a number of polar plots of attenuation vs null point angle vs freq - //It currently only uses the python model to do this. It takes about 40 mins for all plots - //and generates a series of IC_performance_xxxHz.svg files which could be archived - // sh "python plot_ic.py" - } - } - archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true - } - } - } - stage('Stage B tests') { - steps { - dir("${REPO}/test/stage_b") { - viewEnv() { - withVenv { - withMounts([["projects", "projects/hydra_audio", "hydra_audio_stage_b_tests"]]) { - withEnv(["hydra_audio_PATH=$hydra_audio_stage_b_tests_PATH"]) { - runPython("python build_c_code.py") - sh "pytest -s --junitxml=pytest_result.xml" - junit "pytest_result.xml" - } - } - } - } - } - } - } stage('Examples') { steps { dir("${REPO}/examples/bare-metal/aec_1_thread") { viewEnv() { withVenv { - //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/aec_1_thread/bin/avona_example_bare_metal_aec_1_thread.xe --input ../shared_src/test_streams/aec_example_input.wav" + sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/aec_1_thread/bin/avona_example_bare_metal_aec_1_thread.xe --input ../shared_src/test_streams/aec_example_input.wav" } } } dir("${REPO}/examples/bare-metal/aec_2_threads") { viewEnv() { withVenv { - //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/aec_2_threads/bin/avona_example_bare_metal_aec_2_thread.xe --input ../shared_src/test_streams/aec_example_input.wav" + sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/aec_2_threads/bin/avona_example_bare_metal_aec_2_thread.xe --input ../shared_src/test_streams/aec_example_input.wav" // Make sure 1 thread and 2 threads output is bitexact - //sh "diff output.wav ../aec_1_thread/output.wav" + sh "diff output.wav ../aec_1_thread/output.wav" } } } dir("${REPO}/examples/bare-metal/ic") { viewEnv() { withVenv { - //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/ic/bin/avona_example_bare_metal_ic.xe" - //sh "mv output.wav ic_example_output.wav" + sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/ic/bin/avona_example_bare_metal_ic.xe" + sh "mv output.wav ic_example_output.wav" } } - //archiveArtifacts artifacts: "ic_example_output.wav", fingerprint: true + archiveArtifacts artifacts: "ic_example_output.wav", fingerprint: true } dir("${REPO}/examples/bare-metal/vad") { viewEnv() { withVenv { - //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/vad/bin/avona_example_bare_metal_vad.xe" + sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/vad/bin/avona_example_bare_metal_vad.xe" } } } dir("${REPO}/examples/bare-metal/pipeline_single_threaded") { viewEnv() { withVenv { - //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_single_threaded/bin/avona_example_bare_metal_pipeline_single_thread.xe --input ../shared_src/test_streams/pipeline_example_input.wav" + sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_single_threaded/bin/avona_example_bare_metal_pipeline_single_thread.xe --input ../shared_src/test_streams/pipeline_example_input.wav" } } } dir("${REPO}/examples/bare-metal/pipeline_multi_threaded") { viewEnv() { withVenv { - //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_multi_threaded/bin/avona_example_bare_metal_pipeline_multi_thread.xe --input ../shared_src/test_streams/pipeline_example_input.wav" + sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_multi_threaded/bin/avona_example_bare_metal_pipeline_multi_thread.xe --input ../shared_src/test_streams/pipeline_example_input.wav" // Make sure single thread and multi threads pipeline output is bitexact - //sh "diff output.wav ../pipeline_single_threaded/output.wav" + sh "diff output.wav ../pipeline_single_threaded/output.wav" } } } dir("${REPO}/examples/bare-metal/pipeline_alt_arch") { viewEnv() { withVenv { - //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_alt_arch/bin/avona_example_bare_metal_pipeline_alt_arch_st.xe --input ../shared_src/test_streams/pipeline_example_input.wav" - //sh "mv output.wav output_st.wav" + sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_alt_arch/bin/avona_example_bare_metal_pipeline_alt_arch_st.xe --input ../shared_src/test_streams/pipeline_example_input.wav" + sh "mv output.wav output_st.wav" - //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_alt_arch/bin/avona_example_bare_metal_pipeline_alt_arch_mt.xe --input ../shared_src/test_streams/pipeline_example_input.wav" - //sh "mv output.wav output_mt.wav" - //sh "diff output_st.wav output_mt.wav" + sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/pipeline_alt_arch/bin/avona_example_bare_metal_pipeline_alt_arch_mt.xe --input ../shared_src/test_streams/pipeline_example_input.wav" + sh "mv output.wav output_mt.wav" + sh "diff output_st.wav output_mt.wav" } } } dir("${REPO}/examples/bare-metal/agc") { viewEnv() { withVenv { - //sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/agc/bin/avona_example_bare_metal_agc.xe --input ../shared_src/test_streams/agc_example_input.wav" + sh "python ../shared_src/python/run_xcoreai.py ../../../build/examples/bare-metal/agc/bin/avona_example_bare_metal_agc.xe --input ../shared_src/test_streams/agc_example_input.wav" } } } dir("${REPO}/examples/bare-metal/vnr") { viewEnv() { withVenv { - //sh "python host_app.py test_stream_1.wav vnr_out2.bin --run-with-xscope-fileio" //With xscope host in lib xscope_fileio - //sh "python host_app.py test_stream_1.wav vnr_out1.bin" //With xscope host in python - //sh "diff vnr_out1.bin vnr_out2.bin" + sh "python host_app.py test_stream_1.wav vnr_out2.bin --run-with-xscope-fileio" //With xscope host in lib xscope_fileio + sh "python host_app.py test_stream_1.wav vnr_out1.bin" //With xscope host in python + sh "diff vnr_out1.bin vnr_out2.bin" } } } @@ -333,7 +259,7 @@ pipeline { withVenv { withMounts([["projects", "projects/hydra_audio", "hydra_audio_vnr_tests"]]) { withEnv(["hydra_audio_PATH=$hydra_audio_vnr_tests_PATH"]) { - //sh "pytest -n 1 --junitxml=pytest_result.xml" + sh "pytest -n 1 --junitxml=pytest_result.xml" } } } @@ -346,7 +272,7 @@ pipeline { dir("${REPO}/test/lib_vnr/vnr_unit_tests") { viewEnv() { withVenv { - //sh "pytest -n 2 --junitxml=pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" } } } @@ -358,8 +284,8 @@ pipeline { viewEnv() { withVenv { runPython("python build_vnr_feature_extraction.py") - //sh "pytest -s --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -370,8 +296,8 @@ pipeline { dir("${REPO}/test/lib_vad/vad_unit_tests") { viewEnv() { withVenv { - //sh "pytest -n 2 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -382,8 +308,8 @@ pipeline { dir("${REPO}/test/lib_vad/compare_xc_c") { viewEnv() { withVenv { - //sh "pytest -s --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -394,11 +320,11 @@ pipeline { dir("${REPO}/test/lib_vad/test_vad_profile") { viewEnv() { withVenv { - //sh "pytest -s --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } - //archiveArtifacts artifacts: "vad_profile_report.log", fingerprint: true + archiveArtifacts artifacts: "vad_profile_report.log", fingerprint: true } } } @@ -407,8 +333,8 @@ pipeline { dir("${REPO}/test/lib_ns/test_ns_profile") { viewEnv() { withVenv { - //sh "pytest -n 1 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 1 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -420,8 +346,8 @@ pipeline { copyArtifacts filter: '**/*.xe', fingerprintArtifacts: true, projectName: '../lib_noise_suppression/develop', selector: lastSuccessful() viewEnv() { withVenv { - //sh "pytest -n 2 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -432,8 +358,8 @@ pipeline { dir("${REPO}/test/lib_ns/ns_unit_tests") { viewEnv() { withVenv { - //sh "pytest -n 1 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 1 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -451,16 +377,67 @@ pipeline { } } } + stage('IC Python C equivalence') { + steps { + dir("${REPO}/test/lib_ic/py_c_frame_compare") { + viewEnv() { + withVenv { + runPython("python build_ic_frame_proc.py") + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" + } + } + } + } + } stage('IC test profile') { steps { dir("${REPO}/test/lib_ic/test_ic_profile") { viewEnv() { withVenv { - //sh "pytest --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest --junitxml=pytest_result.xml" + junit "pytest_result.xml" + } + } + archiveArtifacts artifacts: "ic_prof.log", fingerprint: true + } + } + } + stage('IC test specification') { + steps { + dir("${REPO}/test/lib_ic/test_ic_spec") { + viewEnv() { + withVenv { + //This test compares the model and C implementation over a range of scenarious for: + //convergence_time, db_suppression, maximum noise added to input (to test for stability) + //and expected group delay. It will fail if these are not met. + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" + sh "python print_stats.py > ic_spec_summary.txt" + //This script generates a number of polar plots of attenuation vs null point angle vs freq + //It currently only uses the python model to do this. It takes about 40 mins for all plots + //and generates a series of IC_performance_xxxHz.svg files which could be archived + // sh "python plot_ic.py" + } + } + archiveArtifacts artifacts: "ic_spec_summary.txt", fingerprint: true + } + } + } + stage('IC characterisation') { + steps { + dir("${REPO}/test/lib_ic/characterise_c_py") { + viewEnv() { + withVenv { + //This test compares the suppression performance across angles between model and C implementation + //and fails if they differ significantly. It requires that the C implementation run with fixed mu + sh "pytest -s --junitxml=pytest_result.xml" //-n 2 fails often so run single threaded and also print result + junit "pytest_result.xml" + //This script sweeps the y_delay value to find what the optimum suppression is across RT60 and angle. + //It's more of a model develpment tool than testing the implementation so not run. It take a few minutes. + // sh "python sweep_ic_delay.py" } } - //archiveArtifacts artifacts: "ic_prof.log", fingerprint: true } } } @@ -471,7 +448,24 @@ pipeline { withVenv { //This is a unit test for ic_calc_vnr_pred function. //It compares the avona output with py_ic model output - //sh "pytest -n1 --junitxml=pytest_result.xml" + sh "pytest -n1 --junitxml=pytest_result.xml" + } + } + } + } + } + stage('Stage B tests') { + steps { + dir("${REPO}/test/stage_b") { + viewEnv() { + withVenv { + withMounts([["projects", "projects/hydra_audio", "hydra_audio_stage_b_tests"]]) { + withEnv(["hydra_audio_PATH=$hydra_audio_stage_b_tests_PATH"]) { + runPython("python build_c_code.py") + sh "pytest -s --junitxml=pytest_result.xml" + junit "pytest_result.xml" + } + } } } } @@ -482,8 +476,8 @@ pipeline { dir("${REPO}/test/lib_adec/de_unit_tests") { viewEnv() { withVenv { - //sh "pytest -n 2 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -498,9 +492,9 @@ pipeline { withEnv(["hydra_audio_PATH=$hydra_audio_test_de_PATH"]) { sh 'mkdir -p ./input_wavs/' sh 'mkdir -p ./output_files/' - //sh "pytest -n 2 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" - //runPython("python print_stats.py") + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" + runPython("python print_stats.py") } } } @@ -515,8 +509,8 @@ pipeline { withVenv { withMounts([["projects", "projects/hydra_audio", "hydra_audio_adec_tests"]]) { withEnv(["hydra_audio_PATH=$hydra_audio_adec_tests_PATH"]) { - //sh "pytest -n 2 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -531,8 +525,8 @@ pipeline { withVenv { withMounts([["projects", "projects/hydra_audio", "hydra_audio_adec_tests"]]) { withEnv(["hydra_audio_PATH=$hydra_audio_adec_tests_PATH"]) { - //sh "pytest -n 1 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 1 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -548,8 +542,8 @@ pipeline { withMounts([["projects", "projects/hydra_audio", "hydra_audio_test_skype"]]) { withEnv(["hydra_audio_PATH=$hydra_audio_test_skype_PATH"]) { sh "./make_dirs.sh" - //sh "pytest -n 2 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -562,8 +556,8 @@ pipeline { dir("${REPO}/test/lib_aec/aec_unit_tests") { viewEnv() { withVenv { - //sh "pytest -n 2 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -580,17 +574,17 @@ pipeline { sh 'mv excluded_tests_quick.txt excluded_tests.txt' } } - //sh "python generate_audio.py" - //sh "pytest -n 2 --junitxml=pytest_result.xml test_process_audio.py" - //sh "cp pytest_result.xml results_process.xml" + sh "python generate_audio.py" + sh "pytest -n 2 --junitxml=pytest_result.xml test_process_audio.py" + sh "cp pytest_result.xml results_process.xml" catchError { - //sh "pytest --junitxml=pytest_result.xml test_check_output.py" + sh "pytest --junitxml=pytest_result.xml test_check_output.py" } - //sh "cp pytest_result.xml results_check.xml" - //sh "python parse_results.py" - //sh "pytest --junitxml=pytest_results.xml test_evaluate_results.py" - //sh "cp pytest_result.xml results_final.xml" - //junit "results_final.xml" + sh "cp pytest_result.xml results_check.xml" + sh "python parse_results.py" + sh "pytest --junitxml=pytest_results.xml test_evaluate_results.py" + sh "cp pytest_result.xml results_final.xml" + junit "results_final.xml" } } } @@ -601,8 +595,8 @@ pipeline { dir("${REPO}/test/lib_agc/test_process_frame") { viewEnv() { withVenv { - //sh "pytest -n 2 --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -613,8 +607,8 @@ pipeline { dir("${REPO}/test/test_hpf") { viewEnv() { withVenv { - //sh "pytest --junitxml=pytest_result.xml" - //junit "pytest_result.xml" + sh "pytest --junitxml=pytest_result.xml" + junit "pytest_result.xml" } } } @@ -629,9 +623,9 @@ pipeline { withVenv { echo "PIPELINE_FULL_RUN set as " + env.PIPELINE_FULL_RUN //Note we have 2 xcore targets and we can run x86 threads too. But in case we have only xcore jobs in the config, limit to 4 so we don't timeout waiting for xtags - //sh "pytest -n 4 --junitxml=pytest_result.xml -vv" + sh "pytest -n 4 --junitxml=pytest_result.xml -vv" // sh "pytest -s --junitxml=pytest_result.xml" //Debug, run single threaded with STDIO captured - //junit "pytest_result.xml" + junit "pytest_result.xml" // Archive below (always section) even if fails } } @@ -644,19 +638,17 @@ pipeline { post { always { //AEC aretfacts - //archiveArtifacts artifacts: "${REPO}/test/lib_adec/test_adec_profile/**/adec_prof*.log", fingerprint: true + archiveArtifacts artifacts: "${REPO}/test/lib_adec/test_adec_profile/**/adec_prof*.log", fingerprint: true //NS artefacts - //archiveArtifacts artifacts: "${REPO}/test/lib_ns/test_ns_profile/ns_prof.log", fingerprint: true - //IC artefacts - archiveArtifacts artifacts: "${REPO}/test/lib_ic/characterise_c_py/pytest_audio/*.wav", fingerprint: true + archiveArtifacts artifacts: "${REPO}/test/lib_ns/test_ns_profile/ns_prof.log", fingerprint: true //VNR artifacts - //archiveArtifacts artifacts: "${REPO}/test/lib_vnr/test_wav_vnr/*.png", fingerprint: true - //archiveArtifacts artifacts: "${REPO}/test/lib_vnr/test_wav_vnr/*.csv", fingerprint: true - //archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/*.png", fingerprint: true - //archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/vnr_prof.log", fingerprint: true + archiveArtifacts artifacts: "${REPO}/test/lib_vnr/test_wav_vnr/*.png", fingerprint: true + archiveArtifacts artifacts: "${REPO}/test/lib_vnr/test_wav_vnr/*.csv", fingerprint: true + archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/*.png", fingerprint: true + archiveArtifacts artifacts: "${REPO}/examples/bare-metal/vnr/vnr_prof.log", fingerprint: true //Pipelines tests - //archiveArtifacts artifacts: "${REPO}/test/pipeline/results_*.csv", fingerprint: true - //archiveArtifacts artifacts: "${REPO}/test/pipeline/keyword_input_*/*.wav", fingerprint: true + archiveArtifacts artifacts: "${REPO}/test/pipeline/results_*.csv", fingerprint: true + archiveArtifacts artifacts: "${REPO}/test/pipeline/keyword_input_*/*.wav", fingerprint: true } cleanup { cleanWs() From 288c003b2476e946afd1d58a9d0f5b4b078a83a9 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Thu, 9 Jun 2022 16:15:50 +0100 Subject: [PATCH 89/92] small changes and cleanup --- Jenkinsfile | 2 +- modules/lib_aec/src/aec_impl.c | 1 - modules/lib_ic/src/ic_low_level.c | 4 +- .../py_c_frame_compare/test_ic_frame.py | 1 - .../config/two_mic_stereo.json | 15 -- .../test_calc_vnr_pred/test_calc_vnr_pred.py | 4 +- test/shared/config/two_mic_stereo.json | 191 ----------------- .../config/two_mic_stereo_prev_arch.json | 192 ------------------ 8 files changed, 5 insertions(+), 405 deletions(-) delete mode 100644 test/lib_ic/test_calc_vnr_pred/config/two_mic_stereo.json delete mode 100644 test/shared/config/two_mic_stereo.json delete mode 100644 test/shared/config/two_mic_stereo_prev_arch.json diff --git a/Jenkinsfile b/Jenkinsfile index 84d2bd88a..5b752a81d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -411,7 +411,7 @@ pipeline { //This test compares the model and C implementation over a range of scenarious for: //convergence_time, db_suppression, maximum noise added to input (to test for stability) //and expected group delay. It will fail if these are not met. - sh "pytest -s --junitxml=pytest_result.xml" + sh "pytest -n 2 --junitxml=pytest_result.xml" junit "pytest_result.xml" sh "python print_stats.py > ic_spec_summary.txt" //This script generates a number of polar plots of attenuation vs null point angle vs freq diff --git a/modules/lib_aec/src/aec_impl.c b/modules/lib_aec/src/aec_impl.c index 7a3c172c3..3346d1a9d 100644 --- a/modules/lib_aec/src/aec_impl.c +++ b/modules/lib_aec/src/aec_impl.c @@ -140,7 +140,6 @@ void aec_forward_fft( temp->hr = bfp_complex_s32_headroom(temp); // TODO Workaround till https://github.com/xmos/lib_xs3_math/issues/96 is fixed memcpy(output, temp, sizeof(bfp_complex_s32_t)); - output->hr = bfp_complex_s32_headroom(output); bfp_fft_unpack_mono(output); input->length = len; } diff --git a/modules/lib_ic/src/ic_low_level.c b/modules/lib_ic/src/ic_low_level.c index b71d9b3b4..970fd75d8 100644 --- a/modules/lib_ic/src/ic_low_level.c +++ b/modules/lib_ic/src/ic_low_level.c @@ -309,7 +309,7 @@ void ic_adaption_controller(ic_state_t *state, uint8_t vad){ //noise_mu = 1.0 - self.smoothed_voice_chance float_s32_t noise_mu = float_s32_sub(one, ad_state->smoothed_voice_chance); - //noise_mu = noise_mu * min(1.0, np.sqrt(np.sqrt(self.output_energy/(self.input_energy + 0.000000001)))) + //noise_mu = noise_mu * min(1.0, np.sqrt(np.sqrt(self.output_energy/(self.input_energy + delta)))) float_s32_t input_plus_delta = float_s32_add_fix(ad_state->input_energy_slow, delta); float_s32_t ratio = float_s32_div(ad_state->output_energy_slow, input_plus_delta); ratio = float_s32_sqrt(ratio); @@ -326,7 +326,7 @@ void ic_adaption_controller(ic_state_t *state, uint8_t vad){ } } - //fast_ratio = self.output_energy0 / (self.input_energy0 + 0.000000001) + //fast_ratio = self.output_energy0 / (self.input_energy0 + delta) input_plus_delta = float_s32_add_fix(ad_state->input_energy_fast, delta); float_s32_t fast_ratio = float_s32_div(ad_state->output_energy_fast, input_plus_delta); diff --git a/test/lib_ic/py_c_frame_compare/test_ic_frame.py b/test/lib_ic/py_c_frame_compare/test_ic_frame.py index 45afc13e0..fb563e926 100644 --- a/test/lib_ic/py_c_frame_compare/test_ic_frame.py +++ b/test/lib_ic/py_c_frame_compare/test_ic_frame.py @@ -28,7 +28,6 @@ fd_length = proc_frame_length // 2 + 1 frame_advance = 240 num_phases = 10 -#input_file = 'pytest_audio_360.wav' input_file = "../../../examples/bare-metal/ic/input.wav" output_file = "output.wav" ap_config_file = "../../shared/config/ic_conf_no_adapt_control.json" diff --git a/test/lib_ic/test_calc_vnr_pred/config/two_mic_stereo.json b/test/lib_ic/test_calc_vnr_pred/config/two_mic_stereo.json deleted file mode 100644 index 020c3f45e..000000000 --- a/test/lib_ic/test_calc_vnr_pred/config/two_mic_stereo.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "frame_advance": 240, - "proc_frame_length": 512, - "phases": 10, - "mu" : 1.0, - "delta" : 7.450580593454381e-09, - "y_channel_delay": 180, - "K" : 1, - "lamda" : 0.9995117188, - "gamma" : 2.0, - "leakage" : 0.995, - "remove_NQ" : true, - "vnr_model" : "specified_in_test", - "adaption_config": "IC_ADAPTION_AUTO" -} diff --git a/test/lib_ic/test_calc_vnr_pred/test_calc_vnr_pred.py b/test/lib_ic/test_calc_vnr_pred/test_calc_vnr_pred.py index 4433439d1..60b5acde1 100644 --- a/test/lib_ic/test_calc_vnr_pred/test_calc_vnr_pred.py +++ b/test/lib_ic/test_calc_vnr_pred/test_calc_vnr_pred.py @@ -5,7 +5,6 @@ this_file_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(this_file_dir, "../../lib_vnr/vnr_unit_tests/feature_extraction")) -sys.path.append(os.path.join(this_file_dir, "../../../../py_ic/py_ic")) # TODO Remove when we're able to add py_ic to requirements.txt. Will happen once lib_interference_canceller/python is no longer used in avona. import test_utils # Use vnr test's test_utils import IC @@ -15,7 +14,8 @@ def test_calc_vnr_pred(target, tflite_model, show_plot=False): np.random.seed(12345) - ic_parameters = json_to_dict(os.path.join(this_file_dir, "config/two_mic_stereo.json")) + ic_parameters = json_to_dict("../../shared/config/ic_conf_no_adapt_control.json") + ic_parameters["adaption_config"] = "IC_ADAPTION_AUTO" ic_parameters["vnr_model"] = str(tflite_model) ifc = IC.adaptive_interference_canceller(**ic_parameters) diff --git a/test/shared/config/two_mic_stereo.json b/test/shared/config/two_mic_stereo.json deleted file mode 100644 index 4637e0f90..000000000 --- a/test/shared/config/two_mic_stereo.json +++ /dev/null @@ -1,191 +0,0 @@ -{ - "x_channel_count": 2, - "y_channel_count": 2, - "frame_advance": 240, - "mic_shift": 0, - "mic_saturate": 0, - "alt_arch": { - "enable": 1, - "aec_mode_hold_s": 3, - "ref_activity_threshold_dB": -60 - }, - "ap_stage_a_conf": { - "x_channel_count": 2, - "y_channel_count": 2, - "adec_enabled": false, // Possible values: true, false, "on_startup" - "mic_shift": 0, - "mic_saturate": 0, - // Note that these AEC parameters are not used when generating XC AEC params/schedule script, only the python currently - "mic_filter": { - "bypass": 1, - "num_stage": 2, - "coeff": { - "1": { - "a" : [0, 0], - "b" : [1, 0, 0] - }, - "2": { - "a" : [0, 0], - "b" : [1, 0, 0] - } - } - }, - "ref_filter": { - "bypass": 1, - "num_stage": 2, - "coeff": { - "1": { - "a" : [0, 0], - "b" : [1, 0, 0] - }, - "2": { - "a" : [0, 0], - "b" : [1, 0, 0] - } - } - }, - "aec_conf": { - "x_channel_count": 2, - "y_channel_count": 1, - "output_channel_count": 2, - "proc_frame_length": 512, - "frame_advance": 240, - "phases": 15, - "phases_shadow": 5, - "K": 1, - "adaption_config": "AEC_ADAPTION_AUTO", - "force_adaption_mu": 1.0, - "mu_scalar": 1.0, - "shadow_mu": 1.0, - "sigma_xx_shift": 6, - "gamma_log2": 5, - "delta_exp": -38, // -32 -6 - "leakage": 1.0, - "freq_smooth": true - }, - "de_conf": { - "x_channel_count": 1, - "y_channel_count": 1, - "output_channel_count": 1, - "proc_frame_length": 512, - "frame_advance": 240, - "phases": 30, - "phases_shadow": 0, - "K": 1, - "adaption_config": "AEC_ADAPTION_FORCE_ON", - "force_adaption_mu": 1.0, - "mu_scalar": 1.0, - "shadow_mu": 0.0, - "sigma_xx_shift": 6, - "gamma_log2": 5, - "delta_exp": -38, // -32 -6 - "leakage": 1.0, - "freq_smooth": true - } - }, - "ap_stage_b_conf": { - "frame_advance": 240, - "passthrough_channel_count": 0, - "mic_shift": 0, - "mic_saturate": 0, - "ic_conf": { - "frame_advance": 240, - "proc_frame_length": 512, - "phases": 10, - "passthrough_channel_count": 0, - "mu" : 1.0, - "delta" : 7.450580593454381e-09, // np.iinfo(np.int32).max * 2.**(-32 - 26) - "delay" : 0, - "K" : 1, - "lamda" : 0.99951171875, // 1.0 - (0.5**11) - "gamma" : 2.0, - "leakage" : 0.995, - "remove_NQ" : true, - "use_noise_minimisation" : true - } - }, - "ap_stage_c_conf": { - "x_channel_count": 0, - "y_channel_count": 2, - "out_channels": 2, - "mic_shift": 0, - "mic_saturate": 0, - "comms_filter": { - "bypass": 0, - "num_stage": 2, - "coeff": { - "1": { - "a" : [-1.92850849, 0.92999644], - "b" : [0.94998178, -1.89996357, 0.94998178] - }, - "2": { - "a" : [-1.9688775, 0.9703966], - "b" : [1, -2, 1] - } - } - }, - "suppression_conf": { - "x_channel_count": 0, - "y_channel_count": 2, - "frame_advance": 240, - "proc_frame_length": 512, - "rate": 16000 - }, - "agc_init_config": { - "ch_init_config": [ - { - "gain_dec": 0.87, - "max_gain": 1000, - "min_gain": 0, - "gain_inc": 1.197, - "adapt": 1, - "adapt_on_vad": 1, - "soft_clipping": 1, - "init_gain": 500, - "upper_threshold": 0.9999, - "lower_threshold": 0.7000, - "lc_enabled": 0, - "lc_n_frame_near": 0, - "lc_n_frame_far": 0, - "lc_corr_threshold": 0, - "lc_gamma_inc":0, - "lc_gamma_dec":0, - "lc_bg_power_gamma":0, - "lc_near_delta_far_act": 0, - "lc_near_delta": 0, - "lc_far_delta": 0, - "lc_gain_max": 0, - "lc_gain_dt": 0, - "lc_gain_silence": 0, - "lc_gain_min": 0 - }, - { - "gain_dec": 0.98804, - "max_gain": 1000, - "min_gain": 0, - "gain_inc": 1.0034, - "adapt": 1, - "adapt_on_vad": 1, - "soft_clipping": 1, - "init_gain": 500, - "upper_threshold": 0.4, - "lower_threshold": 0.4, - "lc_enabled": 1, - "lc_n_frame_near": 34, - "lc_n_frame_far": 17, - "lc_corr_threshold": 0.993, - "lc_gamma_inc": 1.005, - "lc_gamma_dec": 0.995, - "lc_bg_power_gamma": 1.002, - "lc_near_delta_far_act": 100, - "lc_near_delta": 50, - "lc_far_delta": 300, - "lc_gain_max": 1, - "lc_gain_dt": 0.9, - "lc_gain_silence": 0.1, - "lc_gain_min": 0.022387 - } - ] - } - } -} diff --git a/test/shared/config/two_mic_stereo_prev_arch.json b/test/shared/config/two_mic_stereo_prev_arch.json deleted file mode 100644 index cb1207e18..000000000 --- a/test/shared/config/two_mic_stereo_prev_arch.json +++ /dev/null @@ -1,192 +0,0 @@ -{ - "x_channel_count": 2, - "y_channel_count": 2, - "frame_advance": 240, - "mic_shift": 0, - "mic_saturate": 0, - "alt_arch": { - "enable": 0, - "aec_mode_hold_s": 0, - "ref_activity_threshold_dB": -60 - }, - "ap_stage_a_conf": { - "x_channel_count": 2, - "y_channel_count": 2, - "adec_enabled": false, // Possible values: true, false, "on_startup" - "mic_shift": 0, - "mic_saturate": 0, - // Note that these AEC parameters are not used when generating XC AEC params/schedule script, only the python currently - "mic_filter": { - "bypass": 1, - "num_stage": 2, - "coeff": { - "1": { - "a" : [0, 0], - "b" : [1, 0, 0] - }, - "2": { - "a" : [0, 0], - "b" : [1, 0, 0] - } - } - }, - "ref_filter": { - "bypass": 1, - "num_stage": 2, - "coeff": { - "1": { - "a" : [0, 0], - "b" : [1, 0, 0] - }, - "2": { - "a" : [0, 0], - "b" : [1, 0, 0] - } - } - }, - "aec_conf": { - "x_channel_count": 2, - "y_channel_count": 2, - "output_channel_count": 2, - "proc_frame_length": 512, - "frame_advance": 240, - "phases": 10, - "phases_shadow": 5, - "K": 1, - "adaption_config": "AEC_ADAPTION_FORCE_ON", - "force_adaption_mu": 1.0, - "mu_scalar": 2.0, - "shadow_mu": 1.0, - "sigma_xx_shift": 11, - "gamma_log2": 6, - "delta_exp": -38, // -32 -6 - "leakage": 1.0, - "freq_smooth": true - }, - "de_conf": { - "x_channel_count": 1, - "y_channel_count": 1, - "output_channel_count": 1, - "proc_frame_length": 512, - "frame_advance": 240, - "phases": 30, - "phases_shadow": 0, - "K": 1, - "adaption_config": "AEC_ADAPTION_AUTO", - "force_adaption_mu": 1.0, - "mu_scalar": 1.0, - "shadow_mu": 1.0, - "sigma_xx_shift": 11, - "gamma_log2": 6, - "delta_exp": -38, // -32 -6 - "leakage": 1.0, - "freq_smooth": true - } - }, - "ap_stage_b_conf": { - "frame_advance": 240, - "passthrough_channel_count": 0, - "mic_shift": 0, - "mic_saturate": 0, - "ic_conf": { - "frame_advance": 240, - "proc_frame_length": 512, - "phases": 10, - "passthrough_channel_count": 0, - "mu" : 1.0, - "delta" : 7.450580593454381e-09, // np.iinfo(np.int32).max * 2.**(-32 - 26) - "delay" : 0, - "K" : 1, - "lamda" : 0.99951171875, // 1.0 - (0.5**11) - "gamma" : 2.0, - "leakage" : 0.995, - "remove_NQ" : true, - "use_noise_minimisation" : true - } - }, - "ap_stage_c_conf": { - "x_channel_count": 0, - "y_channel_count": 2, - "out_channels": 2, - "mic_shift": 0, - "mic_saturate": 0, - "comms_filter": { - "bypass": 0, - "num_stage": 2, - "coeff": { - "1": { - "a" : [-1.92850849, 0.92999644], - "b" : [0.94998178, -1.89996357, 0.94998178] - }, - "2": { - "a" : [-1.9688775, 0.9703966], - "b" : [1, -2, 1] - } - } - }, - "suppression_conf": { - "x_channel_count": 0, - "y_channel_count": 2, - "frame_advance": 240, - "proc_frame_length": 512, - "rate": 16000 - }, - "agc_init_config": { - "ch_init_config": [ - { - "gain_dec": 0.87, - "max_gain": 1000, - "min_gain": 0, - "gain_inc": 1.197, - "adapt": 1, - "adapt_on_vad": 1, - "soft_clipping": 1, - "init_gain": 500, - "upper_threshold": 0.9999, - "lower_threshold": 0.7000, - "lc_enabled": 0, - "lc_n_frame_near": 0, - "lc_n_frame_far": 0, - "lc_corr_threshold": 0, - "lc_gamma_inc":0, - "lc_gamma_dec":0, - "lc_bg_power_gamma":0, - "lc_near_delta_far_act": 0, - "lc_near_delta": 0, - "lc_far_delta": 0, - "lc_gain_max": 0, - "lc_gain_dt": 0, - "lc_gain_silence": 0, - "lc_gain_min": 0 - - }, - { - "gain_dec": 0.98804, - "max_gain": 1000, - "min_gain": 0, - "gain_inc": 1.0034, - "adapt": 1, - "adapt_on_vad": 1, - "soft_clipping": 1, - "init_gain": 500, - "upper_threshold": 0.4, - "lower_threshold": 0.4, - "lc_enabled": 1, - "lc_n_frame_near": 34, - "lc_n_frame_far": 17, - "lc_corr_threshold": 0.993, - "lc_gamma_inc": 1.005, - "lc_gamma_dec": 0.995, - "lc_bg_power_gamma": 1.002, - "lc_near_delta_far_act": 100, - "lc_near_delta": 50, - "lc_far_delta": 300, - "lc_gain_max": 1, - "lc_gain_dt": 0.9, - "lc_gain_silence": 0.1, - "lc_gain_min": 0.022387 - } - ] - } - } -} From 4148ff6bb84c124bd43f0b753c111759e23fbfbd Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Fri, 10 Jun 2022 11:46:37 +0100 Subject: [PATCH 90/92] working on Shuchita's comments --- modules/lib_ic/api/ic_state.h | 2 +- modules/lib_ic/src/ic.c | 4 ++-- modules/lib_ic/src/ic_low_level.c | 2 ++ test/lib_ic/py_c_frame_compare/test_ic_frame.py | 7 ++++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/lib_ic/api/ic_state.h b/modules/lib_ic/api/ic_state.h index af5632bcc..fc643a95e 100644 --- a/modules/lib_ic/api/ic_state.h +++ b/modules/lib_ic/api/ic_state.h @@ -177,7 +177,7 @@ typedef struct { bfp_s32_t error_bfp[IC_Y_CHANNELS]; /** Storage for Error and error mantissas. Note IFFT is done in-place * so the Error storage is reused for error. */ - int32_t DWORD_ALIGNED Error[IC_Y_CHANNELS][IC_FRAME_LENGTH + FFT_PADDING]; + complex_s32_t DWORD_ALIGNED Error[IC_Y_CHANNELS][IC_FD_FRAME_LENGTH]; /** BFP array pointing to the frequency domain estimate of transfer function. */ bfp_complex_s32_t H_hat_bfp[IC_Y_CHANNELS][IC_X_CHANNELS*IC_FILTER_PHASES]; diff --git a/modules/lib_ic/src/ic.c b/modules/lib_ic/src/ic.c index 48f3339b2..7a8a6a4ca 100644 --- a/modules/lib_ic/src/ic.c +++ b/modules/lib_ic/src/ic.c @@ -69,8 +69,8 @@ void ic_init(ic_state_t *state){ } //initialise Error for(unsigned ch=0; chError_bfp[ch], (complex_s32_t * )state->Error[ch], zero_exp, IC_FD_FRAME_LENGTH, 0); - bfp_s32_init(&state->error_bfp[ch], state->Error[ch], zero_exp, IC_FRAME_LENGTH, 0); + bfp_complex_s32_init(&state->Error_bfp[ch], state->Error[ch], zero_exp, IC_FD_FRAME_LENGTH, 0); + bfp_s32_init(&state->error_bfp[ch], (int32_t *)state->Error[ch], zero_exp, IC_FRAME_LENGTH, 0); } //Initiaise Y_hat for(unsigned ch=0; ch Date: Fri, 10 Jun 2022 16:42:09 +0100 Subject: [PATCH 91/92] bringing old iew file --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5b752a81d..cbaf5db32 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { } environment { REPO = 'sw_avona' - VIEW = getViewName(REPO) + VIEW = 'sw_avona_ic_test.txt' FULL_TEST = """${(params.FULL_TEST_OVERRIDE || env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'main' From 3a86fcb8fb07ef9234dc198ad87b017d0e1f17c2 Mon Sep 17 00:00:00 2001 From: uvvpavel Date: Mon, 13 Jun 2022 15:29:16 +0100 Subject: [PATCH 92/92] switching to develop view file --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index cbaf5db32..5b752a81d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { } environment { REPO = 'sw_avona' - VIEW = 'sw_avona_ic_test.txt' + VIEW = getViewName(REPO) FULL_TEST = """${(params.FULL_TEST_OVERRIDE || env.BRANCH_NAME == 'develop' || env.BRANCH_NAME == 'main'