Skip to content

Commit

Permalink
#384 add ability to cap off parallel hook processes
Browse files Browse the repository at this point in the history
  • Loading branch information
@pliablepixels
pliablepixels committed Apr 10, 2021
1 parent 5d183ae commit 2fc4e79
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 5 deletions.
17 changes: 17 additions & 0 deletions zmeventnotification.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,23 @@ use_hooks = yes

# NOTE: This entire section is only valid if use_hooks is yes above

# When a hook is invoked, the ES forks a child. If you are in a situation
# where your motion sensititivy in ZM is not set properly, you may land up
# triggering hundreds of child processes of zm_detect that may potentially
# crash your system. Note that there are global locks around the ML code which
# are controlled by xxx_max_processes in the objectconfig/mlapiconfig.files
# which will avoid parallel running of models. But this is if you are facing issues
# by the simple fact that too many zm_detect processes are forked (which will apply
# whether you use mlapi or not). While I do feel the core issue is that you need
# to fix your ZM sensitivity, this parameter helps control.

# NOTE: When you put in value for this, any hooks that attempt to kick off
# beyond this limit will simply be ignored. There is no queueing.

# A value of 0 (default) means there are no limits
max_parallel_hooks=0


# Shell script name here to be called every time an alarm is detected
# the script will get passed $1=alarmEventID, $2=alarmMonitorId
# $3 monitor Name, $4 alarm cause
Expand Down
44 changes: 39 additions & 5 deletions zmeventnotification.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,9 @@
'/var/lib/zmeventnotification/misc/escontrol_interface.dat',
DEFAULT_FCM_DATE_FORMAT => '%I:%M %p, %d-%b',
DEFAULT_FCM_ANDROID_PRIORITY=>'high',
DEFAULT_MAX_FCM_PER_MONTH_PER_TOKEN => 8000
DEFAULT_MAX_FCM_PER_MONTH_PER_TOKEN => 8000,

DEFAULT_MAX_PARALLEL_HOOKS => 0,
};

# connection state
Expand Down Expand Up @@ -173,6 +175,7 @@
my $es_terminate = 0;

my $child_forks = 0; # Global tracker of active children
my $parallel_hooks = 0; # Global tracker for active hooks
my $total_forks = 0; # Global tracker of all forks since start

# Declare options.
Expand Down Expand Up @@ -279,6 +282,8 @@

my %fcm_tokens_map;

my $max_parallel_hooks= 0;

my %monitors = ();
my %active_events = ();
my $monitor_reload_time = 0;
Expand Down Expand Up @@ -696,6 +701,12 @@ sub loadEsConfigSettings {
DEFAULT_EVENT_END_NOTIFY_ON_HOOK_SUCCESS
);

$max_parallel_hooks = config_get_val(
$config, 'hook',
'max_parallel_hooks',
DEFAULT_MAX_PARALLEL_HOOKS
);

# get channels and convert to hash

%event_start_notify_on_hook_fail = map { $_ => 1 }
Expand Down Expand Up @@ -807,10 +818,11 @@ sub print_config {
Monitor rules JSON file................${\(value_or_undefined($es_rules_file))}
Use Hooks............................. ${\(yes_or_no($use_hooks))}
Max Parallel Hooks.................... ${\(value_or_undefined($max_parallel_hooks))}
Hook Script on Event Start ........... ${\(value_or_undefined($event_start_hook))}
User Script on Event Start.............${\(value_or_undefined($event_start_hook_notify_userscript))}
Hook Script on Event End.............. ${\(value_or_undefined($event_end_hook))}
User Script on Event End.............${\(value_or_undefined($event_end_hook_notify_userscript))}
User Script on Event End...............${\(value_or_undefined($event_end_hook_notify_userscript))}
Hook Skipped monitors................. ${\(value_or_undefined($hook_skip_monitors))}
Notify on Event Start (hook success).. ${\(value_or_undefined($event_start_notify_on_hook_success))}
Expand Down Expand Up @@ -2374,6 +2386,17 @@ sub processJobs {
delete( $active_events{$mid}->{$eid} );
$child_forks--;
}
elsif ( $job eq 'update_parallel_hooks' ) {
if ($msg eq "add") {
$parallel_hooks++;
}
elsif ($msg eq "del") {
$parallel_hooks--;
}
else {
printError ("Parallel hooks update: command not understood: $msg");
}
}
elsif ( $job eq 'mqtt_publish' ) {
my ( $id, $topic, $payload ) = split( '--SPLIT--', $msg );
printDebug( "Job: MQTT Publish on topic: $topic", 2 );
Expand Down Expand Up @@ -3861,7 +3884,10 @@ sub processNewAlarmsInFork {
if ( $cmd =~ /^(.*)$/ ) {
$cmd = $1;
}
print WRITER "update_parallel_hooks--TYPE--add\n";
my $res = `$cmd`;
print WRITER "update_parallel_hooks--TYPE--del\n";

chomp($res);
my ( $resTxt, $resJsonString ) = parseDetectResults($res);
$hookResult = $? >> 8;
Expand Down Expand Up @@ -4093,7 +4119,11 @@ sub processNewAlarmsInFork {
if ( $cmd =~ /^(.*)$/ ) {
$cmd = $1;
}

print WRITER "update_parallel_hooks--TYPE--add\n";
my $res = `$cmd`;
print WRITER "update_parallel_hooks--TYPE--del\n";

chomp($res);
my ( $resTxt, $resJsonString ) = parseDetectResults($res);
$hookResult = $? >> 8;
Expand Down Expand Up @@ -4420,7 +4450,7 @@ sub initSocketServer {
exit(0);
}
my $elapsed_time_min = ceil((time() - $es_start_time)/60);
printDebug( "----------> Tick START (active forks:$child_forks, total forks:$total_forks, running for:$elapsed_time_min min)<--------------", 2 );
printDebug( "----------> Tick START (active forks:$child_forks, total forks:$total_forks, active hooks: $parallel_hooks running for:$elapsed_time_min min)<--------------", 2 );
if ( $restart_interval
&& ( ( time() - $es_start_time ) > $restart_interval ) )
{
Expand All @@ -4446,7 +4476,7 @@ sub initSocketServer {
checkConnection();
processJobs();

printDebug( "There are $child_forks active child forks...", 2 );
printDebug( "There are $child_forks active child forks & $parallel_hooks zm_detect processes running...", 2 );
my (@newEvents) = checkNewEvents();

#print Dumper(\@newEvents);
Expand All @@ -4468,6 +4498,10 @@ sub initSocketServer {
}

foreach (@newEvents) {
if (($parallel_hooks >= $max_parallel_hooks) && ($max_parallel_hooks != 0)) {
printError("There are $parallel_hooks hooks running as of now. This exceeds your set limit of max_parallel_hooks=$max_parallel_hooks. Ignoring this event. Either increase your max_parallel_hooks value, or, adjust your ZM motion sensitivity ");
return;
}
$child_forks++;
$total_forks++;
if ($cpid = fork() ) {
Expand Down Expand Up @@ -4523,7 +4557,7 @@ sub initSocketServer {
}

check_for_duplicate_token();
printDebug( "---------->Tick END (active forks:$child_forks, total forks:$total_forks)<--------------", 2 );
printDebug( "---------->Tick END (active forks:$child_forks, total forks:$total_forks, active hooks: $parallel_hooks)<--------------", 2 );
},

# called when a new connection comes in
Expand Down

0 comments on commit 2fc4e79

Please sign in to comment.