From ef9a5ac3e7ba71f984e659aded96f919ea62c7c4 Mon Sep 17 00:00:00 2001
From: Samuel Carroll <sam2@blackhillsinfosec.com>
Date: Wed, 11 Aug 2021 11:21:26 -0600
Subject: [PATCH 1/8] Basic push, still need to add the send all logs
 functionality and make sure we create the yaml file on destination during the
 connect sensor install

---
 connect_sensor.sh     |  3 +++
 zeek_log_transport.sh | 49 ++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 51 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 connect_sensor.sh

diff --git a/connect_sensor.sh b/connect_sensor.sh
old mode 100644
new mode 100755
index c473dec..0b474b4
--- a/connect_sensor.sh
+++ b/connect_sensor.sh
@@ -196,6 +196,9 @@
 		sudo chmod 755 /usr/local/bin/zeek_log_transport.sh
 	fi
 
+#======== Download the default zeek-log-transport.yaml configuration file and send to the destination system
+	##TODO write this out!!!
+
 #======== Test that we can ssh to $1 (note that the user may need to accept ssh host key and explain how to confirm it)
 	echo2 "Confirming that we can ssh to $aih_location using the ssh authentication key.  You may be prompted to accept the ssh host key."
 	if can_ssh "$aih_location" "-o" 'PasswordAuthentication=no' -i "$HOME/.ssh/id_rsa_dataimport" ; then
diff --git a/zeek_log_transport.sh b/zeek_log_transport.sh
index 4769220..2008434 100755
--- a/zeek_log_transport.sh
+++ b/zeek_log_transport.sh
@@ -17,6 +17,46 @@ export PATH="/sbin:/usr/sbin:$PATH"		#Note that cron does _NOT_ include /sbin in
 
 default_user_on_aihunter='dataimport'
 
+get_send_logs() {
+        ##These names were found at https://docs.zeek.org/en/master/script-reference/log-files.html
+        declare -a zeek_logs=("conn" "dce_rpc" "dhcp" "dnp3" "dns" "ftp" "http" "irc" "kerberos" "modbus" "modbus_register_change" "mysql" "ntlm" "ntp" "radius" "rdp" "rfb" "sip" "smb_cmd" "smb_files" "smb_mapping" "smtp" "snmp" "socks" "ssh" "ssl" "syslog" "tunnel" "files" "ocsp" "pe" "x509" "netcontrol" "netcontrol_drop" "netcontrol_shunt" "netcontrol_catch_release" "openflow" "intel" "notice" "notice_alarm" "signatures" "traceroute" "known_certs" "known_hosts" "known_modbus" "known_services" "software" "barnyard2" "dpd" "unified2" "unknown_protocols" "weird" "weird_stats" "broker" "capture_loss" "cluster" "config" "loaded_scripts" "packet_filter" "print" "prof" "reporter" "stats" "stderr" "stdout"  "conn-summary")
+        send_logs=()
+
+	destination=$1
+	dest_log_trans=$2
+	local_dest_dir=$3
+
+        `rsync $destination:$dest_log_trans $local_dest_dir`
+
+        ##Parse file for logs
+        request_logs=`cat $local_dest_dir | grep -v :`
+
+	##TODO check if the request logs is set to all if so we should set the send_logs to the zeek_logs variable
+        #if
+
+        ##Ensure we only send the zeek logs
+        for log in $request_logs
+        do
+                if [[ " ${zeek_logs[*]} " == *"${log}"* ]]; then
+                        send_logs+=($log)
+                #else
+                #       echo $log will not be transported
+                fi
+        done
+
+        echo "${send_logs[*]}"
+}
+
+
+get_send_days() {
+	local_yaml_file=$1
+
+	days_back=`cat $local_yaml_file | grep "Days:" | cut -d ':' -f2`
+	rm $local_yaml_file
+
+	echo $days_back
+}
+
 
 can_ssh () {
 	#Test that we can reach the target system over ssh.
@@ -237,8 +277,15 @@ status "Sending logs to rita/aihunter server $aih_location , My name: $my_id , l
 status "Preparing remote directories"
 ssh $extra_ssh_params "$aih_location" "mkdir -p ${remote_top_dir}/$today/ ${remote_top_dir}/$yesterday/ ${remote_top_dir}/$twoda/ ${remote_top_dir}/$threeda/ ${remote_top_dir}/current/"
 
+recv_loc="`pwd`/zeek-log-transport.yaml"
+request_logs=`get_send_logs $aih_location "/etc/AI-Hunter/zeek-log-transport.yaml" "$recv_loc"`
+request_days=`get_send_days $recv_loc`
+logs_str=`echo "${request_logs[*]// /|}"`
+
 cd "$local_tld" || fail "Unable to change to $local_tld"
-send_candidates=`find . -type f -mtime -3 -iname '*.gz' | egrep '(conn|dns|http|ssl|x509|known_certs)' | sort -u`
+
+query="find . -type f -mtime -$request_days -iname '*.gz' | egrep '($logs_str)' | sort -u"
+send_candidates=$(eval "$query")
 if  [ ${#send_candidates} -eq 0 ]; then
 	echo
 	printf "WARNING: No logs found, if your log directory is not $local_tld please use the flag: --localdir [bro_zeek_log_directory]"

From df365c63be5f1fb7c1d54a58872ea5a42a4e9fd5 Mon Sep 17 00:00:00 2001
From: Samuel Carroll <sam2@blackhillsinfosec.com>
Date: Wed, 11 Aug 2021 14:36:40 -0600
Subject: [PATCH 2/8] Pushing the default yaml file for transfer, should be
 needed for testing WARNING: the changes to connect_sensor.sh have not been
 tested and is likely unstable.

---
 connect_sensor.sh           |  6 ++++
 zeek-transport-default.yaml | 72 +++++++++++++++++++++++++++++++++++++
 zeek_log_transport.sh       | 28 ++++++++-------
 3 files changed, 93 insertions(+), 13 deletions(-)
 create mode 100644 zeek-transport-default.yaml

diff --git a/connect_sensor.sh b/connect_sensor.sh
index 0b474b4..baf6f1d 100755
--- a/connect_sensor.sh
+++ b/connect_sensor.sh
@@ -198,6 +198,12 @@
 
 #======== Download the default zeek-log-transport.yaml configuration file and send to the destination system
 	##TODO write this out!!!
+#	if zeek-transport-default.yaml doesn't exist download it form https://raw.githubusercontent.com/activecm/zeek-log-transport/master/zeek-transport-default.yaml
+        if [ ! -s ./zeek-transport-default.yaml ]; then
+                status "Downloading zeek-transport-default.yaml"
+		curl -fsSL https://raw.githubusercontent.com/activecm/zeek-log-transport/master/zeek-transport-default.yaml
+        fi
+
 
 #======== Test that we can ssh to $1 (note that the user may need to accept ssh host key and explain how to confirm it)
 	echo2 "Confirming that we can ssh to $aih_location using the ssh authentication key.  You may be prompted to accept the ssh host key."
diff --git a/zeek-transport-default.yaml b/zeek-transport-default.yaml
new file mode 100644
index 0000000..d2540d8
--- /dev/null
+++ b/zeek-transport-default.yaml
@@ -0,0 +1,72 @@
+Days: 7
+Logs:
+        #These headings are for user friendliness, they do not do anything
+        Default:
+                conn
+                dns
+                http
+                ssl
+                x509
+                known_certs
+        #Troubleshooting:
+                #capture_loss
+                #notice
+                #stats                
+        #All:
+                #dce_rpc
+                #dhcp
+                #dnp3
+                #ftp
+                #irc
+                #kerberos
+                #modbus
+                #modbus_register_change
+                #mysql
+                #ntlm
+                #ntp
+                #radius
+                #rdp
+                #rfb
+                #sip
+                #smb_cmd
+                #smb_files
+                #smb_mapping
+                #smtp
+                #snmp
+                #socks
+                #ssh
+                #syslog
+                #tunnel
+                #files
+                #ocsp
+                #pe
+                #netcontrol
+                #netcontrol_drop
+                #netcontrol_shunt
+                #netcontrol_catch_release
+                #openflow
+                #intel
+                #notice_alarm
+                #signatures
+                #traceroute
+                #known_hosts
+                #known_modbus
+                #known_services
+                #software
+                #barnyard2
+                #dpd
+                #unified2
+                #unknown_protocols
+                #weird
+                #weird_stats
+                #broker
+                #cluster
+                #config
+                #loaded_scripts
+                #packet_filter
+                #print
+                #prof
+                #reporter
+                #stderr
+                #stdout
+                #conn-summary
diff --git a/zeek_log_transport.sh b/zeek_log_transport.sh
index 2008434..f7ceb19 100755
--- a/zeek_log_transport.sh
+++ b/zeek_log_transport.sh
@@ -28,21 +28,23 @@ get_send_logs() {
 
         `rsync $destination:$dest_log_trans $local_dest_dir`
 
-        ##Parse file for logs
-        request_logs=`cat $local_dest_dir | grep -v :`
+	##Parse file for logs skip headerlines (indicated by :) and commented lines (indicated by #)
+        request_logs=`cat $local_dest_dir | grep -v : | grep -v #`
 
 	##TODO check if the request logs is set to all if so we should set the send_logs to the zeek_logs variable
-        #if
-
-        ##Ensure we only send the zeek logs
-        for log in $request_logs
-        do
-                if [[ " ${zeek_logs[*]} " == *"${log}"* ]]; then
-                        send_logs+=($log)
-                #else
-                #       echo $log will not be transported
-                fi
-        done
+	if [[ " ${zeek_logs[*]} " == *"all"* ]]; then
+		send_logs+=("${zeek_logs[@]}")
+	else
+	        ##Ensure we only send the zeek logs
+        	for log in $request_logs
+	        do
+        	        if [[ " ${zeek_logs[*]} " == *"${log}"* ]]; then
+                	        send_logs+=($log)
+	                #else
+        	        #       echo $log will not be transported
+                	fi
+	        done
+	fi
 
         echo "${send_logs[*]}"
 }

From ca1206d87bc6eb7fe691e3ff9d18f8c786eebabf Mon Sep 17 00:00:00 2001
From: Samuel Carroll <sam2@blackhillsinfosec.com>
Date: Mon, 16 Aug 2021 11:06:43 -0600
Subject: [PATCH 3/8] Adding code that will get the default logs if no
 additional log information was found on the remote system.

---
 zeek_log_transport.sh | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/zeek_log_transport.sh b/zeek_log_transport.sh
index f7ceb19..11d70a3 100755
--- a/zeek_log_transport.sh
+++ b/zeek_log_transport.sh
@@ -289,9 +289,13 @@ cd "$local_tld" || fail "Unable to change to $local_tld"
 query="find . -type f -mtime -$request_days -iname '*.gz' | egrep '($logs_str)' | sort -u"
 send_candidates=$(eval "$query")
 if  [ ${#send_candidates} -eq 0 ]; then
-	echo
-	printf "WARNING: No logs found, if your log directory is not $local_tld please use the flag: --localdir [bro_zeek_log_directory]"
-	echo
+	#if we don't have anything we assume that we need to check the last 3 days for logs needed for RITA
+	send_candidates=`find . -type f -mtime -3 -iname '*.gz' | egrep '(conn|dns|http|ssl|x509|known_certs)' | sort -u`
+	if [ ${#send_candidates} -eq 0 ]; then
+		echo
+		printf "WARNING: No logs found, if your log directory is not $local_tld please use the flag: --localdir [bro_zeek_log_directory]"
+		echo
+	fi
 
 fi
 status "Transferring files to $aih_location"

From b5cff1b261ede93e63c79f2ee0658bab991c7702 Mon Sep 17 00:00:00 2001
From: Samuel Carroll <sam2@blackhillsinfosec.com>
Date: Mon, 16 Aug 2021 11:23:33 -0600
Subject: [PATCH 4/8] Removing the code to download the dault log yaml, since
 we modified the send logs script to send 3 day default if it's not on the
 remote system

---
 connect_sensor.sh | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/connect_sensor.sh b/connect_sensor.sh
index baf6f1d..c473dec 100755
--- a/connect_sensor.sh
+++ b/connect_sensor.sh
@@ -196,15 +196,6 @@
 		sudo chmod 755 /usr/local/bin/zeek_log_transport.sh
 	fi
 
-#======== Download the default zeek-log-transport.yaml configuration file and send to the destination system
-	##TODO write this out!!!
-#	if zeek-transport-default.yaml doesn't exist download it form https://raw.githubusercontent.com/activecm/zeek-log-transport/master/zeek-transport-default.yaml
-        if [ ! -s ./zeek-transport-default.yaml ]; then
-                status "Downloading zeek-transport-default.yaml"
-		curl -fsSL https://raw.githubusercontent.com/activecm/zeek-log-transport/master/zeek-transport-default.yaml
-        fi
-
-
 #======== Test that we can ssh to $1 (note that the user may need to accept ssh host key and explain how to confirm it)
 	echo2 "Confirming that we can ssh to $aih_location using the ssh authentication key.  You may be prompted to accept the ssh host key."
 	if can_ssh "$aih_location" "-o" 'PasswordAuthentication=no' -i "$HOME/.ssh/id_rsa_dataimport" ; then

From 28e0a3d6ffb3058b13406a709aac1c8ee656d1f3 Mon Sep 17 00:00:00 2001
From: Samuel Carroll <sam2@blackhillsinfosec.com>
Date: Mon, 16 Aug 2021 15:15:54 -0600
Subject: [PATCH 5/8] Adding requested changes from the code review, and
 changes that were needed to ensure functionality

---
 zeek_log_transport.sh | 134 ++++++++++++++++++++++++++++++++----------
 1 file changed, 102 insertions(+), 32 deletions(-)

diff --git a/zeek_log_transport.sh b/zeek_log_transport.sh
index 11d70a3..e3b1944 100755
--- a/zeek_log_transport.sh
+++ b/zeek_log_transport.sh
@@ -15,38 +15,108 @@
 
 export PATH="/sbin:/usr/sbin:$PATH"		#Note that cron does _NOT_ include /sbin in the path, so attempts to locate the "ip" binary fail without this fix
 
-default_user_on_aihunter='dataimport'
+default_user_on_aihunter="dataimport"
 
 get_send_logs() {
-        ##These names were found at https://docs.zeek.org/en/master/script-reference/log-files.html
-        declare -a zeek_logs=("conn" "dce_rpc" "dhcp" "dnp3" "dns" "ftp" "http" "irc" "kerberos" "modbus" "modbus_register_change" "mysql" "ntlm" "ntp" "radius" "rdp" "rfb" "sip" "smb_cmd" "smb_files" "smb_mapping" "smtp" "snmp" "socks" "ssh" "ssl" "syslog" "tunnel" "files" "ocsp" "pe" "x509" "netcontrol" "netcontrol_drop" "netcontrol_shunt" "netcontrol_catch_release" "openflow" "intel" "notice" "notice_alarm" "signatures" "traceroute" "known_certs" "known_hosts" "known_modbus" "known_services" "software" "barnyard2" "dpd" "unified2" "unknown_protocols" "weird" "weird_stats" "broker" "capture_loss" "cluster" "config" "loaded_scripts" "packet_filter" "print" "prof" "reporter" "stats" "stderr" "stdout"  "conn-summary")
-        send_logs=()
-
-	destination=$1
-	dest_log_trans=$2
-	local_dest_dir=$3
-
-        `rsync $destination:$dest_log_trans $local_dest_dir`
+	##These names were found at https://docs.zeek.org/en/master/script-reference/log-files.html
+	declare -a zeek_logs=(	"barnyard2"
+				"broker"
+				"capture_loss"
+				"cluster"
+				"config"
+				"conn"
+				"conn-summary"
+				"dce_rpc"
+				"dhcp"
+				"dnp3"
+				"dns"
+				"dpd"
+				"files"
+				"ftp"
+				"http"
+				"intel"
+				"irc"
+				"kerberos"
+				"known_certs"
+				"known_hosts"
+				"known_modbus"
+				"known_services"
+				"loaded_scripts"
+				"modbus"
+				"modbus_register_change"
+				"mysql"
+				"netcontrol"
+				"netcontrol_catch_release"
+				"netcontrol_drop"
+				"netcontrol_shunt"
+				"notice"
+				"notice_alarm"
+				"ntlm"
+				"ntp"
+				"ocsp"
+				"openflow"
+				"packet_filter"
+				"pe"
+				"print"
+				"prof"
+				"radius"
+				"rdp"
+				"reporter"
+				"rfb"
+				"signatures"
+				"sip"
+				"smb_cmd"
+				"smb_files"
+				"smb_mapping"
+				"smtp"
+				"snmp"
+				"socks"
+				"software"
+				"ssh"
+				"ssl"
+				"stats"
+				"stderr"
+				"stdout"
+				"syslog"
+				"traceroute"
+				"tunnel"
+				"unified2"
+				"unknown_protocols"
+				"weird"
+				"weird_stats"
+				"x509"
+			)
+	send_logs=()
+
+	destination="$1"
+	dest_log_trans="$2"
+	local_dest_dir="$3"
+
+	rsync $destination:$dest_log_trans $local_dest_dir
 
 	##Parse file for logs skip headerlines (indicated by :) and commented lines (indicated by #)
-        request_logs=`cat $local_dest_dir | grep -v : | grep -v #`
+	echo $local_dest_dir
+	request_logs=`cat $local_dest_dir | grep -v ":" | grep -v "#"`
+	echo ${requst_logs[0]}
 
 	##TODO check if the request logs is set to all if so we should set the send_logs to the zeek_logs variable
-	if [[ " ${zeek_logs[*]} " == *"all"* ]]; then
+	if [[ ${request_logs[*]} == *all* || ${request_logs[*]} == *All* || ${request_logs[*]} == *ALL* ]]; then
+		touch ./all_logs
 		send_logs+=("${zeek_logs[@]}")
 	else
-	        ##Ensure we only send the zeek logs
-        	for log in $request_logs
-	        do
-        	        if [[ " ${zeek_logs[*]} " == *"${log}"* ]]; then
-                	        send_logs+=($log)
-	                #else
-        	        #       echo $log will not be transported
-                	fi
-	        done
+		touch ./select_logs
+		##Ensure we only send the zeek logs
+		for log in $request_logs
+		do
+			if [[ ${zeek_logs[*]} == *${log}* ]]; then
+				send_logs+=($log)
+			#else
+			#       echo $log will not be transported
+			fi
+		done
 	fi
 
-        echo "${send_logs[*]}"
+	echo "${send_logs[*]}"
 }
 
 
@@ -123,14 +193,14 @@ usage () {
 
 require_util () {
 	#Returns true if all binaries listed as parameters exist somewhere in the path, False if one or more missing.
-        while [ -n "$1" ]; do
-                if ! type -path "$1" >/dev/null 2>/dev/null ; then
-                        echo Missing utility "$1". Please install it. >&2
-                        return 1        #False, app is not available.
-                fi
-                shift
-        done
-        return 0        #True, app is there.
+	while [ -n "$1" ]; do
+		if ! type -path "$1" >/dev/null 2>/dev/null ; then
+			echo Missing utility "$1". Please install it. >&2
+			return 1        #False, app is not available.
+		fi
+		shift
+	done
+	return 0	#True, app is there.
 } #End of requireutil
 
 
@@ -287,7 +357,7 @@ logs_str=`echo "${request_logs[*]// /|}"`
 cd "$local_tld" || fail "Unable to change to $local_tld"
 
 query="find . -type f -mtime -$request_days -iname '*.gz' | egrep '($logs_str)' | sort -u"
-send_candidates=$(eval "$query")
+send_candidates="$(eval $query)"
 if  [ ${#send_candidates} -eq 0 ]; then
 	#if we don't have anything we assume that we need to check the last 3 days for logs needed for RITA
 	send_candidates=`find . -type f -mtime -3 -iname '*.gz' | egrep '(conn|dns|http|ssl|x509|known_certs)' | sort -u`
@@ -296,8 +366,8 @@ if  [ ${#send_candidates} -eq 0 ]; then
 		printf "WARNING: No logs found, if your log directory is not $local_tld please use the flag: --localdir [bro_zeek_log_directory]"
 		echo
 	fi
-
 fi
+
 status "Transferring files to $aih_location"
 flock -xn "$HOME/rsync_log_transport.lck" timeout --kill-after=60 7080 $nice_me rsync $rsyncparams -avR -e "ssh $extra_ssh_params" $send_candidates "$aih_location:${remote_top_dir}/" --delay-updates --chmod=Do+rx,Fo+r
 retval=$?

From 79bea6baad61027bc82d040c4ff6e5fe70f471e7 Mon Sep 17 00:00:00 2001
From: Samuel Carroll <sam2@blackhillsinfosec.com>
Date: Tue, 17 Aug 2021 10:11:44 -0600
Subject: [PATCH 6/8] Removing debugging commands from code

---
 zeek_log_transport.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/zeek_log_transport.sh b/zeek_log_transport.sh
index e3b1944..ac4451c 100755
--- a/zeek_log_transport.sh
+++ b/zeek_log_transport.sh
@@ -101,10 +101,8 @@ get_send_logs() {
 
 	##TODO check if the request logs is set to all if so we should set the send_logs to the zeek_logs variable
 	if [[ ${request_logs[*]} == *all* || ${request_logs[*]} == *All* || ${request_logs[*]} == *ALL* ]]; then
-		touch ./all_logs
 		send_logs+=("${zeek_logs[@]}")
 	else
-		touch ./select_logs
 		##Ensure we only send the zeek logs
 		for log in $request_logs
 		do

From 88743d399f3ecd4c1987e88587c7573291cddbf5 Mon Sep 17 00:00:00 2001
From: Samuel Carroll <sam2@blackhillsinfosec.com>
Date: Thu, 9 Sep 2021 12:53:18 -0600
Subject: [PATCH 7/8] Updating scripts for Pull Request, have removed zeek log
 array, copied the remote config to a temp directory, and checked a sensor
 specific location for a config before a default remote config in the
 zeek_log_transport.sh file, I have also commented out the YAML keys, removed
 one level of log names and changed default days to 3 in
 zeek-transport-default.yaml

---
 zeek-transport-default.yaml | 136 ++++++++++++++++++------------------
 zeek_log_transport.sh       | 118 ++++++++-----------------------
 2 files changed, 98 insertions(+), 156 deletions(-)

diff --git a/zeek-transport-default.yaml b/zeek-transport-default.yaml
index d2540d8..9d6cae8 100644
--- a/zeek-transport-default.yaml
+++ b/zeek-transport-default.yaml
@@ -1,72 +1,72 @@
-Days: 7
+Days: 3
 Logs:
         #These headings are for user friendliness, they do not do anything
-        Default:
-                conn
-                dns
-                http
-                ssl
-                x509
-                known_certs
+        #Default:
+        conn
+        dns
+        http
+        ssl
+        x509
+        known_certs
         #Troubleshooting:
-                #capture_loss
-                #notice
-                #stats                
+        #capture_loss
+        #notice
+        #stats
         #All:
-                #dce_rpc
-                #dhcp
-                #dnp3
-                #ftp
-                #irc
-                #kerberos
-                #modbus
-                #modbus_register_change
-                #mysql
-                #ntlm
-                #ntp
-                #radius
-                #rdp
-                #rfb
-                #sip
-                #smb_cmd
-                #smb_files
-                #smb_mapping
-                #smtp
-                #snmp
-                #socks
-                #ssh
-                #syslog
-                #tunnel
-                #files
-                #ocsp
-                #pe
-                #netcontrol
-                #netcontrol_drop
-                #netcontrol_shunt
-                #netcontrol_catch_release
-                #openflow
-                #intel
-                #notice_alarm
-                #signatures
-                #traceroute
-                #known_hosts
-                #known_modbus
-                #known_services
-                #software
-                #barnyard2
-                #dpd
-                #unified2
-                #unknown_protocols
-                #weird
-                #weird_stats
-                #broker
-                #cluster
-                #config
-                #loaded_scripts
-                #packet_filter
-                #print
-                #prof
-                #reporter
-                #stderr
-                #stdout
-                #conn-summary
+        #dce_rpc
+        #dhcp
+        #dnp3
+        #ftp
+        #irc
+        #kerberos
+        #modbus
+        #modbus_register_change
+        #mysql
+        #ntlm
+        #ntp
+        #radius
+        #rdp
+        #rfb
+        #sip
+        #smb_cmd
+        #smb_files
+        #smb_mapping
+        #smtp
+        #snmp
+        #socks
+        #ssh
+        #syslog
+        #tunnel
+        #files
+        #ocsp
+        #pe
+        #netcontrol
+        #netcontrol_drop
+        #netcontrol_shunt
+        #netcontrol_catch_release
+        #openflow
+        #intel
+        #notice_alarm
+        #signatures
+        #traceroute
+        #known_hosts
+        #known_modbus
+        #known_services
+        #software
+        #barnyard2
+        #dpd
+        #unified2
+        #unknown_protocols
+        #weird
+        #weird_stats
+        #broker
+        #cluster
+        #config
+        #loaded_scripts
+        #packet_filter
+        #print
+        #prof
+        #reporter
+        #stderr
+        #stdout
+        #conn-summary
diff --git a/zeek_log_transport.sh b/zeek_log_transport.sh
index ac4451c..8a15e59 100755
--- a/zeek_log_transport.sh
+++ b/zeek_log_transport.sh
@@ -18,74 +18,6 @@ export PATH="/sbin:/usr/sbin:$PATH"		#Note that cron does _NOT_ include /sbin in
 default_user_on_aihunter="dataimport"
 
 get_send_logs() {
-	##These names were found at https://docs.zeek.org/en/master/script-reference/log-files.html
-	declare -a zeek_logs=(	"barnyard2"
-				"broker"
-				"capture_loss"
-				"cluster"
-				"config"
-				"conn"
-				"conn-summary"
-				"dce_rpc"
-				"dhcp"
-				"dnp3"
-				"dns"
-				"dpd"
-				"files"
-				"ftp"
-				"http"
-				"intel"
-				"irc"
-				"kerberos"
-				"known_certs"
-				"known_hosts"
-				"known_modbus"
-				"known_services"
-				"loaded_scripts"
-				"modbus"
-				"modbus_register_change"
-				"mysql"
-				"netcontrol"
-				"netcontrol_catch_release"
-				"netcontrol_drop"
-				"netcontrol_shunt"
-				"notice"
-				"notice_alarm"
-				"ntlm"
-				"ntp"
-				"ocsp"
-				"openflow"
-				"packet_filter"
-				"pe"
-				"print"
-				"prof"
-				"radius"
-				"rdp"
-				"reporter"
-				"rfb"
-				"signatures"
-				"sip"
-				"smb_cmd"
-				"smb_files"
-				"smb_mapping"
-				"smtp"
-				"snmp"
-				"socks"
-				"software"
-				"ssh"
-				"ssl"
-				"stats"
-				"stderr"
-				"stdout"
-				"syslog"
-				"traceroute"
-				"tunnel"
-				"unified2"
-				"unknown_protocols"
-				"weird"
-				"weird_stats"
-				"x509"
-			)
 	send_logs=()
 
 	destination="$1"
@@ -95,24 +27,20 @@ get_send_logs() {
 	rsync $destination:$dest_log_trans $local_dest_dir
 
 	##Parse file for logs skip headerlines (indicated by :) and commented lines (indicated by #)
-	echo $local_dest_dir
+	#echo $local_dest_dir
 	request_logs=`cat $local_dest_dir | grep -v ":" | grep -v "#"`
-	echo ${requst_logs[0]}
-
-	##TODO check if the request logs is set to all if so we should set the send_logs to the zeek_logs variable
-	if [[ ${request_logs[*]} == *all* || ${request_logs[*]} == *All* || ${request_logs[*]} == *ALL* ]]; then
-		send_logs+=("${zeek_logs[@]}")
-	else
-		##Ensure we only send the zeek logs
-		for log in $request_logs
-		do
-			if [[ ${zeek_logs[*]} == *${log}* ]]; then
-				send_logs+=($log)
-			#else
-			#       echo $log will not be transported
-			fi
-		done
-	fi
+	#echo ${requst_logs[0]}
+
+	##Ensure we only send the zeek logs
+	for log in $request_logs
+	do
+		##check requested logs agains a regex that should prevent escaping Zeek TLD
+		if [[ $log =~ ^[a-zA-Z0-9_-]+$ ]]; then
+			send_logs+=($log)
+		#else
+		#  echo "$log will not be transported"
+		fi
+	done
 
 	echo "${send_logs[*]}"
 }
@@ -347,8 +275,22 @@ status "Sending logs to rita/aihunter server $aih_location , My name: $my_id , l
 status "Preparing remote directories"
 ssh $extra_ssh_params "$aih_location" "mkdir -p ${remote_top_dir}/$today/ ${remote_top_dir}/$yesterday/ ${remote_top_dir}/$twoda/ ${remote_top_dir}/$threeda/ ${remote_top_dir}/current/"
 
-recv_loc="`pwd`/zeek-log-transport.yaml"
-request_logs=`get_send_logs $aih_location "/etc/AI-Hunter/zeek-log-transport.yaml" "$recv_loc"`
+# Set the destination location to be a temporary file we have read/write access
+recv_loc="$(mktemp)"
+# Clean up the temporary files on exit
+trap "rm -rf '$recv_loc'" EXIT
+
+# Check this sensor's remote log directory for zeek-log-transport.yaml if this
+# file exists use that for configuration, if not use the default config file
+
+remote_trans_yaml=""
+if ssh -q $aih_location [[ -e "$remote_top_dir/zeek-log-transport.yaml" ]]; then
+	remote_trans_yaml="$remote_top_dir/zeek-log-transport.yaml"
+else
+	remote_trans_yaml="/etc/AI-Hunter/zeek-log-transport.yaml"
+fi
+
+request_logs=`get_send_logs $aih_location "$remote_trans_yaml" "$recv_loc"`
 request_days=`get_send_days $recv_loc`
 logs_str=`echo "${request_logs[*]// /|}"`
 
@@ -357,7 +299,7 @@ cd "$local_tld" || fail "Unable to change to $local_tld"
 query="find . -type f -mtime -$request_days -iname '*.gz' | egrep '($logs_str)' | sort -u"
 send_candidates="$(eval $query)"
 if  [ ${#send_candidates} -eq 0 ]; then
-	#if we don't have anything we assume that we need to check the last 3 days for logs needed for RITA
+	# If send_candidates is empty assume  the last 3 days and default logs needed for RITA
 	send_candidates=`find . -type f -mtime -3 -iname '*.gz' | egrep '(conn|dns|http|ssl|x509|known_certs)' | sort -u`
 	if [ ${#send_candidates} -eq 0 ]; then
 		echo

From 344dbbadc8768e0199172003d68f8902d9571b33 Mon Sep 17 00:00:00 2001
From: ethack <ethack@users.noreply.github.com>
Date: Wed, 27 Oct 2021 13:29:07 -0500
Subject: [PATCH 8/8] Update default logs transferred

---
 zeek-transport-default.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/zeek-transport-default.yaml b/zeek-transport-default.yaml
index 9d6cae8..feea3c3 100644
--- a/zeek-transport-default.yaml
+++ b/zeek-transport-default.yaml
@@ -3,15 +3,16 @@ Logs:
         #These headings are for user friendliness, they do not do anything
         #Default:
         conn
+        conn-summary
         dns
         http
         ssl
         x509
         known_certs
         #Troubleshooting:
-        #capture_loss
-        #notice
-        #stats
+        capture_loss
+        notice
+        stats
         #All:
         #dce_rpc
         #dhcp
@@ -69,4 +70,4 @@ Logs:
         #reporter
         #stderr
         #stdout
-        #conn-summary
+