Skip to content

Commit

Permalink
Merge pull request #134 from netsage-project/master
Browse files Browse the repository at this point in the history 
Final changes to 2.1.2 to create release
  • Loading branch information
@dojosout
dojosout authored Dec 3, 2024
2 parents e8d263a + f349e50 commit 4c220ad
Show file tree
Hide file tree
Showing 21 changed files with 2,442 additions and 525 deletions.
55 changes: 0 additions & 55 deletions Upgrade_2_1_0.sh
View file

This file was deleted.

66 changes: 66 additions & 0 deletions Upgrade_2_1_2.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/bin/bash

# Define file paths
compose_file="docker-compose.override.yml"
map_file="data/logstash-aggregation-maps"

# Function to check if a line has been deleted
check_deleted() {
file=$1
pattern=$2
message=$3
grep -q "$pattern" "$file"
if [ $? -eq 0 ]; then
echo "Error: $message"
exit 1
fi
}

# Function to check if a line has been modified/updated
check_change() {
file=$1
pattern=$2
message=$3
grep -q "$pattern" "$file"
if [ $? -ne 0 ]; then
echo "Error: $message"
exit 1
fi
}

# Check if running as root
if [ "$EUID" -ne 0 ]; then
echo "Error: This script must be run as root."
exit 1
fi

# Update docker-compose.override.yml
sed -i "/logstash/d" "$compose_file"
sed -i "/netsage_pipeline/d" "$compose_file"
sed -i 's/2\.1\.[0-9]*/2.1.2/g' "$compose_file"

# Check if logstash: line was deleted
check_deleted "$compose_file" "logstash:" "Failed to delete the logstash entry."
check_deleted docker-compose.override.yml "netsage_pipeline" "Failed to delete the pipeline_logstash entry"

# Check other changes in docker-compose.override.yml
check_change "$compose_file" "netsage_importer:v2.1.2" "Failed to update pipeline_importer to netsage_importer:v2.1.2"
check_change "$compose_file" "netsage_collector:v2.1.2" "Failed to update netsage-nfdump-collector to netsage_collector:v2.1.2"

# Check and delete data/logstash-aggregation-maps if it exists
if [ -f "$map_file" ]; then
rm "$map_file"

# Verify if the deletion was successful
if [ ! -f "$map_file" ]; then
echo "File $map_file successfully deleted."
else
echo "Error: Failed to delete $map_file."
exit 1
fi
else
echo "File $map_file does not exist. No action needed."
fi

echo "NetSage Ingest Pipeline successfully upgraded to 2.1.2"

159 changes: 85 additions & 74 deletions conf-globus/42-scireg.conf
View file
Original file line number Diff line number Diff line change
@@ -1,133 +1,144 @@

# general approach used here:
# mmdb record gets parsed into [meta][scireg][src] (or [dst])
# this fills in [meta][scireg][src][latitude] and [meta][scireg][src][city_name]
# JSON in [meta][scireg][src][city_name] gets parsed into [meta][scireg][src_json]
# fields in [meta][scireg][src_json] (discipline, resource name and project name) are copied to final locations

filter {

# Tag SOURCE with Science Registry info - get info from fake geoip db holding scireg data
geoip {
id => "42-1"
database => "/var/lib/grnoc/netsage/scireg.mmdb"
database => "/var/lib/netsage/newScireg.mmdb"
default_database_type => "City"
cache_size => 1000
source => "[meta][src_ip]"
target => "[meta][scireg][src]"
tag_on_failure => []
tag_on_failure => ["_scireg_lookup_failure_src_ip"]
add_field => { "[meta][src_location][data_source]" => "scireg" }
}
# break up the json in city_name into indiv fields (target gets totally overwritten with the info from the json)
json {
id => "42-2"
source => "[meta][scireg][src][city_name]"
target => "[meta][scireg][src]"
target => "[meta][scireg][src_json]"
tag_on_failure => ["_jsonparsefailure of city_name"]
}


# Tag DESTINATION with Science Registry info - get info from fake geoip db holding scireg data
# Skip if it's Multicast!
if [meta][dst_organization] != "Multicast" {
geoip {
id => "42-3"
database => "/var/lib/grnoc/netsage/scireg.mmdb"
database => "/var/lib/netsage/newScireg.mmdb"
default_database_type => "City"
cache_size => 1000
source => "[meta][dst_ip]"
target => "[meta][scireg][dst]"
tag_on_failure => []
tag_on_failure => ["_scireg_lookup_failure_dst_ip"]
add_field => { "[meta][dst_location][data_source]" => "scireg" }
}
json {
id => "42-4"
source => "[meta][scireg][dst][city_name]"
target => "[meta][scireg][dst]"
target => "[meta][scireg][dst_json]"
tag_on_failure => ["_jsonparsefailure of city_name"]
}
}

# Move project names to new fields
if [meta][scireg][src][projects][0] {
ruby {
id => "42-5"
code => "
event.set('[meta][scireg][src][project_names]', event.get('[meta][scireg][src][projects]').map{ |n| n['project_name'] })
"
}
}
if [meta][scireg][dst][projects][0] {
ruby {
id => "42-6"
code => "
event.set('[meta][scireg][dst][project_names]', event.get('[meta][scireg][dst][projects]').map{ |n| n['project_name'] })
"
}
}

# if lat/long set in Science Registry, use those
if [meta][scireg][src][latitude] {
if [meta][scireg][src][latitude] {
mutate {
id => "42-7"
copy => {
"[meta][scireg][src][latitude]" => "[meta][src_location][lat]"
"[meta][scireg][src][longitude]" => "[meta][src_location][lon]"
"[meta][scireg][src][org_name]" => "[meta][src_organization]"
"[meta][scireg][src][resource]" => "[meta][src_resource_name]"
"[meta][scireg][src_json][org_name]" => "[meta][src_organization]"
"[meta][scireg][src_json][resource]" => "[meta][src_resource_name]"
}
}
# all scireg should have org_name and resource, but discipline and project are optional
# so only copy if not null
if [meta][scireg][src_json][discipline] != "" {
# copy to legacy field [scireg][src][discipline] too. Note: must use separte mutate for that
mutate {
copy => { "[meta][scireg][src_json][discipline]" => "[meta][src_discipline]" }
}
mutate {
copy => { "[meta][scireg][src_json][discipline]" => "[meta][scireg][src][discipline]" }
}
add_field => { "[meta][src_location][data_source]" => "scireg" }
}

if [meta][scireg][src_json][project] != "" {
mutate {
copy => { "[meta][scireg][src_json][project]" => "[meta][src_project_names]" }
}
mutate {
copy => { "[meta][scireg][src_json][project]" => "[meta][scireg][src][project_names]" }
}
}
}
# if not in scireg, set to none
if ![meta][src_location][data_source] {
mutate {
add_field => { "[meta][src_location][data_source]" => "none" }
}
}

if [meta][scireg][dst][latitude] {
mutate {
id => "42-8"
copy => {
"[meta][scireg][dst][latitude]" => "[meta][dst_location][lat]"
"[meta][scireg][dst][longitude]" => "[meta][dst_location][lon]"
"[meta][scireg][dst][org_name]" => "[meta][dst_organization]"
"[meta][scireg][dst][resource]" => "[meta][dst_resource_name]"
"[meta][scireg][dst][latitude]" => "[meta][dst_location][lat]"
"[meta][scireg][dst][longitude]" => "[meta][dst_location][lon]"
"[meta][scireg][dst_json][org_name]" => "[meta][dst_organization]"
"[meta][scireg][dst_json][resource]" => "[meta][dst_resource_name]"
}
}
if [meta][scireg][dst_json][discipline] != "" {
mutate {
copy => { "[meta][scireg][dst_json][discipline]" => "[meta][dst_discipline]" }
}
mutate {
copy => { "[meta][scireg][dst_json][discipline]" => "[meta][scireg][dst][discipline]" }
}
}

add_field => { "[meta][dst_location][data_source]" => "scireg" }
if [meta][scireg][dst_json][project] != "" {
mutate {
copy => { "[meta][scireg][dst_json][project]" => "[meta][dst_project_names]" }
}
mutate {
copy => { "[meta][scireg][dst_json][project]" => "[meta][scireg][dst][project_names]" }
}
}
}

# if not in scireg, set to none
if ![meta][dst_location][data_source] {
mutate {
add_field => { "[meta][dst_location][data_source]" => "none" }
}
}


# Remove unneeded fields
# note: after SciReg cleanup, be sure to edit this!! XXX
mutate {
id => "42-10"
remove_field => "[meta][scireg][src][org_description]"
remove_field => "[meta][scireg][dst][org_description]"
remove_field => "[meta][scireg][src][org_url]"
remove_field => "[meta][scireg][dst][org_url]"
remove_field => "[meta][scireg][src][org_country_code]"
remove_field => "[meta][scireg][dst][org_country_code]"
remove_field => "[meta][scireg][src][org_latitude]"
remove_field => "[meta][scireg][dst][org_latitude]"
remove_field => "[meta][scireg][src][org_longitude]"
remove_field => "[meta][scireg][dst][org_longitude]"
remove_field => "[meta][scireg][src][location]"
remove_field => "[meta][scireg][dst][location]"
remove_field => "[meta][scireg][src][ip]"
remove_field => "[meta][scireg][dst][ip]"
remove_field => "[meta][scireg][src][latitude]"
remove_field => "[meta][scireg][dst][latitude]"
remove_field => "[meta][scireg][src][longitude]"
remove_field => "[meta][scireg][dst][latitude]"
remove_field => "[meta][scireg][dst][longitude]"
remove_field => "[meta][scireg][src][asn]"
remove_field => "[meta][scireg][dst][asn]"
remove_field => "[meta][scireg][src][description]"
remove_field => "[meta][scireg][dst][description]"
remove_field => "[meta][scireg][src][discipline_description]"
remove_field => "[meta][scireg][dst][discipline_description]"
remove_field => "[meta][scireg][src][role_description]"
remove_field => "[meta][scireg][dst][role_description]"
remove_field => "[meta][scireg][src][country_code]"
remove_field => "[meta][scireg][dst][country_code]"
remove_field => "[meta][scireg][src][ip_block_id]"
remove_field => "[meta][scireg][dst][ip_block_id]"
remove_field => "[meta][scireg][src][addresses]"
remove_field => "[meta][scireg][dst][addresses]"
remove_field => "[meta][scireg][src][addresses_str]"
remove_field => "[meta][scireg][dst][addresses_str]"
remove_field => "[meta][scireg][src][org_abbr]"
remove_field => "[meta][scireg][dst][org_abbr]"
remove_field => "[meta][scireg][src][org_name]"
remove_field => "[meta][scireg][dst][org_abbr]"
#remove_field => "[meta][scireg][src][resource]"
#remove_field => "[meta][scireg][dst][resource]"
remove_field => "[meta][scireg][src][resource_abbr]"
remove_field => "[meta][scireg][dst][resource_abbr]"
# maybe this will get used someday??
remove_field => "[meta][scireg][src][projects]"
remove_field => "[meta][scireg][dst][projects]"
# comment these out for debugging
remove_field => "[meta][scireg][src][city_name]"
remove_field => "[meta][scireg][dst][city_name]"
remove_field => "[meta][scireg][src_json]"
remove_field => "[meta][scireg][dst_json]"
}

}

Loading

0 comments on commit 4c220ad

Please sign in to comment.