Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle notifications with large number of tickets and use docker compose syntax #77

Merged
merged 3 commits into from
Jun 10, 2022
Merged

Handle notifications with large number of tickets and use docker compose syntax #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8ca0dff
Switch from a find to an aggregation query
dav3r Jun 10, 2022
f11e855
Update code to handle side effects of aggregation
dav3r Jun 10, 2022
5485ed9
Use "docker compose" syntax instead of "docker-compose"
dav3r Jun 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions cyhy_report/cyhy_notification/generate_notification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,21 +262,29 @@ def __load_tickets(self, ticket_ids):
tickets should not be saved back to the database because they receive
extra fields from their latest vulnerabilty/port scan.
"""
# We use an aggregation here because a regular find/sort query can
# exceed the memory limit of MongoDB: "Sort operation used more than
# the maximum 33554432 bytes of RAM. Add an index, or specify a
# smaller limit."
tickets = list(
self.__cyhy_db.TicketDoc.find({"_id": {"$in": ticket_ids}}).sort(
self.__cyhy_db.TicketDoc.collection.aggregate(
[
("details.kev", -1),
("details.cvss_base_score", -1),
("time_opened", 1),
("details.name", 1),
]
{"$match": {"_id": {"$in": ticket_ids}}},
{"$sort":
{
"details.kev": -1,
"details.cvss_base_score": -1,
"time_opened": 1,
"details.name": 1
},
}
],
cursor={},
allowDiskUse=True,
)
)

for ticket in tickets:
# Neuter this monstrosity so it can't be saved (easily)
ticket.connection = None

# Flatten structure by copying details to ticket root
ticket.update(ticket["details"])

Expand All @@ -285,7 +293,7 @@ def __load_tickets(self, ticket_ids):
ticket["based_on_vulnscan"] = True
ticket["based_on_portscan"] = False
try:
latest_vuln = ticket.latest_vuln()
latest_vuln = self.__cyhy_db.TicketDoc(ticket).latest_vuln()
except database.VulnScanNotFoundException as e:
print("\n Warning (non-fatal): {}".format(e.message))
# The vuln_scan has likely been archived; get the vuln_scan
Expand All @@ -304,7 +312,7 @@ def __load_tickets(self, ticket_ids):
ticket["based_on_portscan"] = True
ticket["based_on_vulnscan"] = False
try:
latest_port = ticket.latest_port()
latest_port = self.__cyhy_db.TicketDoc(ticket).latest_port()
except database.PortScanNotFoundException as e:
print("\n Warning (non-fatal): {}".format(e.message))
# The port_scan has likely been archived; get the port_scan
Expand Down
3 changes: 2 additions & 1 deletion extras/create_send_notifications.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,8 @@ def main():
os.chdir(CYHY_MAILER_DIR)
p = subprocess.Popen(
[
"docker-compose",
"docker",
"compose",
"-f",
"docker-compose.yml",
"-f",
Expand Down