From 38ac2448543d9fe0d2cdc6fbe59428e0cfc2e89e Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Thu, 29 Apr 2021 12:33:52 +0200 Subject: [PATCH 01/13] Add farmer_server parser: #65 --- src/chia_log/parsers/farmer_server_parser.py | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/chia_log/parsers/farmer_server_parser.py diff --git a/src/chia_log/parsers/farmer_server_parser.py b/src/chia_log/parsers/farmer_server_parser.py new file mode 100644 index 0000000..2a1bf28 --- /dev/null +++ b/src/chia_log/parsers/farmer_server_parser.py @@ -0,0 +1,51 @@ +# std +import re +import logging +from dataclasses import dataclass +from datetime import datetime +from typing import List + +# lib +from dateutil import parser as dateutil_parser + + +@dataclass +class FarmerServerMessage: + """Parsed information from farmer_server logs""" + + timestamp: datetime + peer_hash: str + + +class FarmerServerParser: + """This class can parse info log messages from the chia farmer_server + + You need to have enabled "log_level: INFO" in your chia config.yaml + The chia config.yaml is usually under ~/.chia/mainnet/config/config.yaml + """ + + def __init__(self): + logging.info("Enabled parser for farmer_server activity - peer infos.") + self._regex = re.compile( + r"([0-9:.]*) farmer farmer_server\s*: INFO\s* <\- farming_info " + r"from peer ([0-9a-z.]*) .*" + ) + + def parse(self, logs: str) -> List[FarmerServerMessage]: + """Parses all farmer_server activity messages from a bunch of logs + + :param logs: String of logs - can be multi-line + :returns: A list of parsed messages - can be empty + """ + + parsed_messages = [] + matches = self._regex.findall(logs) + for match in matches: + parsed_messages.append( + FarmerServerMessage( + timestamp=dateutil_parser.parse(match[0]), + peer_hash=match[1], + ) + ) + + return parsed_messages From 0134726c1628bc0fe32d6f886e28f5301ab61aa7 Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Wed, 5 May 2021 15:42:48 +0200 Subject: [PATCH 02/13] Add handler and condition checker for #65 --- .../handlers/condition_checkers/__init__.py | 7 ++ .../remote_harvester_activity.py | 67 +++++++++++++++++++ .../handlers/farmer_server_handler.py | 54 +++++++++++++++ src/chia_log/log_handler.py | 3 +- 4 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 src/chia_log/handlers/condition_checkers/remote_harvester_activity.py create mode 100644 src/chia_log/handlers/farmer_server_handler.py diff --git a/src/chia_log/handlers/condition_checkers/__init__.py b/src/chia_log/handlers/condition_checkers/__init__.py index 8d41173..476c740 100644 --- a/src/chia_log/handlers/condition_checkers/__init__.py +++ b/src/chia_log/handlers/condition_checkers/__init__.py @@ -6,6 +6,7 @@ from src.notifier import Event from ...parsers.finished_signage_point_parser import FinishedSignagePointMessage from ...parsers.harvester_activity_parser import HarvesterActivityMessage +from ...parsers.farmer_server_parser import FarmerServerMessage class FinishedSignageConditionChecker(ABC): @@ -18,3 +19,9 @@ class HarvesterConditionChecker(ABC): @abstractmethod def check(self, obj: HarvesterActivityMessage) -> Optional[Event]: pass + + +class FarmerConditionChecker(ABC): + @abstractmethod + def check(self, obj: FarmerServerMessage) -> Optional[Event]: + pass diff --git a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py new file mode 100644 index 0000000..5d7e840 --- /dev/null +++ b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py @@ -0,0 +1,67 @@ +# std +import logging +from typing import Optional +from dataclasses import dataclass +from datetime import datetime + +# project +from src.notifier import Event, EventService, EventType, EventPriority +from . import FarmerConditionChecker +from ...parsers.farmer_server_parser import FarmerServerMessage + + +@dataclass +class RemoteHarvester: + """Record of last activity""" + last_activity: datetime + peer_hash: str + +class RemoteHarvesterActivity(FarmerConditionChecker): + """The remote harvesters connected to the farmer_server + are not expected to disappear. Disappearing remote harvesters + indicate network or remote system issues. + """ + + def __init__(self): + logging.info("Enabled check for disappearing remote harvesters.") + self._warning_threshold = 300 + self._remote_harvesters = [] + + def check(self, obj: FarmerServerMessage) -> Optional[Event]: + #update last_activity + is_new_harvester = true + for i, remote_harvester in enumerate(self._remote_harvesters): + if remote_harvester.peer_hash == obj.peer_hash: + is_new_harvester = false + self._remote_harvesters[i].last_activity = obj.timestamp + self._remote_harvesters[i].warning_issued = false + logging.info(f"Updating Peer Harvester {obj.peer_hash}") + break + + if is_new_harvester: + new_harvester = RemoteHarvester() + new_harvester.last_activity = obj.timestamp + new_harvester.peer_hash = obj.peer_hash + new_harvester.warning_issued = false + self._remote_harvesters.append(new_harvester) + logging.info(f"New Peer Harvester {obj.peer_hash}") + + + event = None + for i, remote_harvester in enumerate(self._remote_harvesters): + seconds_since_last = (obj.timestamp - remote_harvester.last_activity).seconds + if last_activity > self._warning_threshold: + message = ( + f"Remote Harvester Offline: {remote_harvester.peer_hash} " + f"did not participate for {seconds_since_last} seconds!." + ) + logging.warning(message) + event = Event( + type=EventType.USER, priority=EventPriority.NORMAL, service=EventService.FARMER, message=message + ) + + # Remove offline harvester + self._remote_harvesters.pop(i) + break + + return event diff --git a/src/chia_log/handlers/farmer_server_handler.py b/src/chia_log/handlers/farmer_server_handler.py new file mode 100644 index 0000000..6cd2d95 --- /dev/null +++ b/src/chia_log/handlers/farmer_server_handler.py @@ -0,0 +1,54 @@ +# std +import logging +from typing import List, Optional + +# project +from . import LogHandler +from ..parsers.farmer_server_parser import FarmerServerParser +from .condition_checkers import FarmerConditionChecker +from .condition_checkers.remote_harvester_activity import RemoteHarvesterActivity +from .daily_stats.stats_manager import StatsManager +from src.notifier import Event, EventService, EventType, EventPriority + + +class FarmerServerHandler(LogHandler): + """This handler parses all logs indicating farmer_server + activity and participation of remote harvester in challenges. + It holds a list of condition checkers that are evaluated for + each event to ensure that farming is going smoothly. + """ + + def __init__(self): + self._parser = FarmerServerParser() + self._cond_checkers: List[FarmerConditionChecker] = [ + RemoteHarvesterActivity(), + ] + + def handle(self, logs: str, stats_manager: Optional[StatsManager] = None) -> List[Event]: + """Process incoming logs, check all conditions + and return a list of notable events. + """ + + events = [] + activity_messages = self._parser.parse(logs) + # if stats_manager: + # stats_manager.consume_farmer_messages(activity_messages) + + # Create a keep-alive event if any logs indicating + # activity have been successfully parsed + if len(activity_messages) > 0: + logging.debug(f"Parsed {len(activity_messages)} activity messages") + events.append( + Event( + type=EventType.KEEPALIVE, priority=EventPriority.NORMAL, service=EventService.FARMER, message="" + ) + ) + + # Run messages through all condition checkers + for msg in activity_messages: + for checker in self._cond_checkers: + event = checker.check(msg) + if event: + events.append(event) + + return events diff --git a/src/chia_log/log_handler.py b/src/chia_log/log_handler.py index cce46ef..fcf8b78 100644 --- a/src/chia_log/log_handler.py +++ b/src/chia_log/log_handler.py @@ -5,6 +5,7 @@ from src.chia_log.handlers.daily_stats.stats_manager import StatsManager from src.chia_log.handlers.harvester_activity_handler import HarvesterActivityHandler from src.chia_log.handlers.finished_signage_point_handler import FinishedSignagePointHandler +from src.chia_log.handlers.farmer_server_handler import FarmerServerHandler from src.chia_log.log_consumer import LogConsumerSubscriber, LogConsumer from src.notifier.notify_manager import NotifyManager @@ -28,7 +29,7 @@ def __init__( ): self._notify_manager = notify_manager self._stats_manager = stats_manager - self._handlers = [HarvesterActivityHandler(), FinishedSignagePointHandler()] + self._handlers = [HarvesterActivityHandler(), FinishedSignagePointHandler(), FarmerServerHandler()] log_consumer.subscribe(self) def consume_logs(self, logs: str): From 8586d832973c4c980750935c0f164657fc021c9e Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Wed, 5 May 2021 16:59:28 +0200 Subject: [PATCH 03/13] Add unit tests and bugfixes for farmer_server #65 --- .../remote_harvester_activity.py | 26 +++++----- .../handlers/farmer_server_handler.py | 7 +-- src/chia_log/parsers/farmer_server_parser.py | 4 +- .../handlers/test_farmer_server_handler.py | 44 +++++++++++++++++ .../farmer_server/disappearing_harvester.txt | 16 ++++++ tests/chia_log/logs/farmer_server/nominal.txt | 8 +++ .../parsers/test_farmer_server_parser.py | 49 +++++++++++++++++++ 7 files changed, 134 insertions(+), 20 deletions(-) create mode 100644 tests/chia_log/handlers/test_farmer_server_handler.py create mode 100644 tests/chia_log/logs/farmer_server/disappearing_harvester.txt create mode 100644 tests/chia_log/logs/farmer_server/nominal.txt create mode 100644 tests/chia_log/parsers/test_farmer_server_parser.py diff --git a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py index 5d7e840..52eac70 100644 --- a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py +++ b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py @@ -15,6 +15,7 @@ class RemoteHarvester: """Record of last activity""" last_activity: datetime peer_hash: str + ip_addr: str class RemoteHarvesterActivity(FarmerConditionChecker): """The remote harvesters connected to the farmer_server @@ -29,35 +30,34 @@ def __init__(self): def check(self, obj: FarmerServerMessage) -> Optional[Event]: #update last_activity - is_new_harvester = true + is_new_harvester = True for i, remote_harvester in enumerate(self._remote_harvesters): if remote_harvester.peer_hash == obj.peer_hash: - is_new_harvester = false + is_new_harvester = False self._remote_harvesters[i].last_activity = obj.timestamp - self._remote_harvesters[i].warning_issued = false - logging.info(f"Updating Peer Harvester {obj.peer_hash}") break if is_new_harvester: - new_harvester = RemoteHarvester() - new_harvester.last_activity = obj.timestamp - new_harvester.peer_hash = obj.peer_hash - new_harvester.warning_issued = false + new_harvester = RemoteHarvester( + obj.timestamp, + obj.peer_hash, + obj.ip_addr + ) self._remote_harvesters.append(new_harvester) - logging.info(f"New Peer Harvester {obj.peer_hash}") + logging.info(f"New remote harvester: {obj.ip_addr}") event = None for i, remote_harvester in enumerate(self._remote_harvesters): seconds_since_last = (obj.timestamp - remote_harvester.last_activity).seconds - if last_activity > self._warning_threshold: + if seconds_since_last > self._warning_threshold: message = ( - f"Remote Harvester Offline: {remote_harvester.peer_hash} " - f"did not participate for {seconds_since_last} seconds!." + f"Remote harvester offline: {remote_harvester.ip_addr} " + f"did not participate for {seconds_since_last} seconds!" ) logging.warning(message) event = Event( - type=EventType.USER, priority=EventPriority.NORMAL, service=EventService.FARMER, message=message + type=EventType.USER, priority=EventPriority.HIGH, service=EventService.FARMER, message=message ) # Remove offline harvester diff --git a/src/chia_log/handlers/farmer_server_handler.py b/src/chia_log/handlers/farmer_server_handler.py index 6cd2d95..9612006 100644 --- a/src/chia_log/handlers/farmer_server_handler.py +++ b/src/chia_log/handlers/farmer_server_handler.py @@ -37,12 +37,7 @@ def handle(self, logs: str, stats_manager: Optional[StatsManager] = None) -> Lis # Create a keep-alive event if any logs indicating # activity have been successfully parsed if len(activity_messages) > 0: - logging.debug(f"Parsed {len(activity_messages)} activity messages") - events.append( - Event( - type=EventType.KEEPALIVE, priority=EventPriority.NORMAL, service=EventService.FARMER, message="" - ) - ) + logging.debug(f"Parsed {len(activity_messages)} farmer_server messages") # Run messages through all condition checkers for msg in activity_messages: diff --git a/src/chia_log/parsers/farmer_server_parser.py b/src/chia_log/parsers/farmer_server_parser.py index 2a1bf28..5fd26b6 100644 --- a/src/chia_log/parsers/farmer_server_parser.py +++ b/src/chia_log/parsers/farmer_server_parser.py @@ -15,6 +15,7 @@ class FarmerServerMessage: timestamp: datetime peer_hash: str + ip_addr: str class FarmerServerParser: @@ -28,7 +29,7 @@ def __init__(self): logging.info("Enabled parser for farmer_server activity - peer infos.") self._regex = re.compile( r"([0-9:.]*) farmer farmer_server\s*: INFO\s* <\- farming_info " - r"from peer ([0-9a-z.]*) .*" + r"from peer ([0-9a-z.]*) ([0-9\.:a-f]*)" ) def parse(self, logs: str) -> List[FarmerServerMessage]: @@ -45,6 +46,7 @@ def parse(self, logs: str) -> List[FarmerServerMessage]: FarmerServerMessage( timestamp=dateutil_parser.parse(match[0]), peer_hash=match[1], + ip_addr=match[2] ) ) diff --git a/tests/chia_log/handlers/test_farmer_server_handler.py b/tests/chia_log/handlers/test_farmer_server_handler.py new file mode 100644 index 0000000..83b2111 --- /dev/null +++ b/tests/chia_log/handlers/test_farmer_server_handler.py @@ -0,0 +1,44 @@ +# std +import unittest +from pathlib import Path + +# project +from src.chia_log.handlers import farmer_server_handler +from src.notifier import EventType, EventService, EventPriority + + +class TestFarmerServerHandler(unittest.TestCase): + def setUp(self) -> None: + self.handler = farmer_server_handler.FarmerServerHandler() + self.example_logs_path = Path(__file__).resolve().parents[1] / "logs/farmer_server" + + def testNominal(self): + with open(self.example_logs_path / "nominal.txt") as f: + logs = f.readlines() + + for log in logs: + events = self.handler.handle(log) + self.assertEqual(len(events), 0, "Not expecting any events") + + def testSkippedSignagePoints(self): + with open(self.example_logs_path / "disappearing_harvester.txt") as f: + logs = f.readlines() + + expected_messages = [ + "Remote harvester offline: 178.19.176.201 did not participate for 385 seconds!", + ] + + checked = 0 + for log in logs: + events = self.handler.handle(log) + if len(events) > 0: + self.assertEqual(len(events), 1, "Expected a single event") + self.assertEqual(events[0].type, EventType.USER, "Unexpected type") + self.assertEqual(events[0].priority, EventPriority.HIGH, "Unexpected priority") + self.assertEqual(events[0].service, EventService.FARMER, "Unexpected service") + self.assertEqual(events[0].message, expected_messages[checked], "Unexpected message") + checked += 1 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/chia_log/logs/farmer_server/disappearing_harvester.txt b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt new file mode 100644 index 0000000..9c94a9c --- /dev/null +++ b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt @@ -0,0 +1,16 @@ +2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 +2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:04:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:04:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:04:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:06:24.253 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:06:24.274 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:06:24.297 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:08:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:08:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:08:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:10:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:10:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:10:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 diff --git a/tests/chia_log/logs/farmer_server/nominal.txt b/tests/chia_log/logs/farmer_server/nominal.txt new file mode 100644 index 0000000..0b8228c --- /dev/null +++ b/tests/chia_log/logs/farmer_server/nominal.txt @@ -0,0 +1,8 @@ +2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 +2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:02:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:02:14.995 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 +2021-05-05T16:02:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:02:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 diff --git a/tests/chia_log/parsers/test_farmer_server_parser.py b/tests/chia_log/parsers/test_farmer_server_parser.py new file mode 100644 index 0000000..3c0eaa2 --- /dev/null +++ b/tests/chia_log/parsers/test_farmer_server_parser.py @@ -0,0 +1,49 @@ +# std +import unittest +from pathlib import Path + +# project +from src.chia_log.parsers import farmer_server_parser + + +class TestFarmerServerParser(unittest.TestCase): + def setUp(self) -> None: + self.parser = farmer_server_parser.FarmerServerParser() + self.example_logs_path = Path(__file__).resolve().parents[1] / "logs/farmer_server" + with open(self.example_logs_path / "nominal.txt") as f: + self.nominal_logs = f.read() + + def tearDown(self) -> None: + pass + + def testBasicParsing(self): + for logs in [self.nominal_logs]: + + # Check that important fields are correctly parsed + activity_messages = self.parser.parse(logs) + self.assertNotEqual(len(activity_messages), 0, "No log messages found") + + expected_peer_hashes = ["20388420", "ea61688b", "fe1eb730", "c3e3eb9a", + "20388420", "fe1eb730", "ea61688b", "c3e3eb9a"] + expected_ip_addresses = ["127.0.0.1", "192.168.178.22", "178.19.176.201", + "131.2.211.71", "127.0.0.1", "178.19.176.201", + "192.168.178.22", "131.2.211.71"] + + for msg, peer_hash, ip_addr in zip( + activity_messages, + expected_peer_hashes, + expected_ip_addresses, + ): + self.assertEqual(msg.peer_hash, peer_hash, "Peer hash don't match") + self.assertEqual(msg.ip_addr, ip_addr, "IP Addr don't match") + + # Check arithmetic with parsed timestamps works + prev_timestamp = activity_messages[0].timestamp + for msg in activity_messages[1:]: + seconds_since_last_activity = (msg.timestamp - prev_timestamp).seconds + self.assertLess(seconds_since_last_activity, 10, "Unexpected duration between farmer_server events") + prev_timestamp = msg.timestamp + + +if __name__ == "__main__": + unittest.main() From bf288716001794c6544a130363ea47dec9a9a7d2 Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Sun, 9 May 2021 11:53:40 +0200 Subject: [PATCH 04/13] Fix issues according to PR Review #83 Storing remote harvesters in dict Store ip_addr again if changed Delete deactivated code Change test ip addresses to more dummy looking ones --- .../remote_harvester_activity.py | 22 ++++++++----------- .../handlers/farmer_server_handler.py | 4 ---- .../handlers/test_farmer_server_handler.py | 4 ++-- .../farmer_server/disappearing_harvester.txt | 22 +++++++++---------- tests/chia_log/logs/farmer_server/nominal.txt | 12 +++++----- .../parsers/test_farmer_server_parser.py | 6 ++--- 6 files changed, 31 insertions(+), 39 deletions(-) diff --git a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py index 52eac70..22941f0 100644 --- a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py +++ b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py @@ -26,29 +26,25 @@ class RemoteHarvesterActivity(FarmerConditionChecker): def __init__(self): logging.info("Enabled check for disappearing remote harvesters.") self._warning_threshold = 300 - self._remote_harvesters = [] + self._remote_harvesters = {} def check(self, obj: FarmerServerMessage) -> Optional[Event]: #update last_activity - is_new_harvester = True - for i, remote_harvester in enumerate(self._remote_harvesters): - if remote_harvester.peer_hash == obj.peer_hash: - is_new_harvester = False - self._remote_harvesters[i].last_activity = obj.timestamp - break - - if is_new_harvester: - new_harvester = RemoteHarvester( + if self._remote_harvesters.get(obj.peer_hash, False): + self._remote_harvesters[obj.peer_hash].last_activity = obj.timestamp + self._remote_harvesters[obj.peer_hash].ip_addr = obj.ip_addr + else: + self._remote_harvesters[obj.peer_hash] = RemoteHarvester( obj.timestamp, obj.peer_hash, obj.ip_addr ) - self._remote_harvesters.append(new_harvester) logging.info(f"New remote harvester: {obj.ip_addr}") event = None - for i, remote_harvester in enumerate(self._remote_harvesters): + for peer_hash in self._remote_harvesters: + remote_harvester = self._remote_harvesters[peer_hash] seconds_since_last = (obj.timestamp - remote_harvester.last_activity).seconds if seconds_since_last > self._warning_threshold: message = ( @@ -61,7 +57,7 @@ def check(self, obj: FarmerServerMessage) -> Optional[Event]: ) # Remove offline harvester - self._remote_harvesters.pop(i) + self._remote_harvesters.pop(peer_hash) break return event diff --git a/src/chia_log/handlers/farmer_server_handler.py b/src/chia_log/handlers/farmer_server_handler.py index 9612006..551832b 100644 --- a/src/chia_log/handlers/farmer_server_handler.py +++ b/src/chia_log/handlers/farmer_server_handler.py @@ -31,11 +31,7 @@ def handle(self, logs: str, stats_manager: Optional[StatsManager] = None) -> Lis events = [] activity_messages = self._parser.parse(logs) - # if stats_manager: - # stats_manager.consume_farmer_messages(activity_messages) - # Create a keep-alive event if any logs indicating - # activity have been successfully parsed if len(activity_messages) > 0: logging.debug(f"Parsed {len(activity_messages)} farmer_server messages") diff --git a/tests/chia_log/handlers/test_farmer_server_handler.py b/tests/chia_log/handlers/test_farmer_server_handler.py index 83b2111..206f494 100644 --- a/tests/chia_log/handlers/test_farmer_server_handler.py +++ b/tests/chia_log/handlers/test_farmer_server_handler.py @@ -20,12 +20,12 @@ def testNominal(self): events = self.handler.handle(log) self.assertEqual(len(events), 0, "Not expecting any events") - def testSkippedSignagePoints(self): + def testDisconnectedHarvester(self): with open(self.example_logs_path / "disappearing_harvester.txt") as f: logs = f.readlines() expected_messages = [ - "Remote harvester offline: 178.19.176.201 did not participate for 385 seconds!", + "Remote harvester offline: 255.255.255.255 did not participate for 385 seconds!", ] checked = 0 diff --git a/tests/chia_log/logs/farmer_server/disappearing_harvester.txt b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt index 9c94a9c..cb7ca83 100644 --- a/tests/chia_log/logs/farmer_server/disappearing_harvester.txt +++ b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt @@ -1,16 +1,16 @@ 2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 -2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 255.255.255.255 +2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 2021-05-05T16:04:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:04:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:04:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:04:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:04:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 2021-05-05T16:06:24.253 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:06:24.274 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:06:24.297 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:06:24.274 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:06:24.297 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 2021-05-05T16:08:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:08:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 -2021-05-05T16:08:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:08:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:08:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 2021-05-05T16:10:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:10:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 -2021-05-05T16:10:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:10:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:10:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 diff --git a/tests/chia_log/logs/farmer_server/nominal.txt b/tests/chia_log/logs/farmer_server/nominal.txt index 0b8228c..8c54d27 100644 --- a/tests/chia_log/logs/farmer_server/nominal.txt +++ b/tests/chia_log/logs/farmer_server/nominal.txt @@ -1,8 +1,8 @@ 2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 -2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 255.255.255.255 +2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 2021-05-05T16:02:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:02:14.995 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 -2021-05-05T16:02:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:02:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:02:14.995 farmer farmer_server : INFO <- farming_info from peer fe1eb730 255.255.255.255 +2021-05-05T16:02:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 diff --git a/tests/chia_log/parsers/test_farmer_server_parser.py b/tests/chia_log/parsers/test_farmer_server_parser.py index 3c0eaa2..9d06334 100644 --- a/tests/chia_log/parsers/test_farmer_server_parser.py +++ b/tests/chia_log/parsers/test_farmer_server_parser.py @@ -25,9 +25,9 @@ def testBasicParsing(self): expected_peer_hashes = ["20388420", "ea61688b", "fe1eb730", "c3e3eb9a", "20388420", "fe1eb730", "ea61688b", "c3e3eb9a"] - expected_ip_addresses = ["127.0.0.1", "192.168.178.22", "178.19.176.201", - "131.2.211.71", "127.0.0.1", "178.19.176.201", - "192.168.178.22", "131.2.211.71"] + expected_ip_addresses = ["127.0.0.1", "192.168.178.255", "255.255.255.255", + "100.100.100.100", "127.0.0.1", "255.255.255.255", + "192.168.178.255", "100.100.100.100"] for msg, peer_hash, ip_addr in zip( activity_messages, From c78c7cead29e9aee0077fb759f72b46d7269fe75 Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Tue, 11 May 2021 10:49:09 +0200 Subject: [PATCH 05/13] Add Keepalive events to remote harvester events To detect simultanous failure of all remote harvesters (in case of a network failure) PR #83 Feature #65 --- .../handlers/farmer_server_handler.py | 6 +++++ .../handlers/test_farmer_server_handler.py | 27 ++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/chia_log/handlers/farmer_server_handler.py b/src/chia_log/handlers/farmer_server_handler.py index 551832b..887e170 100644 --- a/src/chia_log/handlers/farmer_server_handler.py +++ b/src/chia_log/handlers/farmer_server_handler.py @@ -34,6 +34,12 @@ def handle(self, logs: str, stats_manager: Optional[StatsManager] = None) -> Lis if len(activity_messages) > 0: logging.debug(f"Parsed {len(activity_messages)} farmer_server messages") + events.append( + # EventService.HARVESTER because this keepalive is indicating a harvester activity + Event( + type=EventType.KEEPALIVE, priority=EventPriority.NORMAL, service=EventService.HARVESTER, message="" + ) + ) # Run messages through all condition checkers for msg in activity_messages: diff --git a/tests/chia_log/handlers/test_farmer_server_handler.py b/tests/chia_log/handlers/test_farmer_server_handler.py index 206f494..8ac9347 100644 --- a/tests/chia_log/handlers/test_farmer_server_handler.py +++ b/tests/chia_log/handlers/test_farmer_server_handler.py @@ -18,25 +18,32 @@ def testNominal(self): for log in logs: events = self.handler.handle(log) - self.assertEqual(len(events), 0, "Not expecting any events") + self.assertEqual(len(events), 1, "Only expecting 1 event for keep-alive") + self.assertEqual(events[0].type, EventType.KEEPALIVE, "Unexpected type") + self.assertEqual(events[0].priority, EventPriority.NORMAL, "Unexpected priority") + self.assertEqual(events[0].service, EventService.HARVESTER, "Unexpected service") def testDisconnectedHarvester(self): with open(self.example_logs_path / "disappearing_harvester.txt") as f: logs = f.readlines() - expected_messages = [ - "Remote harvester offline: 255.255.255.255 did not participate for 385 seconds!", - ] + expected_message = "Remote harvester offline: 255.255.255.255 did not participate for 385 seconds!" + expected_number_events = [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1] checked = 0 - for log in logs: + for log, number_events in zip(logs, expected_number_events): events = self.handler.handle(log) if len(events) > 0: - self.assertEqual(len(events), 1, "Expected a single event") - self.assertEqual(events[0].type, EventType.USER, "Unexpected type") - self.assertEqual(events[0].priority, EventPriority.HIGH, "Unexpected priority") - self.assertEqual(events[0].service, EventService.FARMER, "Unexpected service") - self.assertEqual(events[0].message, expected_messages[checked], "Unexpected message") + print(len(events)) + self.assertEqual(len(events), number_events, "Unexpected number of events") + self.assertEqual(events[0].type, EventType.KEEPALIVE, "Unexpected type") + self.assertEqual(events[0].priority, EventPriority.NORMAL, "Unexpected priority") + self.assertEqual(events[0].service, EventService.HARVESTER, "Unexpected service") + if len(events) == 2: + self.assertEqual(events[1].type, EventType.USER, "Unexpected type") + self.assertEqual(events[1].priority, EventPriority.HIGH, "Unexpected priority") + self.assertEqual(events[1].service, EventService.FARMER, "Unexpected service") + self.assertEqual(events[1].message, expected_message, "Unexpected message") checked += 1 From de4efa256f36920fbe3a87f79aacbe894f6e720b Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Thu, 29 Apr 2021 12:33:52 +0200 Subject: [PATCH 06/13] Add farmer_server parser: #65 --- src/chia_log/parsers/farmer_server_parser.py | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/chia_log/parsers/farmer_server_parser.py diff --git a/src/chia_log/parsers/farmer_server_parser.py b/src/chia_log/parsers/farmer_server_parser.py new file mode 100644 index 0000000..2a1bf28 --- /dev/null +++ b/src/chia_log/parsers/farmer_server_parser.py @@ -0,0 +1,51 @@ +# std +import re +import logging +from dataclasses import dataclass +from datetime import datetime +from typing import List + +# lib +from dateutil import parser as dateutil_parser + + +@dataclass +class FarmerServerMessage: + """Parsed information from farmer_server logs""" + + timestamp: datetime + peer_hash: str + + +class FarmerServerParser: + """This class can parse info log messages from the chia farmer_server + + You need to have enabled "log_level: INFO" in your chia config.yaml + The chia config.yaml is usually under ~/.chia/mainnet/config/config.yaml + """ + + def __init__(self): + logging.info("Enabled parser for farmer_server activity - peer infos.") + self._regex = re.compile( + r"([0-9:.]*) farmer farmer_server\s*: INFO\s* <\- farming_info " + r"from peer ([0-9a-z.]*) .*" + ) + + def parse(self, logs: str) -> List[FarmerServerMessage]: + """Parses all farmer_server activity messages from a bunch of logs + + :param logs: String of logs - can be multi-line + :returns: A list of parsed messages - can be empty + """ + + parsed_messages = [] + matches = self._regex.findall(logs) + for match in matches: + parsed_messages.append( + FarmerServerMessage( + timestamp=dateutil_parser.parse(match[0]), + peer_hash=match[1], + ) + ) + + return parsed_messages From 54a77e04851d5bbffcd55043e51a36cb906109c5 Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Wed, 5 May 2021 15:42:48 +0200 Subject: [PATCH 07/13] Rebasing with 0.5.0 --- .../handlers/condition_checkers/__init__.py | 7 ++ .../remote_harvester_activity.py | 67 +++++++++++++++++++ .../handlers/farmer_server_handler.py | 54 +++++++++++++++ src/chia_log/log_handler.py | 3 +- 4 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 src/chia_log/handlers/condition_checkers/remote_harvester_activity.py create mode 100644 src/chia_log/handlers/farmer_server_handler.py diff --git a/src/chia_log/handlers/condition_checkers/__init__.py b/src/chia_log/handlers/condition_checkers/__init__.py index 8d41173..476c740 100644 --- a/src/chia_log/handlers/condition_checkers/__init__.py +++ b/src/chia_log/handlers/condition_checkers/__init__.py @@ -6,6 +6,7 @@ from src.notifier import Event from ...parsers.finished_signage_point_parser import FinishedSignagePointMessage from ...parsers.harvester_activity_parser import HarvesterActivityMessage +from ...parsers.farmer_server_parser import FarmerServerMessage class FinishedSignageConditionChecker(ABC): @@ -18,3 +19,9 @@ class HarvesterConditionChecker(ABC): @abstractmethod def check(self, obj: HarvesterActivityMessage) -> Optional[Event]: pass + + +class FarmerConditionChecker(ABC): + @abstractmethod + def check(self, obj: FarmerServerMessage) -> Optional[Event]: + pass diff --git a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py new file mode 100644 index 0000000..5d7e840 --- /dev/null +++ b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py @@ -0,0 +1,67 @@ +# std +import logging +from typing import Optional +from dataclasses import dataclass +from datetime import datetime + +# project +from src.notifier import Event, EventService, EventType, EventPriority +from . import FarmerConditionChecker +from ...parsers.farmer_server_parser import FarmerServerMessage + + +@dataclass +class RemoteHarvester: + """Record of last activity""" + last_activity: datetime + peer_hash: str + +class RemoteHarvesterActivity(FarmerConditionChecker): + """The remote harvesters connected to the farmer_server + are not expected to disappear. Disappearing remote harvesters + indicate network or remote system issues. + """ + + def __init__(self): + logging.info("Enabled check for disappearing remote harvesters.") + self._warning_threshold = 300 + self._remote_harvesters = [] + + def check(self, obj: FarmerServerMessage) -> Optional[Event]: + #update last_activity + is_new_harvester = true + for i, remote_harvester in enumerate(self._remote_harvesters): + if remote_harvester.peer_hash == obj.peer_hash: + is_new_harvester = false + self._remote_harvesters[i].last_activity = obj.timestamp + self._remote_harvesters[i].warning_issued = false + logging.info(f"Updating Peer Harvester {obj.peer_hash}") + break + + if is_new_harvester: + new_harvester = RemoteHarvester() + new_harvester.last_activity = obj.timestamp + new_harvester.peer_hash = obj.peer_hash + new_harvester.warning_issued = false + self._remote_harvesters.append(new_harvester) + logging.info(f"New Peer Harvester {obj.peer_hash}") + + + event = None + for i, remote_harvester in enumerate(self._remote_harvesters): + seconds_since_last = (obj.timestamp - remote_harvester.last_activity).seconds + if last_activity > self._warning_threshold: + message = ( + f"Remote Harvester Offline: {remote_harvester.peer_hash} " + f"did not participate for {seconds_since_last} seconds!." + ) + logging.warning(message) + event = Event( + type=EventType.USER, priority=EventPriority.NORMAL, service=EventService.FARMER, message=message + ) + + # Remove offline harvester + self._remote_harvesters.pop(i) + break + + return event diff --git a/src/chia_log/handlers/farmer_server_handler.py b/src/chia_log/handlers/farmer_server_handler.py new file mode 100644 index 0000000..6cd2d95 --- /dev/null +++ b/src/chia_log/handlers/farmer_server_handler.py @@ -0,0 +1,54 @@ +# std +import logging +from typing import List, Optional + +# project +from . import LogHandler +from ..parsers.farmer_server_parser import FarmerServerParser +from .condition_checkers import FarmerConditionChecker +from .condition_checkers.remote_harvester_activity import RemoteHarvesterActivity +from .daily_stats.stats_manager import StatsManager +from src.notifier import Event, EventService, EventType, EventPriority + + +class FarmerServerHandler(LogHandler): + """This handler parses all logs indicating farmer_server + activity and participation of remote harvester in challenges. + It holds a list of condition checkers that are evaluated for + each event to ensure that farming is going smoothly. + """ + + def __init__(self): + self._parser = FarmerServerParser() + self._cond_checkers: List[FarmerConditionChecker] = [ + RemoteHarvesterActivity(), + ] + + def handle(self, logs: str, stats_manager: Optional[StatsManager] = None) -> List[Event]: + """Process incoming logs, check all conditions + and return a list of notable events. + """ + + events = [] + activity_messages = self._parser.parse(logs) + # if stats_manager: + # stats_manager.consume_farmer_messages(activity_messages) + + # Create a keep-alive event if any logs indicating + # activity have been successfully parsed + if len(activity_messages) > 0: + logging.debug(f"Parsed {len(activity_messages)} activity messages") + events.append( + Event( + type=EventType.KEEPALIVE, priority=EventPriority.NORMAL, service=EventService.FARMER, message="" + ) + ) + + # Run messages through all condition checkers + for msg in activity_messages: + for checker in self._cond_checkers: + event = checker.check(msg) + if event: + events.append(event) + + return events diff --git a/src/chia_log/log_handler.py b/src/chia_log/log_handler.py index aa12512..91a89ed 100644 --- a/src/chia_log/log_handler.py +++ b/src/chia_log/log_handler.py @@ -6,6 +6,7 @@ from src.chia_log.handlers.harvester_activity_handler import HarvesterActivityHandler from src.chia_log.handlers.finished_signage_point_handler import FinishedSignagePointHandler from src.chia_log.handlers.wallet_added_coin_handler import WalletAddedCoinHandler +from src.chia_log.handlers.farmer_server_handler import FarmerServerHandler from src.chia_log.log_consumer import LogConsumerSubscriber, LogConsumer from src.notifier.notify_manager import NotifyManager @@ -29,7 +30,7 @@ def __init__( ): self._notify_manager = notify_manager self._stats_manager = stats_manager - self._handlers = [HarvesterActivityHandler(), FinishedSignagePointHandler(), WalletAddedCoinHandler()] + self._handlers = [HarvesterActivityHandler(), FinishedSignagePointHandler(), WalletAddedCoinHandler(), FarmerServerHandler()] log_consumer.subscribe(self) def consume_logs(self, logs: str): From 04dc5573e66ea3b58ea954a0d687b14c0d408ffd Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Wed, 5 May 2021 16:59:28 +0200 Subject: [PATCH 08/13] Add unit tests and bugfixes for farmer_server #65 --- .../remote_harvester_activity.py | 26 +++++----- .../handlers/farmer_server_handler.py | 7 +-- src/chia_log/parsers/farmer_server_parser.py | 4 +- .../handlers/test_farmer_server_handler.py | 44 +++++++++++++++++ .../farmer_server/disappearing_harvester.txt | 16 ++++++ tests/chia_log/logs/farmer_server/nominal.txt | 8 +++ .../parsers/test_farmer_server_parser.py | 49 +++++++++++++++++++ 7 files changed, 134 insertions(+), 20 deletions(-) create mode 100644 tests/chia_log/handlers/test_farmer_server_handler.py create mode 100644 tests/chia_log/logs/farmer_server/disappearing_harvester.txt create mode 100644 tests/chia_log/logs/farmer_server/nominal.txt create mode 100644 tests/chia_log/parsers/test_farmer_server_parser.py diff --git a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py index 5d7e840..52eac70 100644 --- a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py +++ b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py @@ -15,6 +15,7 @@ class RemoteHarvester: """Record of last activity""" last_activity: datetime peer_hash: str + ip_addr: str class RemoteHarvesterActivity(FarmerConditionChecker): """The remote harvesters connected to the farmer_server @@ -29,35 +30,34 @@ def __init__(self): def check(self, obj: FarmerServerMessage) -> Optional[Event]: #update last_activity - is_new_harvester = true + is_new_harvester = True for i, remote_harvester in enumerate(self._remote_harvesters): if remote_harvester.peer_hash == obj.peer_hash: - is_new_harvester = false + is_new_harvester = False self._remote_harvesters[i].last_activity = obj.timestamp - self._remote_harvesters[i].warning_issued = false - logging.info(f"Updating Peer Harvester {obj.peer_hash}") break if is_new_harvester: - new_harvester = RemoteHarvester() - new_harvester.last_activity = obj.timestamp - new_harvester.peer_hash = obj.peer_hash - new_harvester.warning_issued = false + new_harvester = RemoteHarvester( + obj.timestamp, + obj.peer_hash, + obj.ip_addr + ) self._remote_harvesters.append(new_harvester) - logging.info(f"New Peer Harvester {obj.peer_hash}") + logging.info(f"New remote harvester: {obj.ip_addr}") event = None for i, remote_harvester in enumerate(self._remote_harvesters): seconds_since_last = (obj.timestamp - remote_harvester.last_activity).seconds - if last_activity > self._warning_threshold: + if seconds_since_last > self._warning_threshold: message = ( - f"Remote Harvester Offline: {remote_harvester.peer_hash} " - f"did not participate for {seconds_since_last} seconds!." + f"Remote harvester offline: {remote_harvester.ip_addr} " + f"did not participate for {seconds_since_last} seconds!" ) logging.warning(message) event = Event( - type=EventType.USER, priority=EventPriority.NORMAL, service=EventService.FARMER, message=message + type=EventType.USER, priority=EventPriority.HIGH, service=EventService.FARMER, message=message ) # Remove offline harvester diff --git a/src/chia_log/handlers/farmer_server_handler.py b/src/chia_log/handlers/farmer_server_handler.py index 6cd2d95..9612006 100644 --- a/src/chia_log/handlers/farmer_server_handler.py +++ b/src/chia_log/handlers/farmer_server_handler.py @@ -37,12 +37,7 @@ def handle(self, logs: str, stats_manager: Optional[StatsManager] = None) -> Lis # Create a keep-alive event if any logs indicating # activity have been successfully parsed if len(activity_messages) > 0: - logging.debug(f"Parsed {len(activity_messages)} activity messages") - events.append( - Event( - type=EventType.KEEPALIVE, priority=EventPriority.NORMAL, service=EventService.FARMER, message="" - ) - ) + logging.debug(f"Parsed {len(activity_messages)} farmer_server messages") # Run messages through all condition checkers for msg in activity_messages: diff --git a/src/chia_log/parsers/farmer_server_parser.py b/src/chia_log/parsers/farmer_server_parser.py index 2a1bf28..5fd26b6 100644 --- a/src/chia_log/parsers/farmer_server_parser.py +++ b/src/chia_log/parsers/farmer_server_parser.py @@ -15,6 +15,7 @@ class FarmerServerMessage: timestamp: datetime peer_hash: str + ip_addr: str class FarmerServerParser: @@ -28,7 +29,7 @@ def __init__(self): logging.info("Enabled parser for farmer_server activity - peer infos.") self._regex = re.compile( r"([0-9:.]*) farmer farmer_server\s*: INFO\s* <\- farming_info " - r"from peer ([0-9a-z.]*) .*" + r"from peer ([0-9a-z.]*) ([0-9\.:a-f]*)" ) def parse(self, logs: str) -> List[FarmerServerMessage]: @@ -45,6 +46,7 @@ def parse(self, logs: str) -> List[FarmerServerMessage]: FarmerServerMessage( timestamp=dateutil_parser.parse(match[0]), peer_hash=match[1], + ip_addr=match[2] ) ) diff --git a/tests/chia_log/handlers/test_farmer_server_handler.py b/tests/chia_log/handlers/test_farmer_server_handler.py new file mode 100644 index 0000000..83b2111 --- /dev/null +++ b/tests/chia_log/handlers/test_farmer_server_handler.py @@ -0,0 +1,44 @@ +# std +import unittest +from pathlib import Path + +# project +from src.chia_log.handlers import farmer_server_handler +from src.notifier import EventType, EventService, EventPriority + + +class TestFarmerServerHandler(unittest.TestCase): + def setUp(self) -> None: + self.handler = farmer_server_handler.FarmerServerHandler() + self.example_logs_path = Path(__file__).resolve().parents[1] / "logs/farmer_server" + + def testNominal(self): + with open(self.example_logs_path / "nominal.txt") as f: + logs = f.readlines() + + for log in logs: + events = self.handler.handle(log) + self.assertEqual(len(events), 0, "Not expecting any events") + + def testSkippedSignagePoints(self): + with open(self.example_logs_path / "disappearing_harvester.txt") as f: + logs = f.readlines() + + expected_messages = [ + "Remote harvester offline: 178.19.176.201 did not participate for 385 seconds!", + ] + + checked = 0 + for log in logs: + events = self.handler.handle(log) + if len(events) > 0: + self.assertEqual(len(events), 1, "Expected a single event") + self.assertEqual(events[0].type, EventType.USER, "Unexpected type") + self.assertEqual(events[0].priority, EventPriority.HIGH, "Unexpected priority") + self.assertEqual(events[0].service, EventService.FARMER, "Unexpected service") + self.assertEqual(events[0].message, expected_messages[checked], "Unexpected message") + checked += 1 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/chia_log/logs/farmer_server/disappearing_harvester.txt b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt new file mode 100644 index 0000000..9c94a9c --- /dev/null +++ b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt @@ -0,0 +1,16 @@ +2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 +2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:04:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:04:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:04:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:06:24.253 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:06:24.274 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:06:24.297 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:08:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:08:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:08:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:10:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:10:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:10:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 diff --git a/tests/chia_log/logs/farmer_server/nominal.txt b/tests/chia_log/logs/farmer_server/nominal.txt new file mode 100644 index 0000000..0b8228c --- /dev/null +++ b/tests/chia_log/logs/farmer_server/nominal.txt @@ -0,0 +1,8 @@ +2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 +2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:02:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:02:14.995 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 +2021-05-05T16:02:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:02:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 diff --git a/tests/chia_log/parsers/test_farmer_server_parser.py b/tests/chia_log/parsers/test_farmer_server_parser.py new file mode 100644 index 0000000..3c0eaa2 --- /dev/null +++ b/tests/chia_log/parsers/test_farmer_server_parser.py @@ -0,0 +1,49 @@ +# std +import unittest +from pathlib import Path + +# project +from src.chia_log.parsers import farmer_server_parser + + +class TestFarmerServerParser(unittest.TestCase): + def setUp(self) -> None: + self.parser = farmer_server_parser.FarmerServerParser() + self.example_logs_path = Path(__file__).resolve().parents[1] / "logs/farmer_server" + with open(self.example_logs_path / "nominal.txt") as f: + self.nominal_logs = f.read() + + def tearDown(self) -> None: + pass + + def testBasicParsing(self): + for logs in [self.nominal_logs]: + + # Check that important fields are correctly parsed + activity_messages = self.parser.parse(logs) + self.assertNotEqual(len(activity_messages), 0, "No log messages found") + + expected_peer_hashes = ["20388420", "ea61688b", "fe1eb730", "c3e3eb9a", + "20388420", "fe1eb730", "ea61688b", "c3e3eb9a"] + expected_ip_addresses = ["127.0.0.1", "192.168.178.22", "178.19.176.201", + "131.2.211.71", "127.0.0.1", "178.19.176.201", + "192.168.178.22", "131.2.211.71"] + + for msg, peer_hash, ip_addr in zip( + activity_messages, + expected_peer_hashes, + expected_ip_addresses, + ): + self.assertEqual(msg.peer_hash, peer_hash, "Peer hash don't match") + self.assertEqual(msg.ip_addr, ip_addr, "IP Addr don't match") + + # Check arithmetic with parsed timestamps works + prev_timestamp = activity_messages[0].timestamp + for msg in activity_messages[1:]: + seconds_since_last_activity = (msg.timestamp - prev_timestamp).seconds + self.assertLess(seconds_since_last_activity, 10, "Unexpected duration between farmer_server events") + prev_timestamp = msg.timestamp + + +if __name__ == "__main__": + unittest.main() From ac549f64acbfb220cdeeac495e3e288652629c75 Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Sun, 9 May 2021 11:53:40 +0200 Subject: [PATCH 09/13] Fix issues according to PR Review #83 Storing remote harvesters in dict Store ip_addr again if changed Delete deactivated code Change test ip addresses to more dummy looking ones --- .../remote_harvester_activity.py | 22 ++++++++----------- .../handlers/farmer_server_handler.py | 4 ---- .../handlers/test_farmer_server_handler.py | 4 ++-- .../farmer_server/disappearing_harvester.txt | 22 +++++++++---------- tests/chia_log/logs/farmer_server/nominal.txt | 12 +++++----- .../parsers/test_farmer_server_parser.py | 6 ++--- 6 files changed, 31 insertions(+), 39 deletions(-) diff --git a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py index 52eac70..22941f0 100644 --- a/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py +++ b/src/chia_log/handlers/condition_checkers/remote_harvester_activity.py @@ -26,29 +26,25 @@ class RemoteHarvesterActivity(FarmerConditionChecker): def __init__(self): logging.info("Enabled check for disappearing remote harvesters.") self._warning_threshold = 300 - self._remote_harvesters = [] + self._remote_harvesters = {} def check(self, obj: FarmerServerMessage) -> Optional[Event]: #update last_activity - is_new_harvester = True - for i, remote_harvester in enumerate(self._remote_harvesters): - if remote_harvester.peer_hash == obj.peer_hash: - is_new_harvester = False - self._remote_harvesters[i].last_activity = obj.timestamp - break - - if is_new_harvester: - new_harvester = RemoteHarvester( + if self._remote_harvesters.get(obj.peer_hash, False): + self._remote_harvesters[obj.peer_hash].last_activity = obj.timestamp + self._remote_harvesters[obj.peer_hash].ip_addr = obj.ip_addr + else: + self._remote_harvesters[obj.peer_hash] = RemoteHarvester( obj.timestamp, obj.peer_hash, obj.ip_addr ) - self._remote_harvesters.append(new_harvester) logging.info(f"New remote harvester: {obj.ip_addr}") event = None - for i, remote_harvester in enumerate(self._remote_harvesters): + for peer_hash in self._remote_harvesters: + remote_harvester = self._remote_harvesters[peer_hash] seconds_since_last = (obj.timestamp - remote_harvester.last_activity).seconds if seconds_since_last > self._warning_threshold: message = ( @@ -61,7 +57,7 @@ def check(self, obj: FarmerServerMessage) -> Optional[Event]: ) # Remove offline harvester - self._remote_harvesters.pop(i) + self._remote_harvesters.pop(peer_hash) break return event diff --git a/src/chia_log/handlers/farmer_server_handler.py b/src/chia_log/handlers/farmer_server_handler.py index 9612006..551832b 100644 --- a/src/chia_log/handlers/farmer_server_handler.py +++ b/src/chia_log/handlers/farmer_server_handler.py @@ -31,11 +31,7 @@ def handle(self, logs: str, stats_manager: Optional[StatsManager] = None) -> Lis events = [] activity_messages = self._parser.parse(logs) - # if stats_manager: - # stats_manager.consume_farmer_messages(activity_messages) - # Create a keep-alive event if any logs indicating - # activity have been successfully parsed if len(activity_messages) > 0: logging.debug(f"Parsed {len(activity_messages)} farmer_server messages") diff --git a/tests/chia_log/handlers/test_farmer_server_handler.py b/tests/chia_log/handlers/test_farmer_server_handler.py index 83b2111..206f494 100644 --- a/tests/chia_log/handlers/test_farmer_server_handler.py +++ b/tests/chia_log/handlers/test_farmer_server_handler.py @@ -20,12 +20,12 @@ def testNominal(self): events = self.handler.handle(log) self.assertEqual(len(events), 0, "Not expecting any events") - def testSkippedSignagePoints(self): + def testDisconnectedHarvester(self): with open(self.example_logs_path / "disappearing_harvester.txt") as f: logs = f.readlines() expected_messages = [ - "Remote harvester offline: 178.19.176.201 did not participate for 385 seconds!", + "Remote harvester offline: 255.255.255.255 did not participate for 385 seconds!", ] checked = 0 diff --git a/tests/chia_log/logs/farmer_server/disappearing_harvester.txt b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt index 9c94a9c..cb7ca83 100644 --- a/tests/chia_log/logs/farmer_server/disappearing_harvester.txt +++ b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt @@ -1,16 +1,16 @@ 2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 -2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 255.255.255.255 +2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 2021-05-05T16:04:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:04:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:04:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:04:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:04:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 2021-05-05T16:06:24.253 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:06:24.274 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:06:24.297 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:06:24.274 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:06:24.297 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 2021-05-05T16:08:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:08:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 -2021-05-05T16:08:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:08:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:08:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 2021-05-05T16:10:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:10:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 -2021-05-05T16:10:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 +2021-05-05T16:10:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:10:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 diff --git a/tests/chia_log/logs/farmer_server/nominal.txt b/tests/chia_log/logs/farmer_server/nominal.txt index 0b8228c..8c54d27 100644 --- a/tests/chia_log/logs/farmer_server/nominal.txt +++ b/tests/chia_log/logs/farmer_server/nominal.txt @@ -1,8 +1,8 @@ 2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 -2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 255.255.255.255 +2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 2021-05-05T16:02:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:02:14.995 farmer farmer_server : INFO <- farming_info from peer fe1eb730 178.19.176.201 -2021-05-05T16:02:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.22 -2021-05-05T16:02:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 131.2.211.71 +2021-05-05T16:02:14.995 farmer farmer_server : INFO <- farming_info from peer fe1eb730 255.255.255.255 +2021-05-05T16:02:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 diff --git a/tests/chia_log/parsers/test_farmer_server_parser.py b/tests/chia_log/parsers/test_farmer_server_parser.py index 3c0eaa2..9d06334 100644 --- a/tests/chia_log/parsers/test_farmer_server_parser.py +++ b/tests/chia_log/parsers/test_farmer_server_parser.py @@ -25,9 +25,9 @@ def testBasicParsing(self): expected_peer_hashes = ["20388420", "ea61688b", "fe1eb730", "c3e3eb9a", "20388420", "fe1eb730", "ea61688b", "c3e3eb9a"] - expected_ip_addresses = ["127.0.0.1", "192.168.178.22", "178.19.176.201", - "131.2.211.71", "127.0.0.1", "178.19.176.201", - "192.168.178.22", "131.2.211.71"] + expected_ip_addresses = ["127.0.0.1", "192.168.178.255", "255.255.255.255", + "100.100.100.100", "127.0.0.1", "255.255.255.255", + "192.168.178.255", "100.100.100.100"] for msg, peer_hash, ip_addr in zip( activity_messages, From 6b6d40643a472445f5ca7054557ea99e5eb8ab25 Mon Sep 17 00:00:00 2001 From: cbacon93 Date: Tue, 11 May 2021 10:49:09 +0200 Subject: [PATCH 10/13] Add Keepalive events to remote harvester events To detect simultanous failure of all remote harvesters (in case of a network failure) PR #83 Feature #65 --- .../handlers/farmer_server_handler.py | 6 +++++ .../handlers/test_farmer_server_handler.py | 27 ++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/chia_log/handlers/farmer_server_handler.py b/src/chia_log/handlers/farmer_server_handler.py index 551832b..887e170 100644 --- a/src/chia_log/handlers/farmer_server_handler.py +++ b/src/chia_log/handlers/farmer_server_handler.py @@ -34,6 +34,12 @@ def handle(self, logs: str, stats_manager: Optional[StatsManager] = None) -> Lis if len(activity_messages) > 0: logging.debug(f"Parsed {len(activity_messages)} farmer_server messages") + events.append( + # EventService.HARVESTER because this keepalive is indicating a harvester activity + Event( + type=EventType.KEEPALIVE, priority=EventPriority.NORMAL, service=EventService.HARVESTER, message="" + ) + ) # Run messages through all condition checkers for msg in activity_messages: diff --git a/tests/chia_log/handlers/test_farmer_server_handler.py b/tests/chia_log/handlers/test_farmer_server_handler.py index 206f494..8ac9347 100644 --- a/tests/chia_log/handlers/test_farmer_server_handler.py +++ b/tests/chia_log/handlers/test_farmer_server_handler.py @@ -18,25 +18,32 @@ def testNominal(self): for log in logs: events = self.handler.handle(log) - self.assertEqual(len(events), 0, "Not expecting any events") + self.assertEqual(len(events), 1, "Only expecting 1 event for keep-alive") + self.assertEqual(events[0].type, EventType.KEEPALIVE, "Unexpected type") + self.assertEqual(events[0].priority, EventPriority.NORMAL, "Unexpected priority") + self.assertEqual(events[0].service, EventService.HARVESTER, "Unexpected service") def testDisconnectedHarvester(self): with open(self.example_logs_path / "disappearing_harvester.txt") as f: logs = f.readlines() - expected_messages = [ - "Remote harvester offline: 255.255.255.255 did not participate for 385 seconds!", - ] + expected_message = "Remote harvester offline: 255.255.255.255 did not participate for 385 seconds!" + expected_number_events = [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1] checked = 0 - for log in logs: + for log, number_events in zip(logs, expected_number_events): events = self.handler.handle(log) if len(events) > 0: - self.assertEqual(len(events), 1, "Expected a single event") - self.assertEqual(events[0].type, EventType.USER, "Unexpected type") - self.assertEqual(events[0].priority, EventPriority.HIGH, "Unexpected priority") - self.assertEqual(events[0].service, EventService.FARMER, "Unexpected service") - self.assertEqual(events[0].message, expected_messages[checked], "Unexpected message") + print(len(events)) + self.assertEqual(len(events), number_events, "Unexpected number of events") + self.assertEqual(events[0].type, EventType.KEEPALIVE, "Unexpected type") + self.assertEqual(events[0].priority, EventPriority.NORMAL, "Unexpected priority") + self.assertEqual(events[0].service, EventService.HARVESTER, "Unexpected service") + if len(events) == 2: + self.assertEqual(events[1].type, EventType.USER, "Unexpected type") + self.assertEqual(events[1].priority, EventPriority.HIGH, "Unexpected priority") + self.assertEqual(events[1].service, EventService.FARMER, "Unexpected service") + self.assertEqual(events[1].message, expected_message, "Unexpected message") checked += 1 From 96701601f4e2d9e832ab3bc2a329cf6697cd577a Mon Sep 17 00:00:00 2001 From: Marcel Haupt Date: Sat, 22 May 2021 07:58:07 +0200 Subject: [PATCH 11/13] Update parser for chia 1.1.6 --- src/chia_log/parsers/farmer_server_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/chia_log/parsers/farmer_server_parser.py b/src/chia_log/parsers/farmer_server_parser.py index 5fd26b6..d1eceb1 100644 --- a/src/chia_log/parsers/farmer_server_parser.py +++ b/src/chia_log/parsers/farmer_server_parser.py @@ -21,14 +21,14 @@ class FarmerServerMessage: class FarmerServerParser: """This class can parse info log messages from the chia farmer_server - You need to have enabled "log_level: INFO" in your chia config.yaml + You need to have enabled "log_level: DEBUG" in your chia config.yaml The chia config.yaml is usually under ~/.chia/mainnet/config/config.yaml """ def __init__(self): logging.info("Enabled parser for farmer_server activity - peer infos.") self._regex = re.compile( - r"([0-9:.]*) farmer farmer_server\s*: INFO\s* <\- farming_info " + r"([0-9:.]*) farmer farmer_server\s*: DEBUG\s* <\- farming_info " r"from peer ([0-9a-z.]*) ([0-9\.:a-f]*)" ) From d57b079bda6fdab2b87e5409bbc9a2794fc5d5a3 Mon Sep 17 00:00:00 2001 From: Marcel Haupt Date: Sat, 22 May 2021 07:58:56 +0200 Subject: [PATCH 12/13] Update disappearing_harvester.txt --- .../farmer_server/disappearing_harvester.txt | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/chia_log/logs/farmer_server/disappearing_harvester.txt b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt index cb7ca83..2b1d355 100644 --- a/tests/chia_log/logs/farmer_server/disappearing_harvester.txt +++ b/tests/chia_log/logs/farmer_server/disappearing_harvester.txt @@ -1,16 +1,16 @@ -2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 -2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 255.255.255.255 -2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 -2021-05-05T16:04:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:04:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 -2021-05-05T16:04:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 -2021-05-05T16:06:24.253 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:06:24.274 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 -2021-05-05T16:06:24.297 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 -2021-05-05T16:08:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:08:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 -2021-05-05T16:08:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 -2021-05-05T16:10:32.378 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:10:32.420 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 -2021-05-05T16:10:32.660 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:06.632 farmer farmer_server : DEBUG <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:02:06.656 farmer farmer_server : DEBUG <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:06.660 farmer farmer_server : DEBUG <- farming_info from peer fe1eb730 255.255.255.255 +2021-05-05T16:02:06.675 farmer farmer_server : DEBUG <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:04:14.938 farmer farmer_server : DEBUG <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:04:15.169 farmer farmer_server : DEBUG <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:04:16.951 farmer farmer_server : DEBUG <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:06:24.253 farmer farmer_server : DEBUG <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:06:24.274 farmer farmer_server : DEBUG <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:06:24.297 farmer farmer_server : DEBUG <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:08:32.378 farmer farmer_server : DEBUG <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:08:32.420 farmer farmer_server : DEBUG <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:08:32.660 farmer farmer_server : DEBUG <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:10:32.378 farmer farmer_server : DEBUG <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:10:32.420 farmer farmer_server : DEBUG <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:10:32.660 farmer farmer_server : DEBUG <- farming_info from peer ea61688b 192.168.178.255 From cf3b44725740eedc61dcbf98a70beaaccc8437e0 Mon Sep 17 00:00:00 2001 From: Marcel Haupt Date: Sat, 22 May 2021 07:59:15 +0200 Subject: [PATCH 13/13] Update nominal.txt --- tests/chia_log/logs/farmer_server/nominal.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/chia_log/logs/farmer_server/nominal.txt b/tests/chia_log/logs/farmer_server/nominal.txt index 8c54d27..0242f72 100644 --- a/tests/chia_log/logs/farmer_server/nominal.txt +++ b/tests/chia_log/logs/farmer_server/nominal.txt @@ -1,8 +1,8 @@ -2021-05-05T16:02:06.632 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:02:06.656 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 -2021-05-05T16:02:06.660 farmer farmer_server : INFO <- farming_info from peer fe1eb730 255.255.255.255 -2021-05-05T16:02:06.675 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 -2021-05-05T16:02:14.938 farmer farmer_server : INFO <- farming_info from peer 20388420 127.0.0.1 -2021-05-05T16:02:14.995 farmer farmer_server : INFO <- farming_info from peer fe1eb730 255.255.255.255 -2021-05-05T16:02:15.169 farmer farmer_server : INFO <- farming_info from peer ea61688b 192.168.178.255 -2021-05-05T16:02:16.951 farmer farmer_server : INFO <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:02:06.632 farmer farmer_server : DEBUG <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:02:06.656 farmer farmer_server : DEBUG <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:06.660 farmer farmer_server : DEBUG <- farming_info from peer fe1eb730 255.255.255.255 +2021-05-05T16:02:06.675 farmer farmer_server : DEBUG <- farming_info from peer c3e3eb9a 100.100.100.100 +2021-05-05T16:02:14.938 farmer farmer_server : DEBUG <- farming_info from peer 20388420 127.0.0.1 +2021-05-05T16:02:14.995 farmer farmer_server : DEBUG <- farming_info from peer fe1eb730 255.255.255.255 +2021-05-05T16:02:15.169 farmer farmer_server : DEBUG <- farming_info from peer ea61688b 192.168.178.255 +2021-05-05T16:02:16.951 farmer farmer_server : DEBUG <- farming_info from peer c3e3eb9a 100.100.100.100