Fixing VMSS Re-Image 7-Day Timer (#1616)

* Fixing VMSS Re-Image 7-Day Timer

* Updating use of TimeStamp to created_at

* Renaming.

* Updating field name.

* Removing test chagne.

* Updating query to work if init_at entry does not exist yet.

* Changing timer for testing.

* Adding field comment.

* Formatting models.py

* Fixing where save is called.

* Adidng logging.

* Removing logging. Ready for merge.

* Update src/pytypes/onefuzztypes/models.py

Co-authored-by: Joe Ranweiler <joe@lemma.co>

* Formatting.

* Updating datetime.

* Testing after datetime change.

* Removing test.

Co-authored-by: nharper285 <nharper285@gmail.com>
Co-authored-by: Joe Ranweiler <joe@lemma.co>

src/api-service/__app__/onefuzzlib/agent_events.py

@@ -3,6 +3,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+import datetime
 import logging
 from typing import Optional, cast
 from uuid import UUID
@@ -75,7 +76,9 @@ def on_state_update(
         # they send 'init' with reimage_requested, it's because the node was reimaged
         # successfully.
         node.reimage_requested = False
+        node.initialized_at = datetime.datetime.now(datetime.timezone.utc)
         node.set_state(state)
+
         return None
 
     logging.info("node state update: %s from:%s to:%s", machine_id, node.state, state)

src/api-service/__app__/onefuzzlib/workers/nodes.py

@@ -35,7 +35,7 @@
 from .shrink_queue import ShrinkQueue
 
 NODE_EXPIRATION_TIME: datetime.timedelta = datetime.timedelta(hours=1)
-NODE_REIMAGE_TIME: datetime.timedelta = datetime.timedelta(days=7)
+NODE_REIMAGE_TIME: datetime.timedelta = datetime.timedelta(days=6)
 
 # Future work:
 #
@@ -367,8 +367,8 @@ def is_outdated(self) -> bool:
     def is_too_old(self) -> bool:
         return (
             self.scaleset_id is not None
-            and self.timestamp is not None
-            and self.timestamp
+            and self.initialized_at is not None
+            and self.initialized_at
             < datetime.datetime.now(datetime.timezone.utc) - NODE_REIMAGE_TIME
         )
 
@@ -453,15 +453,18 @@ def reimage_long_lived_nodes(cls, scaleset_id: UUID) -> None:
         reasonably up-to-date with OS patches without disrupting running
         fuzzing tasks with patch reboot cycles.
         """
-        time_filter = "Timestamp lt datetime'%s'" % (
-            (datetime.datetime.utcnow() - NODE_REIMAGE_TIME).isoformat()
+        time_filter = "not (initialized_at ge datetime'%s')" % (
+            (
+                datetime.datetime.now(datetime.timezone.utc) - NODE_REIMAGE_TIME
+            ).isoformat()
         )
-        for node in cls.search(
+        reimage_nodes = cls.search(
             query={
                 "scaleset_id": [scaleset_id],
             },
             raw_unchecked_filter=time_filter,
-        ):
+        )
+        for node in reimage_nodes:
             if node.debug_keep_node:
                 logging.info(
                     "removing debug_keep_node for expired node. "

src/pytypes/onefuzztypes/models.py

@@ -568,6 +568,9 @@ class NodeCommandEnvelope(BaseModel):
 
 class Node(BaseModel):
     timestamp: Optional[datetime] = Field(alias="Timestamp")
+
+    # Set only once, when a node is initialized.
+    initialized_at: Optional[datetime]
     pool_name: PoolName
     pool_id: Optional[UUID]
     machine_id: UUID