Merge pull request #270 from valschneider/caching/bugfix

Caching: factorizing / toughening

tests/test_caching.py

@@ -14,44 +14,49 @@
 #
 
 import os
+import json
 import shutil
 import sys
 import unittest
 import utils_tests
 import trappy
 from trappy.ftrace import GenericFTrace
+from trappy.systrace import SysTrace
 
 class TestCaching(utils_tests.SetupDirectory):
     def __init__(self, *args, **kwargs):
         super(TestCaching, self).__init__(
             [("trace_sched.txt", "trace.txt"),
-             ("trace_sched.txt", "trace.raw.txt")],
+             ("trace_sched.txt", "trace.raw.txt"),
+             ("trace_systrace.html", "trace.html")],
             *args,
             **kwargs)
 
     def test_cache_created(self):
         """Test cache creation when enabled"""
         GenericFTrace.disable_cache = False
-        trace = trappy.FTrace()
+        traces = (trappy.FTrace(), trappy.SysTrace(path='./trace.html'))
 
-        trace_path = os.path.abspath(trace.trace_path)
-        trace_dir = os.path.dirname(trace_path)
-        trace_file = os.path.basename(trace_path)
-        cache_dir = '.' + trace_file + '.cache'
+        for trace in traces:
+            trace_path = os.path.abspath(trace.trace_path)
+            trace_dir = os.path.dirname(trace_path)
+            trace_file = os.path.basename(trace_path)
+            cache_dir = '.' + trace_file + '.cache'
 
-        self.assertTrue(cache_dir in os.listdir(trace_dir))
+            self.assertTrue(cache_dir in os.listdir(trace_dir))
 
     def test_cache_not_created(self):
         """Test that cache should not be created when disabled """
         GenericFTrace.disable_cache = True
-        trace = trappy.FTrace()
+        traces = (trappy.FTrace(), trappy.SysTrace(path='./trace.html'))
 
-        trace_path = os.path.abspath(trace.trace_path)
-        trace_dir = os.path.dirname(trace_path)
-        trace_file = os.path.basename(trace_path)
-        cache_dir = '.' + trace_file + '.cache'
+        for trace in traces:
+            trace_path = os.path.abspath(trace.trace_path)
+            trace_dir = os.path.dirname(trace_path)
+            trace_file = os.path.basename(trace_path)
+            cache_dir = '.' + trace_file + '.cache'
 
-        self.assertFalse(cache_dir in os.listdir(trace_dir))
+            self.assertFalse(cache_dir in os.listdir(trace_dir))
 
     def test_compare_cached_vs_uncached(self):
         """ Test that the cached and uncached traces are same """
@@ -107,24 +112,30 @@ def test_invalid_cache_overwritten(self):
         src = os.path.join(utils_tests.TESTS_DIRECTORY, "trace_sched.txt.cache")
         shutil.copytree(src, cache_path)
 
-        md5_path = os.path.join(cache_path, "md5sum")
-        def read_md5sum():
-            with open(md5_path) as f:
-                return f.read()
+        metadata_path = os.path.join(cache_path, "metadata.json")
+
+        def read_metadata():
+            with open(metadata_path, "r") as f:
+                return json.load(f)
+
+        def write_md5(md5):
+            metadata = read_metadata()
+            metadata["md5sum"] = md5
+            with open(metadata_path, "w") as f:
+                json.dump(metadata, f)
+
 
         # Change 1 character of the stored checksum
-        md5sum = read_md5sum()
-        # Sorry, I guess modifying strings in Python is kind of awkward?
-        md5sum_inc = "".join(list(md5sum[:-1]) + [chr(ord(md5sum[-1]) + 1)])
-        with open(md5_path, "w") as f:
-            f.write(md5sum_inc)
+        md5sum = read_metadata()["md5sum"]
+        md5sum_inc = md5sum[:-1] + chr(ord(md5sum[-1]) + 1)
+        write_md5(md5sum_inc)
 
         # Parse a trace, this should delete and overwrite the invalidated cache
         GenericFTrace.disable_cache = False
         trace = trappy.FTrace()
 
         # Check that the modified md5sum was overwritten
-        self.assertNotEqual(read_md5sum(), md5sum_inc,
+        self.assertNotEqual(read_metadata()["md5sum"], md5sum_inc,
                             "The invalid ftrace cache wasn't overwritten")
 
     def test_cache_dynamic_events(self):
@@ -192,6 +203,20 @@ def test_cache_window(self):
 
         self.assertEqual(len(trace1.sched_wakeup.data_frame), 2)
 
+    def test_ftrace_metadata(self):
+        """Test that caching keeps trace metadata"""
+        GenericFTrace.disable_cache = False
+
+        self.test_cache_created()
+
+        trace = trappy.FTrace()
+
+        version = int(trace._version)
+        cpus = int(trace._cpus)
+
+        self.assertEquals(version, 6)
+        self.assertEquals(cpus, 6)
+
     def test_cache_delete_single(self):
         GenericFTrace.disable_cache = False
         trace = trappy.FTrace()
@@ -200,7 +225,7 @@ def test_cache_delete_single(self):
         trace_dir = os.path.dirname(trace_path)
         trace_file = os.path.basename(trace_path)
         cache_dir = '.' + trace_file + '.cache'
-        number_of_trace_categories = 29
+        number_of_trace_categories = 27
         self.assertEquals(len(os.listdir(cache_dir)), number_of_trace_categories)
 
         os.remove(os.path.join(cache_dir, 'SchedWakeup.csv'))

tests/test_ftrace.py

@@ -387,6 +387,12 @@ def test_parse_tracing_mark_write_events(self):
                           "TRACE_MARKER_START")
         self.assertEquals(len(trace.tracing_mark_write.data_frame), 1)
 
+    def test_ftrace_metadata(self):
+        """FTrace class correctly populates metadata"""
+        trace = trappy.FTrace()
+
+        self.assertEquals(int(trace._version), 6)
+        self.assertEquals(int(trace._cpus), 6)
 
 @unittest.skipUnless(utils_tests.trace_cmd_installed(),
                      "trace-cmd not installed")
@@ -443,6 +449,13 @@ def test_both_txt_arb_name(self):
         self.assertTrue(hasattr(trace, "sched_switch"))
         self.assertTrue(len(trace.sched_switch.data_frame) > 0)
 
+    def test_keep_txt(self):
+        """Do not delete trace.txt if trace.dat isn't there"""
+        self.assertFalse(os.path.isfile("trace.dat"))
+        self.assertTrue(os.path.isfile("trace.txt"))
+        trace = trappy.FTrace()
+        self.assertTrue(os.path.isfile("trace.txt"))
+
 class TestFTraceSched(utils_tests.SetupDirectory):
     """Tests using a trace with only sched info and no (or partial) thermal"""
 
@@ -506,13 +519,14 @@ def assert_thermal_in_trace(self, fname):
 
         self.assertTrue(found)
 
-    def test_do_txt_if_not_there(self):
-        """Create trace.txt if it's not there"""
+    def test_do_not_txt(self):
+        """Do not create trace.txt if trace.dat is there"""
+        self.assertTrue(os.path.isfile("trace.dat"))
         self.assertFalse(os.path.isfile("trace.txt"))
 
         trappy.FTrace()
 
-        self.assert_thermal_in_trace("trace.txt")
+        self.assertFalse(os.path.isfile("trace.txt"))
 
     def test_ftrace_arbitrary_trace_dat(self):
         """FTrace() works if asked to parse a binary trace with a filename other than trace.dat"""
@@ -521,27 +535,5 @@ def test_ftrace_arbitrary_trace_dat(self):
 
         dfr = trappy.FTrace(arbitrary_trace_name).thermal.data_frame
 
-        self.assertTrue(os.path.exists("my_trace.txt"))
         self.assertTrue(len(dfr) > 0)
         self.assertFalse(os.path.exists("trace.dat"))
-        self.assertFalse(os.path.exists("trace.txt"))
-
-    def test_regenerate_txt_if_outdated(self):
-        """Regenerate the trace.txt if it's older than the trace.dat"""
-
-        trappy.FTrace()
-
-        # Empty the trace.txt
-        with open("trace.txt", "w") as fout:
-            fout.write("")
-
-        # Set access and modified time of trace.txt to 10 seconds ago
-        now = time.time()
-        os.utime("trace.txt", (now - 10, now - 10))
-
-        # touch trace.dat
-        os.utime("trace.dat", None)
-
-        trappy.FTrace()
-
-        self.assert_thermal_in_trace("trace.txt")

tests/trace_sched.txt.cache/metadata.json

@@ -0,0 +1 @@
+{"params": {"window": [1, 5], "abs_window": [0, null]}, "md5sum": "47be9ccdd36fa0c3646b0d9b0f649da4", "basetime": 6550.018511}