Updates for Python 3.12 compatibility (SYN-6184) (#3355)

synapse/cmds/cron.py

@@ -454,8 +454,7 @@ async def _handle_add(self, core, opts):
 
     @staticmethod
     def _format_timestamp(ts):
-        # N.B. normally better to use fromtimestamp with UTC timezone, but we don't want timezone to print out
-        return datetime.datetime.utcfromtimestamp(ts).isoformat(timespec='minutes')
+        return datetime.datetime.fromtimestamp(ts, datetime.UTC).strftime('%Y-%m-%dT%H:%M')
 
     async def _handle_list(self, core, opts):
         cronlist = await core.listCronJobs()

synapse/lib/stormlib/stix.py

@@ -1144,8 +1144,9 @@ async def bundle(self, config=None):
         return StixBundle(self, self.runt, config)
 
     def timestamp(self, tick):
-        dt = datetime.datetime.utcfromtimestamp(tick / 1000.0)
-        return dt.isoformat(timespec='milliseconds') + 'Z'
+        dt = datetime.datetime.fromtimestamp(tick / 1000.0, datetime.UTC)
+        millis = int(dt.microsecond / 1000)
+        return f'{dt.strftime("%Y-%m-%dT%H:%M:%S")}.{millis:03d}Z'
 
 stix_sdos = {
     'attack-pattern',

synapse/lib/stormtypes.py

@@ -9481,9 +9481,7 @@ async def _methCronJobPack(self):
 
     @staticmethod
     def _formatTimestamp(ts):
-        # N.B. normally better to use fromtimestamp with UTC timezone,
-        # but we don't want timezone to print out
-        return datetime.datetime.utcfromtimestamp(ts).isoformat(timespec='minutes')
+        return datetime.datetime.fromtimestamp(ts, datetime.UTC).strftime('%Y-%m-%dT%H:%M')
 
     async def _methCronJobPprint(self):
         user = self.valu.get('username')

synapse/tests/test_common.py

@@ -183,11 +183,11 @@ def test_common_chunks(self):
             parts = [chunk for chunk in s_common.chunks({1, 2, 3}, 10000)]
 
         # dict is unslicable
-        with self.assertRaises(TypeError) as cm:
+        with self.assertRaises((TypeError, KeyError)) as cm:
             parts = [chunk for chunk in s_common.chunks({1: 2}, 10000)]
 
         # empty dict is caught during the [0:0] slice
-        with self.assertRaises(TypeError) as cm:
+        with self.assertRaises((TypeError, KeyError)) as cm:
             parts = [chunk for chunk in s_common.chunks({}, 10000)]
 
     def test_common_ehex_uhex(self):

synapse/tests/test_lib_link.py

@@ -1,14 +1,47 @@
 import ssl
+import sys
 import socket
 import asyncio
+import multiprocessing
 
+import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.coro as s_coro
 import synapse.lib.link as s_link
 
 import synapse.tests.utils as s_test
 
+
+# Helpers related to spawn link coverage
+async def _spawnTarget(n, info):
+    link = await s_link.fromspawn(info)
+    async with link:
+        await link.send(b'V' * n)
+
+def spawnTarget(n, info):
+    asyncio.run(_spawnTarget(n, info))
+
+async def _spawnHost(n, pipe):
+    link0, sock0 = await s_link.linksock()
+    info = await link0.getSpawnInfo()
+    pipe.send(info)
+    data = sock0.recv(n)
+
+    sock0.close()
+    await link0.fini()
+
+    if data == b'V' * n:
+        return
+
+    return 137
+
+def spawnHost(n, pipe: multiprocessing.Pipe):
+    ret = asyncio.run(_spawnHost(n, pipe))
+    if ret is None:
+        return
+    sys.exit(ret)
+
 class LinkTest(s_test.SynTest):
 
     async def test_link_raw(self):
@@ -154,20 +187,61 @@ def writer(sock):
         await link1.fini()
         sock1.close()
 
-    async def test_link_fromspawn(self):
+    async def test_link_fromspawns(self):
+
+        ctx = multiprocessing.get_context('spawn')
+
+        # Remote use test - this is normally how linksock is used.
 
         link0, sock0 = await s_link.linksock()
 
         info = await link0.getSpawnInfo()
-        link1 = await s_link.fromspawn(info)
 
-        await link1.send(b'V')
-        self.eq(sock0.recv(1), b'V')
+        n = 100000
+        def getproc():
+            proc = ctx.Process(target=spawnTarget, args=(n, info))
+            proc.start()
+            return proc
 
-        sock0.close()
+        proc = await s_coro.executor(getproc)
+
+        self.eq(sock0.recv(n), b'V' * n)
 
+        await s_coro.executor(proc.join)
+
+        sock0.close()
         await link0.fini()
-        await link1.fini()
+
+        # Coverage test
+        mypipe, child_pipe = ctx.Pipe()
+
+        def getproc():
+            proc = ctx.Process(target=spawnHost, args=(n, child_pipe))
+            proc.start()
+            return proc
+
+        proc = await s_coro.executor(getproc)  # type: multiprocessing.Process
+
+        def waitforinfo():
+            nonlocal proc
+            hasdata = mypipe.poll(timeout=30)
+            if not hasdata:
+                raise s_exc.SynErr(mesg='failed to get link info')
+            info = mypipe.recv()
+            return info
+
+        info = await s_coro.executor(waitforinfo)
+
+        link = await s_link.fromspawn(info)
+        await link.send(b'V' * n)
+
+        def waitforjoin():
+            proc.join()
+            return proc.exitcode
+
+        code = await s_coro.executor(waitforjoin)
+        self.eq(code, 0)
+        await link.fini()
 
     async def test_tls_ciphers(self):
         self.thisHostMustNot(platform='darwin')

synapse/tests/test_lib_multislabseqn.py

@@ -13,7 +13,7 @@
 
 class MultiSlabSeqn(s_t_utils.SynTest):
 
-    async def test_multislabseqn(self):
+    async def test_multislabseqn_base(self):
 
         with self.getTestDir() as dirn:
 
@@ -166,10 +166,6 @@ async def getter():
                 exp = [(11, 'foo11b'), (12, 'foo12'), (13, 'foo13b'), (14, 'foo14'), (15, 'done')]
                 self.eq(exp, retn)
 
-                # Give a chance for the non-iterated async generators to get cleaned up
-                await asyncio.sleep(0)
-                await asyncio.sleep(0)
-
                 # Make sure we're not holding onto more than 2 slabs
 
                 # rotate
@@ -181,6 +177,10 @@ async def getter():
                 fns = sorted(s_common.listdir(dirn, glob='*.lmdb'))
                 self.len(3, fns)
 
+                # Slab @ 13 will get fini out
+                _slab = msqn._openslabs[13][0]
+                self.true(await _slab.waitfini(6))
+
                 self.len(2, msqn._openslabs)
 
                 retn = await msqn.get(11)  # first

synapse/tests/test_lib_storm.py

@@ -2966,11 +2966,11 @@ async def test_storm_splicelist(self):
             mesgs = await core.stormlist('[ test:str=bar ]')
 
             tick = mesgs[0][1]['tick']
-            tickdt = datetime.datetime.utcfromtimestamp(tick / 1000.0)
+            tickdt = datetime.datetime.fromtimestamp(tick / 1000.0, datetime.UTC)
             tickstr = tickdt.strftime('%Y/%m/%d %H:%M:%S.%f')
 
             tock = mesgs[-1][1]['tock']
-            tockdt = datetime.datetime.utcfromtimestamp(tock / 1000.0)
+            tockdt = datetime.datetime.fromtimestamp(tock / 1000.0, datetime.UTC)
             tockstr = tockdt.strftime('%Y/%m/%d %H:%M:%S.%f')
 
             await asyncio.sleep(0.01)