restructured

Author: shrddr

lib/__init__.py

@@ -1,19 +1,36 @@
-import math
 import cv2
+import math
 import random
+import shutil
+import os.path
+import pathlib
 import numpy as np
 
-import layers
-import loaders
+from lib import layers
+from lib import loaders
 
 def mil(fp):
     return math.floor(fp*1000000)
 
 def osm_at_tile(tx, ty, z):
-    # returns a link to open iD editor at specified tile
+    # prints a link to open iD editor at specified tile
     lat, lng = layers.wgs_at_tile(tx, ty, z)
     print(f"https://www.openstreetmap.org/edit#map={z}/{lat}/{lng}")
 
+def cleandir(path):
+    # removes a directory and creates it again
+    target = pathlib.Path(path)
+    if os.path.isdir(target):
+        shutil.rmtree(target)
+    target.mkdir(parents=True, exist_ok=True)
+    return target
+
+def outside(point, lefttop, rightbot):
+    return point[0] < lefttop[0] \
+        or point[1] < lefttop[1] \
+        or point[0] >= rightbot[0] \
+        or point[1] >= rightbot[1]
+
 class MercatorPainter:
     # paints an area with dots and lines representing positive examples.
     # everything not painted over is supposed to be negative.

helpers.py renamed to lib/helpers.py

@@ -53,8 +53,11 @@ def tilefile(self, x, y, z):
 
     def xy_fromfile(self, path):
         f = path.name
-        sx = f[1:7]
-        sy = f[8:14]
+        xpos = f.index('x')
+        ypos = f.index('y')
+        dpos = f.index('.')
+        sx = f[xpos+1:ypos]
+        sy = f[ypos+1:dpos]
         return (int(sx), int(sy))
 
     def tileurl(self, x, y, z):

layers.py renamed to lib/layers.py

@@ -1,14 +1,13 @@
-import pathlib
+import cv2
+import random
 import shutil
 import os.path
-import random
-import cv2
-
-import layers
-import loaders
-import helpers
 import tarfile
 
+from lib import layers
+from lib import loaders
+from lib import helpers
+
 # this is 256 for all current imagery providers
 TILESIZE = 256
 
@@ -26,13 +25,6 @@
 
 # validation images of each category
 VALID = 1000
-
-def cleandir(path):
-    target = pathlib.Path(path)
-    if os.path.isdir(target):
-        shutil.rmtree(target)
-    target.mkdir(parents=True, exist_ok=True)
-    return target
 
 if __name__ == "__main__":
 #    box = (27.4583,53.9621,27.5956,53.9739) # north belt
@@ -47,7 +39,7 @@ def cleandir(path):
     train = lamps[:TRAIN]
     valid = lamps[TRAIN:]
 
-    target = cleandir('lamps-expand/train/lamp')
+    target = helpers.cleandir('lamps-expand/train/lamp')
     for lamp in train:        
         h = w = EXPAND_PAD + TILESIZE + EXPAND_PAD
         crop = layers.maxar.getcrop_wgs(lamp, h, w, IMZ)
@@ -57,7 +49,7 @@ def cleandir(path):
         cv2.imwrite(dst, crop)
 
 
-    target = cleandir('lamps-expand/valid/lamp')
+    target = helpers.cleandir('lamps-expand/valid/lamp')
     count = 0
     it = iter(valid)
     while count < VALID:
@@ -83,7 +75,7 @@ def cleandir(path):
     for i in range(VALID):
         data['v'].append(mp.random_negative())
 
-    target = cleandir('lamps-expand/train/nolamp')
+    target = helpers.cleandir('lamps-expand/train/nolamp')
     for (tx,ty) in data['t']:
         wgs = layers.wgs_at_tile(tx, ty, IMZ)
         h = w = EXPAND_PAD + TILESIZE + EXPAND_PAD
@@ -93,7 +85,7 @@ def cleandir(path):
         dst = str(target / f"m_lat{lat}lng{lng}.jpg")
         cv2.imwrite(dst, crop)
 
-    target = cleandir('lamps-expand/valid/nolamp')
+    target = helpers.cleandir('lamps-expand/valid/nolamp')
     for (tx,ty) in data['v']:
         fname = layers.maxar.download(tx, ty, IMZ)
         if fname is not None:

loaders.py renamed to lib/loaders.py

@@ -1,14 +1,11 @@
-import pathlib
+import random
 import shutil
 import os.path
-import random
-import cv2
-
-import layers
-import loaders
-import helpers
 import tarfile
 
+from lib import layers
+from lib import loaders
+from lib import helpers
 
 # this is 256 for all current imagery providers
 TILESIZE = 256
@@ -18,13 +15,6 @@
 
 # if you want to train faster
 LIMIT = 5000
-
-def cleandir(path):
-    target = pathlib.Path(path)
-    if os.path.isdir(target):
-        shutil.rmtree(target)
-    target.mkdir(parents=True, exist_ok=True)
-    return target
 
 if __name__ == "__main__":
 #    box = (27.4583,53.9621,27.5956,53.9739) # north belt
@@ -38,7 +28,7 @@ def cleandir(path):
     random.shuffle(lamps)
     lamps = lamps[:LIMIT]
 
-    target = cleandir('lamps-orig/lamp')
+    target = helpers.cleandir('lamps-orig/lamp')
     for lamp in lamps:
         fname = layers.maxar.gettile_wgs(lamp, IMZ, skipedge=True)
         if fname is not None:
@@ -57,7 +47,7 @@ def cleandir(path):
     while len(batch) < LIMIT:
         batch.append(mp.random_negative())
 
-    target = cleandir('lamps-orig/nolamp')
+    target = helpers.cleandir('lamps-orig/nolamp')
     for (tx,ty) in batch:
         fname = layers.maxar.download(tx, ty, IMZ)
         if fname is not None:

make_expand.py

@@ -0,0 +1,36 @@
+import cv2
+import numpy as np
+from lib import helpers
+from lib import layers
+
+# shows how much is the box covered with cached tiles
+
+IMZ = 18
+
+if __name__ == "__main__": 
+    box = (27.4026,53.8306,27.7003,53.9739)
+    source = layers.maxar.tiledir / f"z{IMZ}"
+    localtiles = [layers.maxar.xy_fromfile(path) for path in source.glob("*.jpg")]
+#    localtiles = localtiles[:1000]
+    
+    W, S, E, N = box
+    txmin, tymin = layers.maxar.tile_at_wgs((N, W), IMZ)
+    txmax, tymax = layers.maxar.tile_at_wgs((S, E), IMZ)
+
+    width = txmax-txmin+2
+    height = tymax-tymin+2
+    print(f"map size: {width}x{height}px")
+    canvas = np.zeros((height, width, 3), np.uint8)
+    
+    for (tx,ty) in localtiles:
+        fname = layers.maxar.download(tx, ty, IMZ)
+        img = cv2.imread(fname)
+        av = img.mean(axis=0).mean(axis=0)
+        x = tx-txmin
+        y = ty-tymin
+        if helpers.outside((x,y), (0,0), (width, height)):
+            continue
+        canvas[y, x] = av
+        
+    cv2.imshow('canvas', canvas)
+    cv2.waitKey(0)

make_original.py

@@ -1,17 +1,17 @@
 import cv2
-import loaders
-import glob
-import imgdl
-import layers
+from pathlib import Path
+from lib import loaders
+from lib import layers
 
 FRAMESIZE = 256
 
-def dir2vid(layer):
+def dir2vid(path):
     images = []
     i = 0
-    for fname in layer.cropdir.glob("*.jpg"):
+    print(Path(path))
+    for fname in Path(path).glob("*.jpg"):
+        print(fname)
         img = cv2.imread(str(fname))
-
         x = 95
         y = 135
         cv2.line(img, (x,y), (x+10,y+10), (0,0,255), 1)
@@ -22,19 +22,17 @@ def dir2vid(layer):
         if i > 10000:
             break
 
-    out = cv2.VideoWriter(f"./all-{layer.name}.avi", cv2.VideoWriter_fourcc(*'DIVX'), 60, (FRAMESIZE,FRAMESIZE))
+    out = cv2.VideoWriter(f"./dir.avi", cv2.VideoWriter_fourcc(*'DIVX'), 60, (FRAMESIZE,FRAMESIZE))
 
     for i in range(len(images)):
         out.write(images[i])
     out.release()
 
-def list2vid(lamps, layer):
+def list2vid(lamps, layer, z):
     print(1)
     images = []
     for lamp in lamps:
-        print("getting crop")
-        img = imgdl.getcrop(None, layer, lamp, FRAMESIZE, FRAMESIZE)
-        print("painting")
+        img = layers.maxar.getcrop_wgs(lamp, FRAMESIZE, FRAMESIZE, z)
         x = 95
         y = 135
         cv2.line(img, (x,y), (x+10,y+10), (0,0,255), 1)
@@ -49,8 +47,8 @@ def list2vid(lamps, layer):
     out.release()
 
 if __name__ == "__main__":
-#    dir2vid(layers.maxar)
-#    dir2vid(layers.dg)
-#    lamps = loaders.bbox(27.5682,53.8469,27.5741,53.8688)
-#    list2vid(lamps, layers.maxar)
-#    list2vid(lamps, layers.dg)
+#    dir2vid(layers.maxar, 19)
+    dir2vid("lamps-expand/train/lamp")
+#    lamps = loaders.query_nodes(27.4026,53.8306,27.7003,53.9739)
+#    list2vid(lamps, layers.maxar, 19)
+#    list2vid(lamps, layers.dg, 19)