Skip to content

Commit aba3802

Browse files
committed
research
1 parent 2686d36 commit aba3802

File tree

9 files changed

+117
-60
lines changed

9 files changed

+117
-60
lines changed

helpers.py

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,14 @@ class MercatorPainter:
1919
# everything not painted over is supposed to be negative.
2020
# uses dict for fast lookup (builds itself on first query).
2121
# also has a function to find a random negative (unpainted) pixel.
22-
def __init__(self, W, S, E, N):
23-
txmin, tymin = layers.tile_at_wgs((N, W))
24-
txmax, tymax = layers.tile_at_wgs((S, E))
22+
def __init__(self, layer, W, S, E, N):
23+
txmin, tymin = layer.tile_at_wgs((N, W))
24+
txmax, tymax = layer.tile_at_wgs((S, E))
2525
area = (txmax-txmin, tymax-tymin)
2626
print(f"paint area: {txmin}..{txmax}, {tymin}..{tymax}")
2727
print(f"dimensions: {area} -> {area[0]*area[1]} tiles total")
2828

29+
self.layer = layer
2930
self.txmin = txmin
3031
self.tymin = tymin
3132
self.width = txmax-txmin+1
@@ -35,7 +36,7 @@ def __init__(self, W, S, E, N):
3536
self.dict = None
3637

3738
def wgs2px(self, latlng):
38-
tx, ty = layers.tile_at_wgs(latlng)
39+
tx, ty = self.layer.tile_at_wgs(latlng)
3940
x = tx - self.txmin
4041
y = ty - self.tymin
4142
return (x,y)
@@ -46,18 +47,18 @@ def add_dot_tile(self, tile, color=255):
4647
y = ty - self.tymin
4748
self.canvas[y][x] = color
4849

49-
def add_dots(self, latlngs, color=255):
50+
def add_dots_wgs(self, latlngs, color=255):
5051
for latlng in latlngs:
5152
x, y = self.wgs2px(latlng)
5253
self.canvas[y][x] = color
5354

54-
def add_line(self, latlng1, latlng2, width):
55+
def add_line_wgs(self, latlng1, latlng2, width):
5556
# the lines are actually curves in mercator but we don't care
5657
p1 = self.wgs2px(latlng1)
5758
p2 = self.wgs2px(latlng2)
5859
cv2.line(self.canvas, p1, p2, 255, width)
5960

60-
def add_polyline(self, latlngs, width):
61+
def add_polyline_wgs(self, latlngs, width):
6162
pixels = [self.wgs2px(l) for l in latlngs]
6263
pixels = np.array(pixels)
6364
cv2.polylines(self.canvas, [pixels], False, 255, width)
@@ -108,6 +109,7 @@ def find_random(self):
108109
if self.contains(tile):
109110
continue
110111
else:
112+
self.add_dot_tile(tile)
111113
return tile
112114

113115
if __name__ == "__main__":

layers.py

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,6 @@ def project2web(latlng):
1616
y = TILESIZE * (0.5 - math.log((1 + siny) / (1 - siny)) / (4 * math.pi))
1717
return (x, y)
1818

19-
def tile_at_wgs(latlng, z=19):
20-
# returns index of tile which contains a location
21-
scale = 1 << z
22-
wc = project2web(latlng)
23-
tx = math.floor(wc[0] * scale / TILESIZE)
24-
ty = math.floor(wc[1] * scale / TILESIZE)
25-
return (tx,ty)
26-
2719
def wgs_at_tile(tx, ty, z=19):
2820
# converts tile index to EPSG:3857 (0..1) then to EPSG:4326 (degrees)
2921
scale = 1 << z
@@ -41,7 +33,7 @@ def __init__(self, name):
4133
self.flipy = False
4234
self.offsetx = 0
4335
self.offsety = 0
44-
self.tiledir = Path("tiles") / name
36+
self.tiledir = Path("../tiles") / name
4537

4638
def tilefile(self, x, y, z):
4739
return self.tiledir / f"x{x}y{y}z{z}.jpg"
@@ -72,10 +64,42 @@ def download(self, x, y, z=19):
7264
raise IOError(f"{r.status_code} at {url}'")
7365
return str(fname)
7466

75-
def gettile_wgs(self, latlng, z=19):
67+
def tile_at_wgs(self, latlng, z=19):
68+
# returns index of tile which contains a location
69+
scale = 1 << z
70+
wc = project2web(latlng)
71+
# pixel in world
72+
px = wc[0] * scale + self.offsetx
73+
py = wc[1] * scale + self.offsety
74+
# tile in world
75+
tx = math.floor(px / TILESIZE)
76+
ty = math.floor(py / TILESIZE)
77+
return (tx, ty)
78+
79+
def gettile_wgs(self, latlng, z=19, skipedge=False):
7680
# returns tile at location (as filename)
77-
x,y = tile_at_wgs(latlng, z)
78-
fname = self.download(x, y, z)
81+
# returns None if skipedge is enabled and location is indeed close to edge
82+
scale = 1 << z
83+
wc = project2web(latlng)
84+
# pixel in world
85+
px = wc[0] * scale + self.offsetx
86+
py = wc[1] * scale + self.offsety
87+
# tile in world
88+
tx = math.floor(px / TILESIZE)
89+
ty = math.floor(py / TILESIZE)
90+
# pixel in tile
91+
rx = (px - tx) % TILESIZE
92+
ry = (py - ty) % TILESIZE
93+
94+
if skipedge:
95+
EDGE_THRESH = 10 # px
96+
edge = (rx < EDGE_THRESH) or (rx >= TILESIZE-EDGE_THRESH) \
97+
or (ry < EDGE_THRESH) or (ry >= TILESIZE-EDGE_THRESH)
98+
if edge:
99+
print("edge")
100+
return None
101+
102+
fname = self.download(tx, ty, z)
79103
return fname
80104

81105
def tiles_near_wgs(self, latlng, scale, h, w):
@@ -148,4 +172,4 @@ def getcrop_wgs(self, latlng, h, w, z=19):
148172
dg = Imagery("dg")
149173
dg.url = "https://c.tiles.mapbox.com/v4/digitalglobe.316c9a2e/{z}/{x}/{y}.png?access_token=pk.eyJ1IjoiZGlnaXRhbGdsb2JlIiwiYSI6ImNqZGFrZ2c2dzFlMWgyd2x0ZHdmMDB6NzYifQ.9Pl3XOO82ArX94fHV289Pg"
150174

151-
print(maxar.xy_fromfile(Path(r"tiles\maxar\x302117y168688z19.jpg")))
175+
print(maxar.xy_fromfile(Path(r"..\tiles\maxar\x302117y168688z19.jpg")))

loaders.py

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,21 @@
11
import overpass
22
import json
33
import os.path
4-
import time
5-
import datetime
64
from math import floor
75

8-
import helpers
9-
import layers
10-
116
def mil(fp):
127
return floor(fp*1000000)
138

149
def query_nodes(W, S, E, N):
1510
# queries overpass or fetches cached result if available
1611
# returns list of (lat, lng) tuples
17-
fname = f"./in/bbox{mil(W)}_{mil(S)}_{mil(E)}_{mil(N)}.json"
12+
fname = f"./overpass/bbox{mil(W)}_{mil(S)}_{mil(E)}_{mil(N)}.json"
1813
if os.path.isfile(fname):
1914
with open(fname) as json_file:
2015
return json.load(json_file)
2116

2217
api = overpass.API()
23-
query = f"node[\"highway\"=\"street_lamp\"]({S}, {W}, {N}, {E})"
18+
query = f"""node["highway"="street_lamp"]({S}, {W}, {N}, {E})"""
2419
response = api.get(query, responseformat='json', verbosity='skel')
2520

2621
nodes = [(e['lat'], e['lon']) for e in response['elements']]
@@ -32,7 +27,7 @@ def query_nodes(W, S, E, N):
3227

3328

3429
def query_ways(W, S, E, N):
35-
fname = f"./in/ways_bbox{mil(W)}_{mil(S)}_{mil(E)}_{mil(N)}.json"
30+
fname = f"./overpass/ways_bbox{mil(W)}_{mil(S)}_{mil(E)}_{mil(N)}.json"
3631
if os.path.isfile(fname):
3732
with open(fname) as json_file:
3833
return json.load(json_file)
@@ -74,6 +69,5 @@ def query_poly(bounds):
7469
# https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Polygon_evaluator
7570
pass
7671

77-
7872
if __name__ == "__main__":
7973
pass

make.py

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@
1010
import tarfile
1111

1212
# use one original tile or expand it for later cropping
13-
ORIGINAL = False
13+
MAKE_POSITIVE_ORIGINAL = 0
14+
MAKE_POSITIVE_EXPANDED = 0
15+
MAKE_NEGATIVE_ORIGINAL = 0
16+
MAKE_NEGATIVE_EXPANDED = 1
1417

1518
# this is 256 for all current imagery providers
1619
TILESIZE = 256
@@ -19,7 +22,7 @@
1922
# known satellite imagery has up to 40px offset.
2023
# maximum acceptable padding is (128-40)=88
2124
# which translates to image size 256+88*2=432
22-
PADDING = 50
25+
PADDING = 88
2326

2427
if __name__ == "__main__":
2528
# box = (27.4583,53.9621,27.5956,53.9739) # north belt
@@ -31,19 +34,20 @@
3134
lamps = loaders.query_nodes(*box)
3235
print("lamps:", len(lamps))
3336

34-
if ORIGINAL:
35-
# only use one tile where lamp exists. should fail
36-
# miserably when the lamp is on the edge of tile
37+
if MAKE_POSITIVE_ORIGINAL:
38+
# only use one tile where lamp exists
3739
target = pathlib.Path('lamps-orig/lamp')
3840
target.mkdir(parents=True, exist_ok=True)
3941

4042
for lamp in lamps:
41-
fname = layers.maxar.gettile_wgs(lamp)
42-
dst = target / ("m_" + os.path.basename(fname))
43-
shutil.copy(fname, dst)
43+
fname = layers.maxar.gettile_wgs(lamp, skipedge=True)
44+
if fname is not None:
45+
dst = target / ("m_" + os.path.basename(fname))
46+
shutil.copy(fname, dst)
4447

45-
else:
46-
# use a bigger picture for later random cropping (augmentation)
48+
if MAKE_POSITIVE_EXPANDED:
49+
# use a bigger picture, centered at the object,
50+
# for later random cropping (augmentation)
4751
target = pathlib.Path('lamps-center/lamp')
4852
target.mkdir(parents=True, exist_ok=True)
4953

@@ -61,11 +65,11 @@
6165

6266
BATCHSIZE = len(lamps)
6367

64-
mp = helpers.MercatorPainter(*box)
65-
mp.add_dots(lamps)
68+
mp = helpers.MercatorPainter(layers.maxar, *box)
69+
mp.add_dots_wgs(lamps)
6670
roads = loaders.query_ways(*box)
6771
for nodes in roads.values():
68-
mp.add_polyline(nodes, width=2)
72+
mp.add_polyline_wgs(nodes, width=2)
6973

7074
source = layers.maxar.tiledir
7175
localtiles = [layers.maxar.xy_fromfile(path) for path in source.glob("*.jpg")]
@@ -82,38 +86,37 @@
8286
while BATCHSIZE > len(batch):
8387
batch.append(mp.find_random())
8488

85-
if ORIGINAL: # only use one tile
86-
# TODO: discard if lamp is close to the edge
87-
# TODO: handle imagery offset
89+
if MAKE_NEGATIVE_ORIGINAL:
90+
# only use one tile
8891
target = pathlib.Path('lamps-orig/nolamp')
8992
target.mkdir(parents=True, exist_ok=True)
9093
for (tx,ty) in batch:
9194
fname = layers.maxar.download(tx,ty)
9295
dst = target / ("m_" + os.path.basename(fname))
9396
shutil.copy(fname, dst)
9497

95-
else:
98+
if MAKE_NEGATIVE_EXPANDED:
9699
# expand the tile for later cropping
97100
target = pathlib.Path('lamps-center/nolamp')
98101
target.mkdir(parents=True, exist_ok=True)
99102
for (tx,ty) in batch:
100103
wgs = layers.wgs_at_tile(tx, ty)
101-
h, w = (356, 356)
104+
h = w = PADDING + TILESIZE + PADDING
102105
crop = layers.maxar.getcrop_wgs(wgs, h, w)
103106
lat = helpers.mil(wgs[0])
104107
lng = helpers.mil(wgs[1])
105108
dst = str(target / f"m_lat{lat}lng{lng}z19.jpg")
106109
cv2.imwrite(dst, crop)
107110

108-
if ORIGINAL:
111+
if MAKE_POSITIVE_ORIGINAL and MAKE_NEGATIVE_ORIGINAL:
109112
tarball = "./lamps-orig.tar"
110113
if os.path.exists(tarball):
111114
os.remove(tarball)
112115
tar = tarfile.open(tarball, "w")
113116
tar.add("./lamps-orig")
114117
tar.close()
115118

116-
else:
119+
if MAKE_POSITIVE_EXPANDED and MAKE_NEGATIVE_EXPANDED:
117120
tarball = "./lamps-center.tar"
118121
if os.path.exists(tarball):
119122
os.remove(tarball)
File renamed without changes.

random_crop.png

25.8 KB
Loading

readme.md

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,68 @@
1+
## What's up?
2+
3+
These scripts prepare the data to teach a classification network which tells apart satellite imagery tiles with [streetlamp](https://wiki.openstreetmap.org/wiki/Tag:highway%3Dstreet_lamp)s or with no streetlamps. The tests only use the latest and the clearest imagery layer for my city called Maxar.
4+
15
## Original tiles
26

3-
These use skew and rotate to augment (which is hardly acceptable for satellite imagery). Also, must be problematic when the lamp is on tile edge. Sometimes imagery offset makes the object appear on a wrong tile, this is probably a major roadblock.
7+
### First attempt
8+
9+
Satellite imagery providers serve data in 256x256 tiles. The first approach is just fetch a tile which contains a lamp and use that as a positive example. Every tile that does not contain a high-level road (highway=tertiary and up) is supposed to be negative.
10+
11+
The problem is some lamps are at the tile edge and possibly cross the boundary. Sometimes [imagery offset](https://wiki.openstreetmap.org/wiki/Using_Imagery#Frequent_mistakes) makes the object appear on a different tile than it should, which produces false positive example.
412

513
```
614
tfms = get_transforms(do_flip=False)
715
data = ImageDataBunch.from_folder(path, train=".", valid_pct=0.1, ds_tfms=tfms, size=256)
816
```
917

10-
Best I got was 3.9% error
18+
Best I got was 3.1-3.9% validation error and it just doesn't train any further.
19+
20+
### Discard "edge cases"
21+
22+
Here I drop all positive examples where the base of street lamp is less than 10px away from tile edge. Dealing with offset is tricky, as it depends on both imagery properties and OSM mappers in the area. I did eyeball an average offset for my area, but for other cities it can be anything. You can use `video.py` to do that, it lets you look through lots of imagery quickly.
23+
24+
Negative examples are still one random non-road tile.
25+
26+
This dataset converges to 3% error.
1127

1228
## Expanded tiles
1329

14-
This is where I turn off skew, rotate and zoom, because crop should be sufficient. Turns out crop doesn't work like that and must be turned on explicitly - see below. This however shows the performance with virtually no augmentation.
30+
### Damn it works.. oh wait
31+
32+
The best thing about satellite imagery is that it's huge and can be scrolled in every direction almost infinitely. If you need more information about a location you can always look at adjacent tiles. This method just fetches a larger square of 356 pixels around every known streetlamp. These will get randomly cropped later in training process. Negative examples are expanded to 356px too, just for consistency.
33+
34+
Turns out, these runs had no crop at all - see below. This however shows the performance with virtually no augmentation.
1535

1636
```
17-
tfms = get_transforms(do_flip=False, max_warp=0, max_zoom=0, max_rotate=0
37+
# just resizes 356->256px with no crop!
38+
tfms = get_transforms(do_flip=False, max_warp=0, max_zoom=0, max_rotate=0)
1839
data = ImageDataBunch.from_folder(path, train=".", valid_pct=0.1, ds_tfms=tfms, size=256)
1940
```
2041

21-
432px: 1.3% error
42+
The validation error comes down to:
2243

23-
356px: frozen 1ep 3.8% > unfrozen 2ep 1.3-1.7% > 2ep 0.7-1.0%
44+
| | Deeper layers frozen, 1 epoch train | Unfreeze, 2 epochs | 2 more epochs |
45+
| ----- | ----------------------------------- | ------------------ | ------------- |
46+
| 432px | | | 1.3% |
47+
| 356px | 3.8% | 1.3-1.7% | 0.7-1.0% |
48+
49+
### The real thing
50+
51+
Train set cropping seems important for real-life applications, because input at inference time will include streetlamps at any part of a tile, not just center of it. That's why we need to crop randomly at train time. Random cropping turned out tricky in fast.ai, here's my hack that looks like it's doing the right thing:
2452

2553
```
26-
tfms = [[crop(size=256, row_pct=(0,1), col_pct=(0,1))],[]]
54+
# crops random 256x256 piece out of larger
55+
# input image both at train and validation
56+
tfms = [[crop(size=256, row_pct=(0,1), col_pct=(0,1))],
57+
[crop(size=256, row_pct=(0,1), col_pct=(0,1))]]
2758
data = ImageDataBunch.from_folder(path, train=".", valid_pct=0.1, ds_tfms=tfms, size=None)
2859
```
2960

30-
356px: frozen 1ep 4-6% > unfrozen 2ep 0.1-0.4% > 2ep 0.2%
61+
Results are indeed much better:
62+
63+
| | Deeper layers frozen, 1 epoch train | Unfreeze, 2 epochs | 2 more epochs |
64+
| ----- | ----------------------------------- | ------------------ | ------------- |
65+
| 356px | 4-6% | 0.1-0.4% / 3.4% | 0.2% |
66+
| 432px | 4.8-5.5% | 3.8-4.7% | 5.5% |
3167

32-
As my set only contains 10k images, this means just 2 incorrect validation results O_o
68+
As my set only contains 10k images, and validation is 10% of that, 0.1% error means just one incorrect validation result. Oh my.

tiles/dg/.gitignore

Lines changed: 0 additions & 1 deletion
This file was deleted.

tiles/maxar/.gitignore

Lines changed: 0 additions & 1 deletion
This file was deleted.

0 commit comments

Comments
 (0)