public release

.gitignore

@@ -0,0 +1,14 @@
+*.iml
+.gradle
+/local.properties
+/.idea
+.DS_Store
+/build
+tools/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties
+*CMakeLists.txt.user
+app/src/main/assets/encoding.data
+app/src/main/assets/network.data

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "tools/thirdparty/cxxopts"]
+	path = tools/thirdparty/cxxopts
+	url = https://github.com/jarro2783/cxxopts.git
+[submodule "tools/thirdparty/tiny-cuda-nn"]
+	path = tools/thirdparty/tiny-cuda-nn
+	url = https://github.com/t-artikov/tiny-cuda-nn.git

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Timur Artikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,12 @@
+## Map rendering using a neural network
+
+Based on the great article [Instant Neural Graphics Primitives with a Multiresolution Hash Encoding](https://nvlabs.github.io/instant-ngp/assets/mueller2022instant.pdf).
+
+![map](https://github.com/t-artikov/neuromap/blob/master/screenshot.png)
+
+The repository consists of:
+
+- **tools/trainer** - a program for training a neural network on raster tiles.
+- **tools/tiles_to_image** - converts tiles from custom format to PNG image (for testing purposes).
+- **tools/inference** - generates an image from a trained neural network (for testing purposes).
+- **app** - An Android application that displays a neural map in real time.

app/.gitignore

@@ -0,0 +1 @@
+/build

app/build.gradle

@@ -0,0 +1,55 @@
+plugins {
+    id 'com.android.application'
+    id 'org.jetbrains.kotlin.android'
+}
+
+android {
+    compileSdk 32
+
+    defaultConfig {
+        applicationId "me.tartikov.neuromap"
+        minSdk 21
+        targetSdk 32
+        versionCode 1
+        versionName "1.0"
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+    }
+
+    buildTypes {
+        release {
+            signingConfig signingConfigs.debug
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
+    aaptOptions {
+        noCompress 'network.data', 'encoding.data'
+    }
+}
+
+dependencies {
+    implementation 'androidx.appcompat:appcompat:1.4.2'
+    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.6.0'
+    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.0'
+    implementation 'com.otaliastudios:zoomlayout:1.9.0'
+}
+
+preBuild.doFirst {
+    def files = [
+        "./src/main/assets/encoding.data",
+        "./src/main/assets/network.data"
+    ]
+    for(f in files) {
+        if (!file(f).exists()) {
+            throw new GradleException("$f is missing")
+        }
+    }
+}

app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

app/src/main/AndroidManifest.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="me.tartikov.neuromap">
+
+    <uses-feature android:glEsVersion="0x00030001" android:required="true" />
+
+    <application
+        android:name="Application"
+        android:allowBackup="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.NeuroMap">
+        <activity
+            android:name=".MainActivity"
+            android:exported="true"
+            android:configChanges="orientation|screenSize|screenLayout|keyboardHidden">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+</manifest>

app/src/main/assets/grid.fs

@@ -0,0 +1,96 @@
+#version 310 es
+#extension GL_EXT_texture_buffer : require
+
+precision highp float;
+precision highp int;
+
+in vec2 v_position;
+
+uniform highp samplerBuffer u_encodingTexture;
+
+uniform int u_level;
+uniform float u_levelOpacity;
+
+const int encodingLevelCount = 12;
+const int encodingFeaturesPerLevel = 4;
+const int encodingCount = encodingLevelCount * encodingFeaturesPerLevel;
+const int encodingPackedCount = encodingCount / 4;
+uniform int u_encodingResolutions[encodingLevelCount];
+uniform int u_encodingParamCounts[encodingLevelCount];
+uniform int u_encodingParamOffsets[encodingLevelCount];
+
+const int networkInputCount = encodingCount;
+const int networkInputPackedCount = networkInputCount / 4;
+const int networkHiddenCount = 8;
+const int networkHiddenPackedCount = networkHiddenCount / 4;
+const int networkWeightCount = networkInputCount * networkHiddenCount;
+const int networkWeightPackedCount = networkWeightCount / 4;
+uniform vec4 u_networkWeights[networkWeightPackedCount];
+
+layout(location = 0) out vec4 f_color0;
+layout(location = 1) out vec4 f_color1;
+
+
+uint hash(uint x, uint y, uint resolution, uint paramCount) {
+    if (resolution * resolution > paramCount) {
+        return (y * 2654435761u) ^ x;
+    } else {
+        return y * resolution + x;
+    }
+}
+
+vec4 getEncoding(int x, int y, int level) {
+    uint resolution = uint(u_encodingResolutions[level]);
+    uint paramCount = uint(u_encodingParamCounts[level]);
+    uint paramOffset = uint(u_encodingParamOffsets[level]);
+
+    uint index = hash(uint(x), uint(y), resolution, paramCount) % paramCount;
+    return texelFetch(u_encodingTexture, int(index + paramOffset));
+}
+
+vec4 getEncodingInterpolated(vec2 position, int level) {
+    if (level > u_level) {
+        return vec4(0.0);
+    }
+    int resolution = u_encodingResolutions[level];
+    vec2 scaledPosition = position * float(resolution - 1);
+    vec2 intPosition;
+    vec2 fracPosition = modf(scaledPosition, intPosition);
+    int ix = int(intPosition.x);
+    int iy = int(intPosition.y);
+
+    vec4 v00 = getEncoding(ix, iy, level);
+    vec4 v10 = getEncoding(ix + 1, iy, level);
+    vec4 v01 = getEncoding(ix, iy + 1, level);
+    vec4 v11 = getEncoding(ix + 1, iy + 1, level);
+
+    vec4 v0 = mix(v00, v10, fracPosition.x);
+    vec4 v1 = mix(v01, v11, fracPosition.x);
+    return mix(v0, v1, fracPosition.y);
+}
+
+void main() {
+    vec2 position = clamp(v_position, vec2(0.0), vec2(1.0));
+
+    vec4 encodings[encodingPackedCount];
+    for (int i = 0; i < encodingLevelCount; i++) {
+        encodings[i] = getEncodingInterpolated(position, i);
+    }
+    encodings[u_level] *= u_levelOpacity;
+
+    vec4 hidden[networkHiddenPackedCount];
+    for (int i = 0; i < networkHiddenPackedCount; i++) {
+        hidden[i] = vec4(0.0);
+        for (int j = 0; j < networkInputPackedCount; j++) {
+            vec4 inputValue = encodings[j];
+            hidden[i] += vec4(
+                dot(inputValue, u_networkWeights[(i * 4) * networkInputPackedCount + j]),
+                dot(inputValue, u_networkWeights[(i * 4 + 1) * networkInputPackedCount + j]),
+                dot(inputValue, u_networkWeights[(i * 4 + 2) * networkInputPackedCount + j]),
+                dot(inputValue, u_networkWeights[(i * 4 + 3) * networkInputPackedCount + j])
+            );
+        }
+    }
+    f_color0 = hidden[0];
+    f_color1 = hidden[1];
+}

app/src/main/assets/grid.vs

@@ -0,0 +1,13 @@
+#version 310 es
+
+precision highp float;
+
+in vec2 a_position;
+out vec2 v_position;
+
+uniform vec4 u_gridRect;
+
+void main() {
+    gl_Position = vec4(a_position * 2.0 - vec2(1.0, 1.0), 0.0, 1.0);
+    v_position = a_position * u_gridRect.zw + u_gridRect.xy;
+}

app/src/main/assets/map.fs

@@ -0,0 +1,51 @@
+#version 310 es
+
+precision highp float;
+
+in vec2 v_position;
+
+const int gridTextureCount = 2;
+uniform highp sampler2D u_gridTextures[gridTextureCount];
+
+const int networkHiddenCount = 8;
+const int networkOutputCount = 4;
+const int networkWeightCount = networkHiddenCount * networkOutputCount;
+
+const int networkHiddenPackedCount = networkHiddenCount / 4;
+const int networkWeightPackedCount = networkWeightCount / 4;
+
+uniform vec4 u_networkWeights[networkWeightPackedCount];
+
+out vec4 f_color;
+
+vec4 get_color(vec4 v) {
+    const vec4 r = vec4(246.0/255.0, 242.0/255.0, 217.0/255.0, 1.0);
+    const vec4 g = vec4(209.0/255.0, 230.0/255.0, 161.0/255.0, 1.0);
+    const vec4 b = vec4(164.0/255.0, 216.0/255.0, 235.0/255.0, 1.0);
+    if (v.r > v.g) {
+        if (v.r > v.b) return r;
+        else return b;
+    }
+    else {
+        if (v.g > v.b) return g;
+        else return b;
+    }
+}
+
+void main() {
+    vec4 hidden[networkHiddenPackedCount];
+    hidden[0] = texture(u_gridTextures[0], v_position);
+    hidden[1] = texture(u_gridTextures[1], v_position);
+
+    vec4 outputs = vec4(0.0);
+    for (int j = 0; j < networkHiddenPackedCount; j++) {
+        vec4 inputValue = tanh(hidden[j]);
+        outputs += vec4(
+            dot(inputValue, u_networkWeights[j]),
+            dot(inputValue, u_networkWeights[networkHiddenPackedCount + j]),
+            dot(inputValue, u_networkWeights[2 * networkHiddenPackedCount + j]),
+            dot(inputValue, u_networkWeights[3 * networkHiddenPackedCount + j])
+        );
+    }
+    f_color = get_color(outputs);
+}

app/src/main/assets/map.vs

@@ -0,0 +1,13 @@
+#version 310 es
+
+precision highp float;
+
+in vec2 a_position;
+out vec2 v_position;
+
+uniform vec4 u_screenSpaceGridRect;
+
+void main() {
+    gl_Position = vec4((a_position * 2.0 - vec2(1.0)) * vec2(1.0, -1.0), 0.0, 1.0);
+    v_position = a_position * u_screenSpaceGridRect.zw + u_screenSpaceGridRect.xy;
+}

app/src/main/java/me/tartikov/neuromap/Application.kt

@@ -0,0 +1,11 @@
+package me.tartikov.neuromap
+
+class Application : android.app.Application() {
+    init {
+        instance = this
+    }
+    companion object {
+        lateinit var instance: Application
+            private set
+    }
+}

app/src/main/java/me/tartikov/neuromap/AssetUtils.kt

@@ -0,0 +1,38 @@
+package me.tartikov.neuromap
+
+import java.io.BufferedReader
+import java.io.FileInputStream
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.nio.FloatBuffer
+import java.nio.ShortBuffer
+import java.nio.channels.FileChannel
+
+fun loadStringFromAsset(fileName: String): String {
+    return Application.instance.assets
+        .open(fileName).bufferedReader().use(BufferedReader::readText)
+}
+
+fun loadBufferFromAsset(fileName: String): ByteBuffer {
+    Application.instance.assets.openFd(fileName).use { descriptor ->
+        FileInputStream(descriptor.fileDescriptor).use { stream ->
+            return stream.channel.map(
+                FileChannel.MapMode.READ_ONLY,
+                descriptor.startOffset,
+                descriptor.declaredLength
+            )
+        }
+    }
+}
+
+fun loadFloatBufferFromAsset(fileName: String): FloatBuffer {
+    return loadBufferFromAsset(fileName).apply {
+        order(ByteOrder.nativeOrder())
+    }.asFloatBuffer()
+}
+
+fun loadShortBufferFromAsset(fileName: String): ShortBuffer {
+    return loadBufferFromAsset(fileName).apply {
+        order(ByteOrder.nativeOrder())
+    }.asShortBuffer()
+}