starting on-chip software debugger (microsoft#2057)

This implements a debugger interface over HID running on interrupts in user space. It can step through the code and show simple values.

It also has the following fixes:
* only build natively tsprj/blockprj, not all the libraries
* fixes in re-connecting to HID
* support for events sent over HID

cli/buildengine.ts

@@ -316,7 +316,7 @@ function msdDeployCoreAsync(res: ts.pxtc.CompileResult) {
     if (pxt.appTarget.serial && pxt.appTarget.serial.useHF2) {
         let f = res.outfiles[pxtc.BINARY_UF2]
         let blocks = pxtc.UF2.parseFile(U.stringToUint8Array(atob(f)))
-        return hid.hf2DeviceAsync()
+        return hid.initAsync()
             .then(dev => dev.flashAsync(blocks))
     }
 

cli/cli.ts

@@ -23,6 +23,7 @@ import * as commandParser from './commandparser';
 import * as hid from './hid';
 import * as serial from './serial';
 import * as gdb from './gdb';
+import * as clidbg from './clidbg';
 import * as pyconv from './pyconv';
 
 const rimraf: (f: string, opts: any, cb: () => void) => void = require('rimraf');
@@ -1622,7 +1623,7 @@ function buildTargetCoreAsync() {
             .then(res => {
                 cfg.bundledpkgs[path.basename(dirname)] = res
             })
-            .then(testForBuildTargetAsync)
+            .then(() => testForBuildTargetAsync(isPrj))
             .then((compileOpts) => {
                 // For the projects, we need to save the base HEX file to the offline HEX cache
                 if (isPrj && pxt.appTarget.compile.hasHex) {
@@ -2604,26 +2605,35 @@ function testAssemblers(): Promise<void> {
 }
 
 
-function testForBuildTargetAsync(): Promise<pxtc.CompileOptions> {
+function testForBuildTargetAsync(useNative: boolean): Promise<pxtc.CompileOptions> {
     let opts: pxtc.CompileOptions
     return mainPkg.loadAsync()
         .then(() => {
             copyCommonFiles();
             let target = mainPkg.getTargetOptions()
             if (target.hasHex)
                 target.isNative = true
+            if (!useNative)
+                target.isNative = false
             return mainPkg.getCompileOptionsAsync(target)
         })
         .then(o => {
             opts = o
             opts.testMode = true
             opts.ast = true
-            return pxtc.compile(opts)
+            if (useNative)
+                return pxtc.compile(opts)
+            else {
+                pxt.log("  skip native build of non-project")
+                return null
+            }
         })
         .then(res => {
-            reportDiagnostics(res.diagnostics);
-            if (!res.success) U.userError("Compiler test failed")
-            simulatorCoverage(res, opts)
+            if (res) {
+                reportDiagnostics(res.diagnostics);
+                if (!res.success) U.userError("Compiler test failed")
+                simulatorCoverage(res, opts)
+            }
         })
         .then(() => opts);
 }
@@ -3317,26 +3327,42 @@ function prepBuildOptionsAsync(mode: BuildOption, quick = false) {
         })
 }
 
+function dbgTestAsync() {
+    return buildCoreAsync({
+        mode: BuildOption.JustBuild,
+        debug: true
+    })
+        .then(clidbg.startAsync)
+}
+
 interface BuildCoreOptions {
     mode: BuildOption;
 
+    debug?: boolean;
+
     // docs
     locs?: boolean;
     docs?: boolean;
     fileFilter?: string;
     createOnly?: boolean;
 }
 
-function buildCoreAsync(buildOpts: BuildCoreOptions): Promise<pxtc.CompileOptions> {
+function buildCoreAsync(buildOpts: BuildCoreOptions): Promise<pxtc.CompileResult> {
     let compileOptions: pxtc.CompileOptions;
+    let compileResult: pxtc.CompileResult;
     ensurePkgDir();
     return prepBuildOptionsAsync(buildOpts.mode)
         .then((opts) => {
             compileOptions = opts;
             opts.breakpoints = buildOpts.mode === BuildOption.DebugSim;
+            if (buildOpts.debug) {
+                opts.breakpoints = true
+                opts.justMyCode = true
+            }
             return pxtc.compile(opts);
         })
         .then((res): Promise<void | pxtc.CompileOptions> => {
+            compileResult = res
             U.iterMap(res.outfiles, (fn, c) => {
                 if (fn !== pxtc.BINARY_JS) {
                     mainPkg.host().writeFile(mainPkg, "built/" + fn, c);
@@ -3403,7 +3429,7 @@ function buildCoreAsync(buildOpts: BuildCoreOptions): Promise<pxtc.CompileOption
             }
         })
         .then(() => {
-            return compileOptions;
+            return compileResult;
         });
 }
 
@@ -4343,6 +4369,7 @@ function initCommands() {
     advancedCommand("testdir", "compile files in directory one by one", testDirAsync, "<dir>");
     advancedCommand("testconv", "test TD->TS converter", testConverterAsync, "<jsonurl>");
     advancedCommand("testpkgconflicts", "tests package conflict detection logic", testPkgConflictsAsync);
+    advancedCommand("testdbg", "tests hardware debugger", dbgTestAsync);
 
     advancedCommand("buildtarget", "build pxtarget.json", buildTargetAsync);
     advancedCommand("uploadtrg", "upload target release", pc => uploadTargetAsync(pc.arguments[0]), "<label>");

cli/clidbg.ts

@@ -0,0 +1,39 @@
+import * as hid from './hid';
+import * as fs from "fs";
+
+import Cloud = pxt.Cloud;
+import U = pxt.Util;
+import D = pxt.HWDBG;
+
+
+export function startAsync(compileRes: pxtc.CompileResult) {
+    return hid.initAsync()
+        .then(d => {
+            hid.connectSerial(d)
+
+            D.postMessage = msg => {
+                if (msg.subtype != "breakpoint") {
+                    console.log(msg)
+                    return
+                }
+                let bmsg = msg as pxsim.DebuggerBreakpointMessage
+
+                console.log("GLOBALS", bmsg.globals)
+                for (let s of bmsg.stackframes)
+                    console.log(s.funcInfo.functionName, s.locals)
+
+                let brkMatch = compileRes.breakpoints.filter(b => b.id == bmsg.breakpointId)[0]
+                if (!brkMatch) {
+                    console.log("Invalid breakpoint ID", msg)
+                    return
+                }
+                let lines = fs.readFileSync(brkMatch.fileName, "utf8").split(/\n/)
+
+                console.log(">>>", lines.slice(brkMatch.line, brkMatch.endLine + 1).join(" ;; "))
+                Promise.delay(500)
+                    .then(() => D.resumeAsync(false))
+            }
+
+            return D.startDebugAsync(compileRes, d)
+        })
+}

cli/hid.ts

@@ -32,14 +32,14 @@ export function listAsync() {
 }
 
 export function serialAsync() {
-    return hf2DeviceAsync()
+    return initAsync()
         .then(d => {
             connectSerial(d)
         })
 }
 
 export function dmesgAsync() {
-    return hf2DeviceAsync()
+    return initAsync()
         .then(d => d.talkAsync(pxt.HF2.HF2_CMD_DMESG)
             .then(resp => {
                 console.log(U.fromUTF8(U.uint8ArrayToString(resp)))
@@ -69,12 +69,8 @@ export function hf2ConnectAsync(path: string) {
 }
 
 let hf2Dev: Promise<HF2.Wrapper>
-export function hf2DeviceAsync(path: string = null): Promise<HF2.Wrapper> {
+export function initAsync(path: string = null): Promise<HF2.Wrapper> {
     if (!hf2Dev) {
-        let devs = getHF2Devices()
-        if (devs.length == 0)
-            return Promise.reject(new HIDError("no devices found"))
-        path = devs[0].path
         hf2Dev = hf2ConnectAsync(path)
     }
     return hf2Dev
@@ -100,17 +96,29 @@ export class HIDError extends Error {
 
 export class HidIO implements HF2.PacketIO {
     dev: any;
+    private path: string;
 
     onData = (v: Uint8Array) => { };
+    onEvent = (v: Uint8Array) => { };
     onError = (e: Error) => { };
 
-    constructor(private path: string) {
+    constructor(private requestedPath: string) {
         this.connect()
     }
 
     private connect() {
         const hid = getHID();
         U.assert(hid)
+
+        if (this.requestedPath == null) {
+            let devs = getHF2Devices()
+            if (devs.length == 0)
+                throw new HIDError("no devices found")
+            this.path = devs[0].path
+        } else {
+            this.path = this.requestedPath
+        }
+
         this.dev = new HID.HID(this.path)
         this.dev.on("data", (v: Buffer) => {
             //console.log("got", v.toString("hex"))

cli/server.ts

@@ -428,6 +428,16 @@ function initSocketServer(wsPort: number, hostname: string) {
                             case "init":
                                 return hio.reconnectAsync()
                                     .then(() => {
+                                        hio.io.onEvent = v => {
+                                            if (!ws) return
+                                            ws.send(JSON.stringify({
+                                                op: "event",
+                                                result: {
+                                                    path: msg.arg.path,
+                                                    data: U.toHex(v),
+                                                }
+                                            }))
+                                        }
                                         hio.onSerial = (v, isErr) => {
                                             if (!ws) return
                                             ws.send(JSON.stringify({

docs/debugger.md

@@ -0,0 +1,104 @@
+# On-device Software Debugger
+
+PXT supports debugging the generated TypeScript code. This is done via
+a USB HID interface implemented in software and doesn't require an interface 
+(debugging) chip. The USB HID interface uses 
+[HF2 protocol](https://github.com/Microsoft/uf2/blob/master/hf2.md).
+Messages from the host (computer connected to the device being debugged)
+are handled in the USB interrupt of the device.
+
+Debugging needs to be enabled during compilation. The switch will cause breakpoint
+code to be inserted before every statement. It's possible to enable debug for all
+TypeScript code, or only user code (i.e, excluding everything under `pxt_modules/**`).
+
+## Debugger features (TODO)
+
+* [x] inspecting globals (including `uint16` and friends)
+* [x] inspecting tagged values, doubles, strings, buffers
+* [x] `debugger` statement support
+* [x] step-into support
+* [x] step-over support
+* [x] continue support
+* [x] inspecting locals in current function, and up the call stack
+
+* [ ] stack-walking with first-class functions
+* [ ] inspecting contents of records
+* [ ] inspecting contents of arrays
+* [ ] inspecting contents of maps
+* [ ] inspecting contents of reference-locals (captured ones)
+* [ ] inspecting active threads
+* [ ] user-defined breakpoints (other than `debugger` statement)
+* [ ] handling of non-intentional faults (right now we get infinite loop; not sure how useful this is without GDB)
+
+## Debugger implementation
+
+The debug mode is controlled by the first global variable. In C++ code it's
+referred to as `globals[0]`, while in assembly it's `[r6, #0]` (`r6` holds `globals`
+while TypeScript code is executing).
+
+The C++ side of the code sits in `hf2.cpp` file in `core` library (in `pxt-common-packages`).
+Current link: https://github.com/Microsoft/pxt-common-packages/blob/dbg/libs/core/hf2.cpp#L182
+
+There are currently 4 possible values of `globals[0]`
+* a valid heap address, somewhere in the middle - normal execution
+* `0` - debugger connected, no single stepping mode
+* `1` - debugger connected, step-into mode
+* `3` - debugger connected, step-over mode (replaced with `0` on first hit)
+
+The host will start by resetting the device into debug mode with a specific HF2
+message. This causes `globals[0]` to be set to `1`.
+After that, the device will send a message to the host saying it's halted.
+Then the host will eventually set `globals[0]` to `0`, `1` or `3` and
+resume execution via another HF2 message.
+
+TS `debugger` statement is translated into a word fetch from `globals[0] - 4`.
+In any mode other than the normal execution, it will cause a fault - either alignment fault
+or accessing memory at `0xfffffffc`.
+
+```
+    ldr r0, [r6, #0]
+    subs r0, r0, #4
+    ldr r0, [r0, #0]
+```
+
+Additionally, in front of all other statements, provided debugging is enabled,
+we try to load memory location pointed to by `globals[0]`. If `globals[0]`
+is `1` or `3` (but not `0` or valid heap address), this will cause alignment fault.
+
+```
+    ldr r0, [r6, #0]
+    ldr r0, [r0, #0]
+```
+
+All hard faults are caught by a single handler. It then looks at
+the instruction causing fault and if it is `ldr r0, [r0, #0]`,
+it will return to a function which pauses all other user
+fibers and executes `fiber_sleep()` in a loop.
+It also sends a HF2 message to the host stating that the device
+if halted. At some point, a HF2 message from the host clears
+a flag causing the loop to stop and the device to resume execution after the 
+`ldr r0, [r0, #0]` instruction (after unpausing all other user fibers).
+
+This is enough to support step-into and continue modes of operation.
+To support step-over, a instructions sequence similar to the one above 
+is used at the entry of every procedure, before even the `push {lr}`:
+
+```
+    ldr r0, [r6, #0]
+    ldr r0, [r0, #4]
+```
+
+If the fault handler detects the fault instruction to be `ldr r0, [r0, #4]`
+and `globals[0]` is `3` it sets `globals[0]` to `0` and modifies the
+contents of `lr` register by setting its highest bit. In all cases, when it's done,
+it just resumes execution after the `ldr r0, [r0, #4]` instruction
+(without entering the pause loop).
+
+Thus, in step-over mode, entry to any user procedure will set `globals[0]` to
+`0` preventing further instruction breakpoints from firing. When this particular
+function tries to return, we get another fault, where the `pc` is set to
+an address with highest bit set. This is detected, the `globals[0]` is set 
+back to `3`, the `pc` is corrected and execution continues.
+
+
+

docs/language.md

@@ -106,7 +106,7 @@ There are two compilation strategies available - the legacy strategy used by the
 micro:bit target, and a tagged strategy used by the upcoming SAMD21 targets, as well as all
 the other targets going forward (possibly including new version of the micro:bit target).
 
-In the legacy strategy, there are some semantic differences with JavaScript,
+In the **legacy strategy**, there are some semantic differences with JavaScript,
 particularly:
 * numbers are 32 bit signed integers with wrap-around semantics; 
   in JavaScript they are 64 bit floating points
@@ -157,8 +157,8 @@ strategy.
 Compared to a typical dynamic JavaScript engine, PXT compiles code statically,
 giving rise to significant time and space performance improvements:
 * user programs are compiled directly to machine code, and are
-  never in any byte-code form that needs to be interpreted; this results in execution
-  10-20x faster than a typical JS interpreter
+  never in any byte-code form that needs to be interpreted; this results in
+  much faster execution than a typical JS interpreter
 * there is no RAM overhead for user-code - all code sits in flash; in a dynamic VM
   there are usually some data-structures representing code
 * due to lack of boxing for small integers and static class layout the memory consumption for objects
@@ -200,6 +200,13 @@ The supported types are:
 If you attempt to store a number exceeding the range of the small int type, only
 the lowest 8 or 16 bits will be stored. There is no clamping nor overflow exceptions.
 
+If you just use `number` type (or specify no type at all) in tagged strategy,
+then if the number fits in signed 31 bits, 4 bytes of memory will be used.
+Otherwise, the 4 bytes will point to a heap-allocated double (all together,
+with memory allocator overhead, around 20 bytes).
+
+In legacy strategy, `number` is equivalent to `int32`, and there is no `uint32`.
+
 ### Limitations
 
 * arrays of int types are currently not supported; you can use a `Buffer` instead

libs/pxt-common/pxt-core.d.ts

@@ -215,7 +215,7 @@ interface RegExp { }
 
 type uint8 = number;
 type uint16 = number;
-//type uint32 = number;
+type uint32 = number;
 type int8 = number;
 type int16 = number;
 type int32 = number;