From 2ed160377362fc05f749a45b975d9019c1da98ff Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Mon, 18 Feb 2019 20:25:15 -0500 Subject: [PATCH 01/15] Keypoints visual circles --- .../src/components/annotator/Annotation.vue | 22 ++++++- client/src/components/annotator/Category.vue | 2 +- .../annotator/tools/KeypointTool.vue | 61 +++++++++++++++++++ client/src/views/Annotator.vue | 8 +++ 4 files changed, 91 insertions(+), 2 deletions(-) create mode 100755 client/src/components/annotator/tools/KeypointTool.vue diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index 1393671d..e5843772 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -166,6 +166,7 @@ export default { isVisible: true, color: this.annotation.color, compoundPath: null, + keypointsPath: null, metadata: [], isEmpty: true, name: "", @@ -210,10 +211,13 @@ export default { } } + + if (this.keypointsPath != null) this.keypointsPath.remove(); if (this.compoundPath != null) this.compoundPath.remove(); // Create new compoundpath this.compoundPath = new paper.CompoundPath(); + this.keypointsPath = new paper.CompoundPath(); if (json != null) { // Import data directroy from paperjs object @@ -337,6 +341,11 @@ export default { this.compoundPath = this.pervious.pop(); this.compoundPath.fullySelected = this.isCurrent; }, + addKeypoint(point) { + let pointPath = new paper.Path.Circle(point, 5); + + this.keypointsPath.addChild(pointPath); + }, /** * Unites current annotation path with anyother path. * @param {paper.CompoundPath} compound compound to unite current annotation path with @@ -381,9 +390,12 @@ export default { this.compoundPath.fillColor = this.color; let h = Math.round(this.compoundPath.fillColor.hue); - let l = Math.round((this.compoundPath.fillColor.lightness - 0.2) * 100); + let l = Math.round(this.compoundPath.fillColor.lightness * 50); let s = Math.round(this.compoundPath.fillColor.saturation * 100); this.compoundPath.strokeColor = "hsl(" + h + "," + s + "%," + l + "%)"; + + this.keypointsPath.fillColor = "hsl(" + h + "," + s + "%," + l * 2.5 + "%)"; + this.keypointsPath.strokeColor = "hsl(" + h + "," + s + "%," + l * 0.5 + "%)"; }, export() { if (this.compoundPath == null) this.createCompoundPath(); @@ -432,6 +444,7 @@ export default { if (this.compoundPath == null) return; this.compoundPath.visible = newVisible; + }, compoundPath() { if (this.compoundPath == null) return; @@ -441,6 +454,13 @@ export default { this.setColor(); this.isEmpty = this.compoundPath.isEmpty(); }, + keypointsPath() { + if (this.keypointsPath == null) return; + + this.keypointsPath.visible = this.isVisible; + this.$parent.group.addChild(this.keypointsPath); + this.setColor(); + }, annotation() { this.initAnnotation(); }, diff --git a/client/src/components/annotator/Category.vue b/client/src/components/annotator/Category.vue index cf39da10..4e0c89d6 100755 --- a/client/src/components/annotator/Category.vue +++ b/client/src/components/annotator/Category.vue @@ -336,7 +336,7 @@ export default { if (this.group != null) { this.group.fillColor = this.color; let h = Math.round(this.group.fillColor.hue); - let l = Math.round((this.group.fillColor.lightness - 0.2) * 100); + let l = Math.round(this.group.fillColor.lightness * 50); let s = Math.round(this.group.fillColor.saturation * 100); this.group.strokeColor = "hsl(" + h + "," + s + "%," + l + "%)"; } diff --git a/client/src/components/annotator/tools/KeypointTool.vue b/client/src/components/annotator/tools/KeypointTool.vue new file mode 100755 index 00000000..92f824c2 --- /dev/null +++ b/client/src/components/annotator/tools/KeypointTool.vue @@ -0,0 +1,61 @@ + diff --git a/client/src/views/Annotator.vue b/client/src/views/Annotator.vue index 3b2c892a..da3ca457 100755 --- a/client/src/views/Annotator.vue +++ b/client/src/views/Annotator.vue @@ -39,6 +39,12 @@ @setcursor="setCursor" ref="eraser" /> + +
@@ -186,6 +192,7 @@ import SelectTool from "@/components/annotator/tools/SelectTool"; import MagicWandTool from "@/components/annotator/tools/MagicWandTool"; import EraserTool from "@/components/annotator/tools/EraserTool"; import BrushTool from "@/components/annotator/tools/BrushTool"; +import KeypointTool from "@/components/annotator/tools/KeypointTool"; import CopyAnnotationsButton from "@/components/annotator/tools/CopyAnnotationsButton"; import CenterButton from "@/components/annotator/tools/CenterButton"; @@ -219,6 +226,7 @@ export default { MagicWandTool, EraserTool, BrushTool, + KeypointTool, DownloadButton, SaveButton, SettingsButton, From c72048b904f8df04e80c5d5d96fc4732a79cf0d9 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Tue, 19 Feb 2019 22:26:37 -0500 Subject: [PATCH 02/15] Keypoints paperjs object --- .../src/components/annotator/Annotation.vue | 45 ++++++-- client/src/components/annotator/Category.vue | 18 ++-- client/src/libs/keypoints.js | 102 ++++++++++++++++++ 3 files changed, 148 insertions(+), 17 deletions(-) create mode 100644 client/src/libs/keypoints.js diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index e5843772..68fb04a9 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -117,6 +117,7 @@ import paper from "paper"; import axios from "axios"; import simplifyjs from "simplify-js"; +import Keypoint from "@/libs/keypoints"; import { mapMutations } from "vuex"; import UndoAction from "@/undo"; @@ -166,12 +167,14 @@ export default { isVisible: true, color: this.annotation.color, compoundPath: null, - keypointsPath: null, + keypoints: [], metadata: [], isEmpty: true, name: "", uuid: "", - pervious: [] + pervious: [], + skeleton: [[1, 2], [3, 5], [3, 6]], + count: 0 }; }, methods: { @@ -211,13 +214,10 @@ export default { } } - - if (this.keypointsPath != null) this.keypointsPath.remove(); if (this.compoundPath != null) this.compoundPath.remove(); // Create new compoundpath this.compoundPath = new paper.CompoundPath(); - this.keypointsPath = new paper.CompoundPath(); if (json != null) { // Import data directroy from paperjs object @@ -342,9 +342,32 @@ export default { this.compoundPath.fullySelected = this.isCurrent; }, addKeypoint(point) { - let pointPath = new paper.Path.Circle(point, 5); + this.count++; + let keypoint = new Keypoint(point.x, point.y, { + indexLabel: this.count + }); + + let edges = this.skeleton.filter(e => e.includes(this.count)); + console.log(this.count) + edges.forEach(edge => { + let otherIndex = edge[0]; + if (otherIndex == keypoint.indexLabel) + otherIndex = edge[1]; + + let otherKeypoint = this.keypoints.find(k => k.indexLabel === otherIndex); + if (otherKeypoint != null) { + let line = new paper.Path.Line(otherKeypoint, keypoint); + keypoint.addLine(line); + otherKeypoint.path.bringToFront(); + keypoint.path.bringToFront(); + } + }) + + this.keypoints.push(keypoint); + this.setColor(); + }, + drawLines() { - this.keypointsPath.addChild(pointPath); }, /** * Unites current annotation path with anyother path. @@ -392,10 +415,10 @@ export default { let h = Math.round(this.compoundPath.fillColor.hue); let l = Math.round(this.compoundPath.fillColor.lightness * 50); let s = Math.round(this.compoundPath.fillColor.saturation * 100); - this.compoundPath.strokeColor = "hsl(" + h + "," + s + "%," + l + "%)"; - this.keypointsPath.fillColor = "hsl(" + h + "," + s + "%," + l * 2.5 + "%)"; - this.keypointsPath.strokeColor = "hsl(" + h + "," + s + "%," + l * 0.5 + "%)"; + let hsl = "hsl(" + h + "," + s + "%," + l + "%)"; + this.compoundPath.strokeColor = hsl; + this.keypoints.forEach(k => (k.color = hsl)); }, export() { if (this.compoundPath == null) this.createCompoundPath(); @@ -444,6 +467,8 @@ export default { if (this.compoundPath == null) return; this.compoundPath.visible = newVisible; + + this.keypoints.forEach(k => (k.visible = newVisible)); }, compoundPath() { diff --git a/client/src/components/annotator/Category.vue b/client/src/components/annotator/Category.vue index 4e0c89d6..bd144fd4 100755 --- a/client/src/components/annotator/Category.vue +++ b/client/src/components/annotator/Category.vue @@ -325,20 +325,21 @@ export default { setColor() { if (!this.isVisible) return; + let annotations = this.$refs.annotation; if (this.showAnnotations) { - let annotations = this.$refs.annotation; - if (annotations) { - annotations.forEach(annotation => { - annotation.setColor(); - }); - } + annotations.forEach(a => a.setColor()); } else { if (this.group != null) { this.group.fillColor = this.color; let h = Math.round(this.group.fillColor.hue); let l = Math.round(this.group.fillColor.lightness * 50); let s = Math.round(this.group.fillColor.saturation * 100); - this.group.strokeColor = "hsl(" + h + "," + s + "%," + l + "%)"; + let hsl = "hsl(" + h + "," + s + "%," + l + "%)"; + this.group.strokeColor = hsl; + + annotations.forEach(a => { + a.keypoints.forEach(k => k.color = hsl); + }); } } }, @@ -392,6 +393,9 @@ export default { isVisible(newVisible) { if (this.group == null) return; this.group.visible = newVisible; + this.$refs.annotation.forEach(a => { + a.keypoints.forEach(k => (k.visible = newVisible)); + }) this.setColor(); }, showAnnotations(showing) { diff --git a/client/src/libs/keypoints.js b/client/src/libs/keypoints.js new file mode 100644 index 00000000..ad049d33 --- /dev/null +++ b/client/src/libs/keypoints.js @@ -0,0 +1,102 @@ +import paper from "paper"; + +export default class extends paper.Point { + static NOT_LABELED = 0; + static LABELED_NOT_VISIBLE = 1; + static LABELED_VISIBLE = 2; + + constructor(x, y, args) { + super(x, y); + + this.path = null; + + this.radius = args.radius || 5; + this.label = args.label || ""; + this.indexLabel = args.indexLabel || -1; + this.edges = args.edges || []; + this.visibility = args.visibility || this.NOT_LABELED; + + this._draw(); + this.color = "white"; + } + + fillColor() { + if (this.path == null) return; + + switch (this.visibility) { + case this.NOT_LABELED: + this.path.fillColor = "black"; + break; + case this.LABELED_NOT_VISIBLE: + this.path.fillColor = "white"; + break; + default: + this.path.fillColor = this.color; + } + } + + addLine(line) { + this.edges.push(line); + line.strokeWidth = 2; + line.strokeColor = this.color; + } + + _draw() { + if (this.path !== null) { + this.path.remove(); + } + + this.path = new paper.Path.Circle(this, this.radius); + this.path.strokeColor = this.color; + this.path.strokeWidth = 3; + this.path.keypoint = this; + + this.fillColor(); + } + + set visible(val) { + this.path.visible = val; + this.edges.forEach(e => (e.visible = val)); + } + + get visible() { + return this.path.visible; + } + + set visibility(val) { + this._visibility = val; + this.fillColor(); + } + + get visibility() { + return this._visibility; + } + + set radius(val) { + this._radius = val; + this._draw(); + } + + get radius() { + return this._radius; + } + + set color(val) { + this._color = val; + this.strokeColor = val; + this.fillColor = val; + this.edges.forEach(e => (e.strokeColor = val)); + } + + get color() { + return this.path.fillColor; + } + + set strokeColor(val) { + this.path.strokeColor = val; + } + + get strokeColor() { + return this.path.strokeColor; + } +} From 99ddacd7e2068b4029ede489b4fc83192e3ec50a Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Wed, 20 Feb 2019 09:19:28 -0500 Subject: [PATCH 03/15] Add keypoints to models --- app/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models.py b/app/models.py index 07dab566..d9b3d929 100644 --- a/app/models.py +++ b/app/models.py @@ -221,6 +221,8 @@ class AnnotationModel(db.DynamicDocument): height = db.IntField() color = db.StringField() + + keypoints = db.ListField(default=[]) metadata = db.DictField(default={}) paper_object = db.ListField(default=[]) @@ -311,6 +313,11 @@ class CategoryModel(db.DynamicDocument): deleted = db.BooleanField(default=False) deleted_date = db.DateTimeField() + # keypoints + edges = db.ListField(default=[]) + point_labels = db.ListField(default=[]) + + @classmethod def bulk_create(cls, categories): @@ -462,6 +469,7 @@ def api_json(self): "name": self.name } + class CocoImportModel(db.DynamicDocument): id = db.SequenceField(primary_key=True) creator = db.StringField(required=True) From 0b67eede4314d97dc3e95a40a35477be6da1eeb1 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Wed, 20 Feb 2019 16:29:57 -0500 Subject: [PATCH 04/15] Created Keypoints object --- .../src/components/annotator/Annotation.vue | 2 +- client/src/libs/keypoints.js | 281 ++++++++++++++++-- 2 files changed, 255 insertions(+), 28 deletions(-) diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index 68fb04a9..626c5df0 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -117,7 +117,7 @@ import paper from "paper"; import axios from "axios"; import simplifyjs from "simplify-js"; -import Keypoint from "@/libs/keypoints"; +import { Keypoint, keypoints } from "@/libs/keypoints"; import { mapMutations } from "vuex"; import UndoAction from "@/undo"; diff --git a/client/src/libs/keypoints.js b/client/src/libs/keypoints.js index ad049d33..e6cb51cf 100644 --- a/client/src/libs/keypoints.js +++ b/client/src/libs/keypoints.js @@ -1,33 +1,251 @@ import paper from "paper"; -export default class extends paper.Point { - static NOT_LABELED = 0; - static LABELED_NOT_VISIBLE = 1; - static LABELED_VISIBLE = 2; +export class Keypoints extends paper.Group { + constructor(labels, edges, keypoints, args) { + super(); + args = args || {}; + this.labels = new Set(labels); + this._edges = {}; + + this._lines = {}; + this._labelled = {}; + this._keypoints = []; + + this.strokeColor = args.strokeColor || "red"; + this.lineWidth = args.strokeWidth || 4; + + edges.forEach(e => this.addEdge(e)); + + if (keypoints) { + for (let i = 0; i < keypoints.length; i += 3) { + let x = keypoints[i], + y = keypoints[i + 1], + v = keypoints[i + 2]; + let point = new Keypoint(x, y, { visibility: v, indexLabel: i }); + this.addKeypoint(point); + } + } + } + + addKeypoint(keypoint) { + keypoint.keypoints = this; + keypoint.path.keypoints = this; + keypoint.color = this.strokeColor; + keypoint.path.strokeWidth = this.strokeWidth; + + let indexLabel = keypoint.indexLabel; + if (this._labelled.hasOwnProperty(indexLabel)) { + keypoint.indexLabel = -1; + } else { + this._labelled[indexLabel] = keypoint; + } + + this._keypoints.push(keypoint); + this.addChild(keypoint.path); + this._drawLines(keypoint); + } + + moveKeypoint(point, keypoint) { + let indexLabel = keypoint.indexLabel; + + let edges = this._edges[indexLabel]; + if (edges) { + edges.forEach(i => { + let line = this.getLine([i, indexLabel]); + if (line) { + // We need to move the line aswell + for (let i = 0; i < line.segments.length; i++) { + let segment = line.segments[i]; + if (segment.point.isClose(keypoint, 0)) { + segment.point = point; + break; + } + } + } + }); + } + keypoint.move(point); + } + + set visible(val) { + this._visible = val; + this._keypoints.forEach(k => (k.visible = val)); + Object.values(this._lines).forEach(l => (l.visible = val)); + } + + get visible() { + return this._visible; + } + + set color(val) { + this._color = val; + this.strokeColor = val; + this._keypoints.forEach(k => (k.color = val)); + } + + get color() { + return this._color; + } + + set lineWidth(val) { + this._lineWidth = val; + this.strokeWidth = val; + this._keypoints.forEach(k => (k.path.storkeWidth = val)); + } + + get lineWidth() { + return this._lineWidth; + } + + set radius(val) { + this._radius = val; + this._keypoints.forEach(k => (k.radius = val)); + } + + get radius() { + return this._radius; + } + + exportJSON() { + let array = []; + this._keypoints.forEach(k => { + array.push(...[k.x, k.y, k.visibility]); + }); + return array; + } + + addEdge(edge) { + if (edge.length !== 2) return; + + let i1 = edge[0]; + let i2 = edge[1]; + + // If labels convert to indexs + if (typeof i1 == "string") i1 = this.getLabelIndex(i1); + if (typeof i2 == "string") i2 = this.getLabelIndex(i2); + if (i1 < 0 || i2 < 0) return; + + this._addEdgeIndex(i1, i2); + this._addEdgeIndex(i2, i1); + + // Draw line if points exist + let k1 = this._labelled[i1]; + let k2 = this._labelled[i2]; + if (k1 && k2) { + this._drawLine(edge, k1, k2); + } + } + + getLabelIndex(label) { + return this.labels.find(l => l == label); + } + + _addEdgeIndex(index1, index2) { + if (this._edges.hasOwnProperty(index1)) { + if (!this._edges[index1].has(index2)) this._edges[index1].add(index2); + } else { + this._edges[index1] = new Set([index2]); + } + } + + /** + * Draws lines to other keypoints if they exist + */ + _drawLines(keypoint) { + if (keypoint.indexLabel < 0) return; + if (!this._edges.hasOwnProperty(keypoint.indexLabel)) return; + + let otherIndices = this._edges[keypoint.indexLabel]; + otherIndices.forEach(i => { + let k2 = this._labelled[i]; + if (!k2) return; + + let edge = [keypoint.indexLabel, i]; + this._drawLine(edge, keypoint, k2); + }); + } + + /** + * Draws a line between two keypoints and hashes to a table for quick look up + * @param {list} edge array of two elementings contain the index edges + * @param {Keypoint} firstKeypoint first keypoint object + * @param {Keypoint} secondKeypoint second keypoint object + */ + _drawLine(edge, firstKeypoint, secondKeypoint) { + let h = this._hashEdge(edge); + if (this._lines[h]) return; + + let line = new paper.Path.Line(firstKeypoint, secondKeypoint); + line.strokeColor = this.strokeColor; + line.strokeWidth = this.strokeWidth; + if (firstKeypoint.path.isBelow(secondKeypoint.path)) { + line.insertBelow(firstKeypoint.path); + } else { + line.insertBelow(secondKeypoint.path); + } + + this._lines[h] = line; + } + + /** + * Returns paperjs path of line [O(1) lookup time] + * @param {list} edge array of two elementing contains the index edges + * @returns paperjs object path of the line or undefind if not found + */ + getLine(edge) { + let h = this._hashEdge(edge); + return this._lines[h]; + } + + /** + * Uses cantor pairing function to has two numbers + * @param {list} edge array of two elementing contains the index edges + */ + _hashEdge(edge) { + // Order doesn't matter so can sort first + let min = Math.min(edge[0], edge[1]); + let max = Math.max(edge[0], edge[1]); + // Cantor pairing function + let add = min + max; + return (1 / 2) * add * (add - 1) - max; + } +} + +/** + * Keypoint visibility types as defined by the COCO format + */ +export let VisibilityType = { + NOT_LABELED: 0, + LABELED_NOT_VISIBLE: 1, + LABELED_VISIBLE: 2 +}; + +export class Keypoint extends paper.Point { constructor(x, y, args) { super(x, y); + args = args || {}; this.path = null; - this.radius = args.radius || 5; this.label = args.label || ""; + this.radius = args.radius || 5; this.indexLabel = args.indexLabel || -1; - this.edges = args.edges || []; - this.visibility = args.visibility || this.NOT_LABELED; - + this.visibility = args.visibility || VisibilityType.NOT_LABELED; + this.visible = args.visible || true; this._draw(); - this.color = "white"; + this.color = args.color || "red"; + this.setFillColor(); } - fillColor() { + setFillColor() { if (this.path == null) return; switch (this.visibility) { - case this.NOT_LABELED: + case VisibilityType.NOT_LABELED: this.path.fillColor = "black"; break; - case this.LABELED_NOT_VISIBLE: + case VisibilityType.LABELED_NOT_VISIBLE: this.path.fillColor = "white"; break; default: @@ -35,37 +253,47 @@ export default class extends paper.Point { } } - addLine(line) { - this.edges.push(line); - line.strokeWidth = 2; - line.strokeColor = this.color; + move(point) { + this.x = point.x; + this.y = point.y; + this._draw(); } _draw() { + let storkeWidth = 1; if (this.path !== null) { + storkeWidth = this.path.strokeWidth; this.path.remove(); } this.path = new paper.Path.Circle(this, this.radius); + this.path.onMouseDrag = event => { + if (this.keypoints) { + this.keypoints.moveKeypoint(event.point, this); + } else { + this.move(event.point); + } + }; this.path.strokeColor = this.color; - this.path.strokeWidth = 3; + this.path.strokeWidth = storkeWidth; + this.path.visible = this.visible; this.path.keypoint = this; - this.fillColor(); + this.setFillColor(); } set visible(val) { + this._visible = val; this.path.visible = val; - this.edges.forEach(e => (e.visible = val)); } get visible() { - return this.path.visible; + return this._visible; } set visibility(val) { this._visibility = val; - this.fillColor(); + this.setFillColor(); } get visibility() { @@ -83,20 +311,19 @@ export default class extends paper.Point { set color(val) { this._color = val; - this.strokeColor = val; - this.fillColor = val; - this.edges.forEach(e => (e.strokeColor = val)); + this.path.strokeColor = val; + this.setFillColor(); } get color() { - return this.path.fillColor; + return this._color; } set strokeColor(val) { - this.path.strokeColor = val; + this.color = val; } get strokeColor() { - return this.path.strokeColor; + return this.color; } } From 22e843827ab91a7ed6236afbbaae89b84afceab6 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Wed, 20 Feb 2019 21:52:41 -0500 Subject: [PATCH 05/15] Keypoint visualization logic --- app/api/annotator.py | 7 +- app/models.py | 7 +- .../src/components/annotator/Annotation.vue | 188 ++++++++++++++---- client/src/components/annotator/Category.vue | 34 +++- .../components/annotator/tools/SelectTool.vue | 14 +- client/src/libs/keypoints.js | 110 ++++++++-- 6 files changed, 276 insertions(+), 84 deletions(-) diff --git a/app/api/annotator.py b/app/api/annotator.py index e95278ad..53071495 100644 --- a/app/api/annotator.py +++ b/app/api/annotator.py @@ -46,7 +46,11 @@ def post(self): if db_category is None: continue - db_category.update(set__color=category.get('color')) + db_category.update( + set__color=category.get('color'), + set__keypoint_edges=category.get('keypoint_edges', []), + set__keypoint_labels=category.get('keypoint_labels', []) + ) # Iterate every annotation from the data annotations for annotation in category.get('annotations', []): @@ -63,6 +67,7 @@ def post(self): # Update annotation in database db_annotation.update( + set__keypoints=annotation.get('keypoints', []), set__metadata=annotation.get('metadata'), set__color=annotation.get('color') ) diff --git a/app/models.py b/app/models.py index d9b3d929..2fa280b4 100644 --- a/app/models.py +++ b/app/models.py @@ -222,7 +222,7 @@ class AnnotationModel(db.DynamicDocument): color = db.StringField() - keypoints = db.ListField(default=[]) + UNKNOWN = db.ListField(default=[]) metadata = db.DictField(default={}) paper_object = db.ListField(default=[]) @@ -313,9 +313,8 @@ class CategoryModel(db.DynamicDocument): deleted = db.BooleanField(default=False) deleted_date = db.DateTimeField() - # keypoints - edges = db.ListField(default=[]) - point_labels = db.ListField(default=[]) + keypoint_edges = db.ListField(default=[]) + keypoint_labels = db.ListField(default=[]) @classmethod diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index 626c5df0..12b1f08f 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -51,6 +51,64 @@ style="float:right" /> +
e.includes(this.count)); - console.log(this.count) - edges.forEach(edge => { - let otherIndex = edge[0]; - if (otherIndex == keypoint.indexLabel) - otherIndex = edge[1]; - - let otherKeypoint = this.keypoints.find(k => k.indexLabel === otherIndex); - if (otherKeypoint != null) { - let line = new paper.Path.Line(otherKeypoint, keypoint); - keypoint.addLine(line); - otherKeypoint.path.bringToFront(); - keypoint.path.bringToFront(); + onClick: (event) => { + this.currentKeypoint = event.target.keypoint + let id = `#keypointSettings${this.annotation.id}`; + let indexLabel = this.currentKeypoint.indexLabel; + + this.keypoint.tag = indexLabel == -1 ? [] : [indexLabel.toString()]; + this.keypoint.visibility = this.currentKeypoint.visibility; + + $(id).modal("show"); } - }) - - this.keypoints.push(keypoint); - this.setColor(); - }, - drawLines() { + }); + this.keypoints.addKeypoint(keypoint); + this.isEmpty = this.compoundPath.isEmpty() && this.keypoints.isEmpty(); }, /** * Unites current annotation path with anyother path. @@ -418,7 +486,7 @@ export default { let hsl = "hsl(" + h + "," + s + "%," + l + "%)"; this.compoundPath.strokeColor = hsl; - this.keypoints.forEach(k => (k.color = hsl)); + this.keypoints.color = hsl; }, export() { if (this.compoundPath == null) this.createCompoundPath(); @@ -437,6 +505,15 @@ export default { asString: false, precision: 1 }); + + if (!this.keypoints.isEmpty()) { + annotationData.keypoints = this.keypoints.exportJSON( + this.keypointLabels, + this.annotation.width, + this.annotation.height + ); + } + this.compoundPath.fullySelected = this.isCurrent; if (this.annotation.paper_object !== json) { annotationData.compoundPath = json; @@ -467,9 +544,7 @@ export default { if (this.compoundPath == null) return; this.compoundPath.visible = newVisible; - - this.keypoints.forEach(k => (k.visible = newVisible)); - + this.keypoints.visible = newVisible; }, compoundPath() { if (this.compoundPath == null) return; @@ -477,14 +552,10 @@ export default { this.compoundPath.visible = this.isVisible; this.$parent.group.addChild(this.compoundPath); this.setColor(); - this.isEmpty = this.compoundPath.isEmpty(); + this.isEmpty = this.compoundPath.isEmpty() && this.keypoints.isEmpty(); }, - keypointsPath() { - if (this.keypointsPath == null) return; - - this.keypointsPath.visible = this.isVisible; - this.$parent.group.addChild(this.keypointsPath); - this.setColor(); + keypoints() { + this.isEmpty = this.compoundPath.isEmpty() && this.keypoints.isEmpty(); }, annotation() { this.initAnnotation(); @@ -492,6 +563,14 @@ export default { isCurrent() { if (this.compoundPath == null) return; this.compoundPath.fullySelected = this.isCurrent; + }, + "keypoint.tag"(newVal) { + let id = newVal.length === 0 ? -1 : newVal[0]; + this.keypoints.setKeypointIndex(this.currentKeypoint, id) + this.tagRecomputeCounter++; + }, + "keypoint.visibility"(newVal) { + this.currentKeypoint.visibility = newVal; } }, computed: { @@ -518,6 +597,29 @@ export default { if (search === String(this.annotation.id)) return true; if (search === String(this.index + 1)) return true; return this.name.toLowerCase().includes(this.search); + }, + hsl() { + if (this.compoundPath == null) return [0, 0, 0]; + let h = Math.round(this.compoundPath.fillColor.hue); + let l = Math.round(this.compoundPath.fillColor.lightness * 50); + let s = Math.round(this.compoundPath.fillColor.saturation * 100); + + return [h, s, l] + }, + keypointLabelTags() { + this.tagRecomputeCounter; + let tags = {}; + + for(let i = 0; i < this.keypointLabels.length; i++) { + // Include it tags if it is the current keypoint or not in use. + if ( + (this.keypoints && !this.keypoints._labelled[i+1]) || + (this.currentKeypoint &&this.currentKeypoint.indexLabel == i+1) + ) { + tags[i+1] = this.keypointLabels[i]; + } + } + return tags; } }, sockets: { diff --git a/client/src/components/annotator/Category.vue b/client/src/components/annotator/Category.vue index bd144fd4..44c5666b 100755 --- a/client/src/components/annotator/Category.vue +++ b/client/src/components/annotator/Category.vue @@ -69,6 +69,8 @@ @click="onAnnotationClick(listIndex)" :opacity="opacity" :index="listIndex" + :keypointEdges="keypoint.edges" + :keypointLabels="keypoint.labels" ref="annotation" :hover="hover.annotation" @deleted="annotationDeleted" @@ -102,6 +104,16 @@
+ +
+ + +
@@ -124,10 +136,11 @@ import paper from "paper"; import axios from "axios"; import Annotation from "@/components/annotator/Annotation"; +import TagsInput from "@/components/TagsInput"; export default { name: "Category", - components: { Annotation }, + components: { Annotation, TagsInput }, props: { category: { type: Object, @@ -166,6 +179,10 @@ export default { return { group: null, color: this.category.color, + keypoint: { + labels: this.category.keypoint_labels, + edges: this.category.keypoint_edges + }, selectedAnnotation: -1, showAnnotations: false, isVisible: false, @@ -237,7 +254,9 @@ export default { visualize: this.isVisible, color: this.color, metadata: [], - annotations: [] + annotations: [], + keypoint_labels: this.keypoint.labels, + keypoint_edges: this.keypoint.edges }; if (refs.hasOwnProperty("annotation")) { @@ -327,7 +346,7 @@ export default { let annotations = this.$refs.annotation; if (this.showAnnotations) { - annotations.forEach(a => a.setColor()); + if (annotations) annotations.forEach(a => a.setColor()); } else { if (this.group != null) { this.group.fillColor = this.color; @@ -337,9 +356,7 @@ export default { let hsl = "hsl(" + h + "," + s + "%," + l + "%)"; this.group.strokeColor = hsl; - annotations.forEach(a => { - a.keypoints.forEach(k => k.color = hsl); - }); + if (annotations) annotations.forEach(a => a.keypoints.color = hsl); } } }, @@ -393,9 +410,8 @@ export default { isVisible(newVisible) { if (this.group == null) return; this.group.visible = newVisible; - this.$refs.annotation.forEach(a => { - a.keypoints.forEach(k => (k.visible = newVisible)); - }) + let annotations = this.$refs.annotation; + if (annotations) annotations.forEach(a => a.keypoints.visible = newVisible); this.setColor(); }, showAnnotations(showing) { diff --git a/client/src/components/annotator/tools/SelectTool.vue b/client/src/components/annotator/tools/SelectTool.vue index d426eacb..5f82d69f 100755 --- a/client/src/components/annotator/tools/SelectTool.vue +++ b/client/src/components/annotator/tools/SelectTool.vue @@ -46,14 +46,7 @@ export default { fill: false, tolerance: 5, match: hit => { - if (this.point == null) return true; - if ( - hit.item instanceof paper.Path || - hit.item instanceof paper.CompoundPath - ) { - return !hit.item.hasOwnProperty("indicator"); - } - return true; + return !hit.item.hasOwnProperty("indicator"); } } }; @@ -177,6 +170,8 @@ export default { if (this.point != null) { this.edit.canMove = this.point.contains(event.point); + } else { + this.edit.canMove = false; } }, createPoint(point) { @@ -190,8 +185,7 @@ export default { this.point.indicator = true; }, onMouseDrag(event) { - if (this.segment) { - if (!this.edit.canMove) return; + if (this.segment && this.edit.canMove) { this.createPoint(event.point); this.segment.point = event.point; } diff --git a/client/src/libs/keypoints.js b/client/src/libs/keypoints.js index e6cb51cf..a4f1701b 100644 --- a/client/src/libs/keypoints.js +++ b/client/src/libs/keypoints.js @@ -1,11 +1,10 @@ import paper from "paper"; export class Keypoints extends paper.Group { - constructor(labels, edges, keypoints, args) { + constructor(edges, keypoints, width, height, args) { super(); args = args || {}; - this.labels = new Set(labels); this._edges = {}; this._lines = {}; @@ -15,16 +14,46 @@ export class Keypoints extends paper.Group { this.strokeColor = args.strokeColor || "red"; this.lineWidth = args.strokeWidth || 4; + edges = edges || []; edges.forEach(e => this.addEdge(e)); if (keypoints) { + console.log(keypoints); for (let i = 0; i < keypoints.length; i += 3) { - let x = keypoints[i], - y = keypoints[i + 1], + let x = keypoints[i] - width / 2, + y = keypoints[i + 1] - height / 2, v = keypoints[i + 2]; - let point = new Keypoint(x, y, { visibility: v, indexLabel: i }); - this.addKeypoint(point); + console.log(x, y, v); + if (v !== 3) { + let point = new Keypoint(x, y, { visibility: v, indexLabel: i }); + this.addKeypoint(point); + } + } + } + } + + isEmpty() { + return this._keypoints.length === 0; + } + + setKeypointIndex(keypoint, newIndex) { + let oldIndex = keypoint.indexLabel; + if (newIndex == oldIndex) return; + + keypoint.indexLabel = parseInt(newIndex); + + if (oldIndex >= 0) { + delete this._labelled[oldIndex]; + + let otherIndices = this._edges[oldIndex]; + if (otherIndices) { + otherIndices.forEach(i => this.removeLine([i, oldIndex])); } + // TODO: Remove assoicated lines + } + if (newIndex >= 0) { + this._labelled[newIndex] = keypoint; + this._drawLines(keypoint); } } @@ -48,8 +77,8 @@ export class Keypoints extends paper.Group { moveKeypoint(point, keypoint) { let indexLabel = keypoint.indexLabel; - let edges = this._edges[indexLabel]; + if (edges) { edges.forEach(i => { let line = this.getLine([i, indexLabel]); @@ -107,11 +136,30 @@ export class Keypoints extends paper.Group { return this._radius; } - exportJSON() { + exportJSON(labels, width, height) { let array = []; + for (let i = 0; i < labels.length; i++) { + let j = i * 3; + array[j] = 0; + array[j + 1] = 0; + array[j + 2] = 3; + } + this._keypoints.forEach(k => { - array.push(...[k.x, k.y, k.visibility]); + let center = new paper.Point(width / 2, height / 2); + let point = k.clone().add(center); + let index = k.indexLabel; + + if (index == -1) { + array.push(...[Math.round(point.x), Math.round(point.y), k.visibility]); + } else { + index -= 1; + array[index] = Math.round(point.x); + array[index + 1] = Math.round(point.y); + array[index + 2] = Math.round(k.visibility); + } }); + return array; } @@ -179,6 +227,8 @@ export class Keypoints extends paper.Group { let line = new paper.Path.Line(firstKeypoint, secondKeypoint); line.strokeColor = this.strokeColor; line.strokeWidth = this.strokeWidth; + line.indicator = true; + if (firstKeypoint.path.isBelow(secondKeypoint.path)) { line.insertBelow(firstKeypoint.path); } else { @@ -188,6 +238,15 @@ export class Keypoints extends paper.Group { this._lines[h] = line; } + removeLine(edge) { + let h = this._hashEdge(edge); + let line = this._lines[h]; + if (line) { + line.remove(); + delete this._lines[h]; + } + } + /** * Returns paperjs path of line [O(1) lookup time] * @param {list} edge array of two elementing contains the index edges @@ -218,7 +277,8 @@ export class Keypoints extends paper.Group { export let VisibilityType = { NOT_LABELED: 0, LABELED_NOT_VISIBLE: 1, - LABELED_VISIBLE: 2 + LABELED_VISIBLE: 2, + UNKNOWN: 3 }; export class Keypoint extends paper.Point { @@ -233,6 +293,9 @@ export class Keypoint extends paper.Point { this.indexLabel = args.indexLabel || -1; this.visibility = args.visibility || VisibilityType.NOT_LABELED; this.visible = args.visible || true; + + this.onClick = args.onClick; + this._draw(); this.color = args.color || "red"; this.setFillColor(); @@ -267,21 +330,34 @@ export class Keypoint extends paper.Point { } this.path = new paper.Path.Circle(this, this.radius); - this.path.onMouseDrag = event => { - if (this.keypoints) { - this.keypoints.moveKeypoint(event.point, this); - } else { - this.move(event.point); - } - }; + + this.path.onMouseDown = this.onMouseDown; + this.path.onMouseUp = this.onMouseUp; + this.path.onMouseDrag = this.onMouseDrag; + this.path.onDoubleClick = this.onDoubleClick; + this.path.onClick = this.onClick; + + this.path.indicator = true; this.path.strokeColor = this.color; this.path.strokeWidth = storkeWidth; this.path.visible = this.visible; this.path.keypoint = this; + this.path.keypoints = this.keypoints; this.setFillColor(); } + onMouseDrag(event) { + let keypoint = event.target.keypoint; + let keypoints = event.target.keypoints; + + if (keypoints) { + keypoints.moveKeypoint(event.point, keypoint); + } else { + keypoint.move(event.point); + } + } + set visible(val) { this._visible = val; this.path.visible = val; From 2ed33f8a80f7a6dc5b0d7926180a2867ec9dcb3a Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Wed, 20 Feb 2019 22:20:34 -0500 Subject: [PATCH 06/15] Fixed saving keypoint logic --- .../src/components/annotator/Annotation.vue | 46 +++++++++++++------ client/src/components/annotator/Category.vue | 9 ++-- .../components/annotator/tools/BrushTool.vue | 2 +- .../components/annotator/tools/EraserTool.vue | 2 +- .../annotator/tools/KeypointTool.vue | 26 ++--------- .../annotator/tools/MagicWandTool.vue | 2 +- .../annotator/tools/PolygonTool.vue | 2 +- client/src/components/cards/ImageCard.vue | 2 +- client/src/components/tasks/Task.vue | 2 +- client/src/libs/keypoints.js | 18 +------- client/src/views/Annotator.vue | 2 +- client/src/views/Dataset.vue | 7 ++- 12 files changed, 51 insertions(+), 69 deletions(-) diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index 12b1f08f..b9fafe2e 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -226,11 +226,11 @@ export default { }, keypointEdges: { type: Array, - default: [] + required: true }, keypointLabels: { type: Array, - default: [] + required: true } }, data() { @@ -299,7 +299,20 @@ export default { // Create new compoundpath this.compoundPath = new paper.CompoundPath(); - this.keypoints = new Keypoints(this.edges, this.annotation.keypoints, width, height); + this.keypoints = new Keypoints(this.edges); + + let keypoints = this.annotation.keypoints; + if (keypoints) { + for (let i = 0; i < keypoints.length; i += 3) { + let x = keypoints[i] - width / 2, + y = keypoints[i + 1] - height / 2, + v = keypoints[i + 2]; + + if (v !== 3) { + this.addKeypoint(new paper.Point(x, y), v, i / 3 + 1); + } + } + } if (json != null) { // Import data directroy from paperjs object @@ -420,16 +433,19 @@ export default { this.compoundPath = this.pervious.pop(); this.compoundPath.fullySelected = this.isCurrent; }, - addKeypoint(point) { + addKeypoint(point, visibility, index) { let keypoint = new Keypoint(point.x, point.y, { - onClick: (event) => { - this.currentKeypoint = event.target.keypoint + visibility: visibility || 0, + indexLabel: index || -1, + onClick: event => { + if (!this.$parent.isCurrent) return; + this.currentKeypoint = event.target.keypoint; let id = `#keypointSettings${this.annotation.id}`; let indexLabel = this.currentKeypoint.indexLabel; this.keypoint.tag = indexLabel == -1 ? [] : [indexLabel.toString()]; this.keypoint.visibility = this.currentKeypoint.visibility; - + $(id).modal("show"); } }); @@ -484,7 +500,7 @@ export default { let l = Math.round(this.compoundPath.fillColor.lightness * 50); let s = Math.round(this.compoundPath.fillColor.saturation * 100); - let hsl = "hsl(" + h + "," + s + "%," + l + "%)"; + let hsl = "hsl(" + h + "," + s + "%," + l + "%)"; this.compoundPath.strokeColor = hsl; this.keypoints.color = hsl; }, @@ -544,7 +560,7 @@ export default { if (this.compoundPath == null) return; this.compoundPath.visible = newVisible; - this.keypoints.visible = newVisible; + this.keypoints.visible = newVisible; }, compoundPath() { if (this.compoundPath == null) return; @@ -566,7 +582,7 @@ export default { }, "keypoint.tag"(newVal) { let id = newVal.length === 0 ? -1 : newVal[0]; - this.keypoints.setKeypointIndex(this.currentKeypoint, id) + this.keypoints.setKeypointIndex(this.currentKeypoint, id); this.tagRecomputeCounter++; }, "keypoint.visibility"(newVal) { @@ -604,19 +620,19 @@ export default { let l = Math.round(this.compoundPath.fillColor.lightness * 50); let s = Math.round(this.compoundPath.fillColor.saturation * 100); - return [h, s, l] + return [h, s, l]; }, keypointLabelTags() { this.tagRecomputeCounter; let tags = {}; - for(let i = 0; i < this.keypointLabels.length; i++) { + for (let i = 0; i < this.keypointLabels.length; i++) { // Include it tags if it is the current keypoint or not in use. if ( - (this.keypoints && !this.keypoints._labelled[i+1]) || - (this.currentKeypoint &&this.currentKeypoint.indexLabel == i+1) + (this.keypoints && !this.keypoints._labelled[i + 1]) || + (this.currentKeypoint && this.currentKeypoint.indexLabel == i + 1) ) { - tags[i+1] = this.keypointLabels[i]; + tags[i + 1] = this.keypointLabels[i]; } } return tags; diff --git a/client/src/components/annotator/Category.vue b/client/src/components/annotator/Category.vue index 44c5666b..bc81475a 100755 --- a/client/src/components/annotator/Category.vue +++ b/client/src/components/annotator/Category.vue @@ -69,8 +69,8 @@ @click="onAnnotationClick(listIndex)" :opacity="opacity" :index="listIndex" - :keypointEdges="keypoint.edges" - :keypointLabels="keypoint.labels" + :keypoint-edges="keypoint.edges" + :keypoint-labels="keypoint.labels" ref="annotation" :hover="hover.annotation" @deleted="annotationDeleted" @@ -356,7 +356,7 @@ export default { let hsl = "hsl(" + h + "," + s + "%," + l + "%)"; this.group.strokeColor = hsl; - if (annotations) annotations.forEach(a => a.keypoints.color = hsl); + if (annotations) annotations.forEach(a => (a.keypoints.color = hsl)); } } }, @@ -411,7 +411,8 @@ export default { if (this.group == null) return; this.group.visible = newVisible; let annotations = this.$refs.annotation; - if (annotations) annotations.forEach(a => a.keypoints.visible = newVisible); + if (annotations) + annotations.forEach(a => (a.keypoints.visible = newVisible)); this.setColor(); }, showAnnotations(showing) { diff --git a/client/src/components/annotator/tools/BrushTool.vue b/client/src/components/annotator/tools/BrushTool.vue index 0a53d320..28a054fe 100755 --- a/client/src/components/annotator/tools/BrushTool.vue +++ b/client/src/components/annotator/tools/BrushTool.vue @@ -118,7 +118,7 @@ export default { if (active) { this.tool.activate(); - localStorage.setItem("editorTool", this.name) + localStorage.setItem("editorTool", this.name); } }, /** diff --git a/client/src/components/annotator/tools/EraserTool.vue b/client/src/components/annotator/tools/EraserTool.vue index c4d1168a..8be6591b 100755 --- a/client/src/components/annotator/tools/EraserTool.vue +++ b/client/src/components/annotator/tools/EraserTool.vue @@ -116,7 +116,7 @@ export default { if (active) { this.tool.activate(); - localStorage.setItem("editorTool", this.name) + localStorage.setItem("editorTool", this.name); } }, /** diff --git a/client/src/components/annotator/tools/KeypointTool.vue b/client/src/components/annotator/tools/KeypointTool.vue index 92f824c2..f85a896f 100755 --- a/client/src/components/annotator/tools/KeypointTool.vue +++ b/client/src/components/annotator/tools/KeypointTool.vue @@ -1,10 +1,5 @@ diff --git a/client/src/components/annotator/tools/MagicWandTool.vue b/client/src/components/annotator/tools/MagicWandTool.vue index 94ab31e4..eb30b3d2 100755 --- a/client/src/components/annotator/tools/MagicWandTool.vue +++ b/client/src/components/annotator/tools/MagicWandTool.vue @@ -36,7 +36,7 @@ export default { isActive(active) { if (active) { this.tool.activate(); - localStorage.setItem("editorTool", this.name) + localStorage.setItem("editorTool", this.name); } } }, diff --git a/client/src/components/annotator/tools/PolygonTool.vue b/client/src/components/annotator/tools/PolygonTool.vue index 5f7f701e..e9afa985 100755 --- a/client/src/components/annotator/tools/PolygonTool.vue +++ b/client/src/components/annotator/tools/PolygonTool.vue @@ -224,7 +224,7 @@ export default { isActive(active) { if (active) { this.tool.activate(); - localStorage.setItem("editorTool", this.name) + localStorage.setItem("editorTool", this.name); } }, /** diff --git a/client/src/components/cards/ImageCard.vue b/client/src/components/cards/ImageCard.vue index 3777b504..905017e6 100755 --- a/client/src/components/cards/ImageCard.vue +++ b/client/src/components/cards/ImageCard.vue @@ -96,7 +96,7 @@ export default { return { hover: false, showAnnotations: true, - loaderUrl: require("@/assets/loader.gif"), + loaderUrl: require("@/assets/loader.gif") }; }, methods: { diff --git a/client/src/components/tasks/Task.vue b/client/src/components/tasks/Task.vue index ff8a0244..73ea12ed 100644 --- a/client/src/components/tasks/Task.vue +++ b/client/src/components/tasks/Task.vue @@ -100,7 +100,7 @@ export default { showLogs: "getLogs", completed() { if (this.showLogs) { - this.getLogs() + this.getLogs(); } } }, diff --git a/client/src/libs/keypoints.js b/client/src/libs/keypoints.js index a4f1701b..8a1dc207 100644 --- a/client/src/libs/keypoints.js +++ b/client/src/libs/keypoints.js @@ -1,7 +1,7 @@ import paper from "paper"; export class Keypoints extends paper.Group { - constructor(edges, keypoints, width, height, args) { + constructor(edges, args) { super(); args = args || {}; @@ -16,20 +16,6 @@ export class Keypoints extends paper.Group { edges = edges || []; edges.forEach(e => this.addEdge(e)); - - if (keypoints) { - console.log(keypoints); - for (let i = 0; i < keypoints.length; i += 3) { - let x = keypoints[i] - width / 2, - y = keypoints[i + 1] - height / 2, - v = keypoints[i + 2]; - console.log(x, y, v); - if (v !== 3) { - let point = new Keypoint(x, y, { visibility: v, indexLabel: i }); - this.addKeypoint(point); - } - } - } } isEmpty() { @@ -153,7 +139,7 @@ export class Keypoints extends paper.Group { if (index == -1) { array.push(...[Math.round(point.x), Math.round(point.y), k.visibility]); } else { - index -= 1; + index = (index - 1) * 3; array[index] = Math.round(point.x); array[index + 1] = Math.round(point.y); array[index + 2] = Math.round(k.visibility); diff --git a/client/src/views/Annotator.vue b/client/src/views/Annotator.vue index da3ca457..702c39ac 100755 --- a/client/src/views/Annotator.vue +++ b/client/src/views/Annotator.vue @@ -546,7 +546,7 @@ export default { }, selectLastEditorTool() { - this.activeTool = localStorage.getItem("editorTool") || "Select" + this.activeTool = localStorage.getItem("editorTool") || "Select"; }, setCursor(newCursor) { diff --git a/client/src/views/Dataset.vue b/client/src/views/Dataset.vue index 8e6711c0..0e49df9e 100755 --- a/client/src/views/Dataset.vue +++ b/client/src/views/Dataset.vue @@ -240,7 +240,7 @@ import PanelToggle from "@/components/PanelToggle"; import JQuery from "jquery"; import { mapMutations } from "vuex"; -let $ = JQuery +let $ = JQuery; export default { name: "Dataset", @@ -374,8 +374,8 @@ export default { this.$router.push({ path: "/tasks", query: { id: this.importing.id } }); return; } - - $('#cocoUpload').modal('show'); + + $("#cocoUpload").modal("show"); }, importCOCO() { let uploaded = document.getElementById("coco"); @@ -426,7 +426,6 @@ export default { }, sockets: { taskProgress(data) { - if (data.id === this.scan.id) { this.scan.progress = data.progress; } From f189606de21fbb14f0af905aefd4b7b59a97ce34 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Thu, 21 Feb 2019 20:07:48 -0500 Subject: [PATCH 07/15] Create Keypoints panel --- client/src/components/PanelInputDropdown.vue | 88 +++++++++++++++++++ client/src/components/PanelText.vue | 28 ++++++ .../src/components/annotator/Annotation.vue | 71 +++++++++++++-- .../annotator/panels/KeypointPanel.vue | 60 +++++++++++++ client/src/libs/keypoints.js | 1 + client/src/views/Annotator.vue | 13 ++- 6 files changed, 250 insertions(+), 11 deletions(-) create mode 100755 client/src/components/PanelInputDropdown.vue create mode 100755 client/src/components/PanelText.vue create mode 100755 client/src/components/annotator/panels/KeypointPanel.vue diff --git a/client/src/components/PanelInputDropdown.vue b/client/src/components/PanelInputDropdown.vue new file mode 100755 index 00000000..0cee2df5 --- /dev/null +++ b/client/src/components/PanelInputDropdown.vue @@ -0,0 +1,88 @@ + + + + + diff --git a/client/src/components/PanelText.vue b/client/src/components/PanelText.vue new file mode 100755 index 00000000..8b8a25d3 --- /dev/null +++ b/client/src/components/PanelText.vue @@ -0,0 +1,28 @@ + + + + + diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index b9fafe2e..b8193a84 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -249,7 +249,8 @@ export default { currentKeypoint: null, keypoint: { tag: [], - visibility: 0 + visibility: 0, + label: -1 }, tagRecomputeCounter: 0 }; @@ -433,12 +434,37 @@ export default { this.compoundPath = this.pervious.pop(); this.compoundPath.fullySelected = this.isCurrent; }, - addKeypoint(point, visibility, index) { + addKeypoint(point, visibility, label) { + + visibility = visibility || this.keypoint.visibility; + label = label || parseInt(this.keypoint.label); + let keypoint = new Keypoint(point.x, point.y, { visibility: visibility || 0, - indexLabel: index || -1, + indexLabel: label || -1, onClick: event => { - if (!this.$parent.isCurrent) return; + // if (this.$parent.isCurrent) return; + let keypoint = event.target.keypoint; + if (this.currentKeypoint && this.currentKeypoint != keypoint) { + let i1 = this.currentKeypoint.indexLabel; + let i2 = keypoint.indexLabel; + if (this.keypoints && i1 > 0 && i2 > 0) { + + if (!this.keypoints.getLine([i1, i2])) { + this.keypoints.addEdge([i1, i2]); + } else { + this.keypoints.removeLine([i1, i2]); + } + + this.currentKeypoint = null; + return; + } + } + + this.currentKeypoint = event.target.keypoint; + }, + onDoubleClick: event => { + // if (!this.$parent.isCurrent) return; this.currentKeypoint = event.target.keypoint; let id = `#keypointSettings${this.annotation.id}`; let indexLabel = this.currentKeypoint.indexLabel; @@ -452,6 +478,8 @@ export default { this.keypoints.addKeypoint(keypoint); this.isEmpty = this.compoundPath.isEmpty() && this.keypoints.isEmpty(); + + this.tagRecomputeCounter++; }, /** * Unites current annotation path with anyother path. @@ -622,19 +650,41 @@ export default { return [h, s, l]; }, - keypointLabelTags() { + notUsedKeypointLabels() { this.tagRecomputeCounter; let tags = {}; for (let i = 0; i < this.keypointLabels.length; i++) { // Include it tags if it is the current keypoint or not in use. - if ( - (this.keypoints && !this.keypoints._labelled[i + 1]) || - (this.currentKeypoint && this.currentKeypoint.indexLabel == i + 1) - ) { + if (this.keypoints && !this.keypoints._labelled[i + 1]) { tags[i + 1] = this.keypointLabels[i]; } } + + return tags; + }, + usedKeypointLabels() { + this.tagRecomputeCounter; + let tags = {}; + + for (let i = 0; i < this.keypointLabels.length; i++) { + if (!this.keypoints || this.keypoints._labelled[i + 1]) { + tags[i + 1] = this.keypointLabels[i]; + } + } + + return tags; + }, + keypointLabelTags() { + this.tagRecomputeCounter; + let tags = this.notUsedKeypointLabels; + + Object.keys(this.usedKeypointLabels).forEach(i => { + if (this.currentKeypoint && i == this.currentKeypoint.indexLabel) { + tags[i] = this.usedKeypointLabels[i]; + } + }); + return tags; } }, @@ -659,6 +709,9 @@ export default { }, mounted() { this.initAnnotation(); + $(`#keypointSettings${this.annotation.id}`).on('hidden.bs.modal', () => { + this.currentKeypoint = null; + }) } }; diff --git a/client/src/components/annotator/panels/KeypointPanel.vue b/client/src/components/annotator/panels/KeypointPanel.vue new file mode 100755 index 00000000..3c6af9b5 --- /dev/null +++ b/client/src/components/annotator/panels/KeypointPanel.vue @@ -0,0 +1,60 @@ + + + diff --git a/client/src/libs/keypoints.js b/client/src/libs/keypoints.js index 8a1dc207..119c4c1d 100644 --- a/client/src/libs/keypoints.js +++ b/client/src/libs/keypoints.js @@ -281,6 +281,7 @@ export class Keypoint extends paper.Point { this.visible = args.visible || true; this.onClick = args.onClick; + this.onDoubleClick = args.onDoubleClick; this._draw(); this.color = args.color || "red"; diff --git a/client/src/views/Annotator.vue b/client/src/views/Annotator.vue index 702c39ac..a5070f20 100755 --- a/client/src/views/Annotator.vue +++ b/client/src/views/Annotator.vue @@ -43,7 +43,7 @@
@@ -160,6 +160,13 @@
+ +
+ +
@@ -210,6 +217,7 @@ import SelectPanel from "@/components/annotator/panels/SelectPanel"; import MagicWandPanel from "@/components/annotator/panels/MagicWandPanel"; import BrushPanel from "@/components/annotator/panels/BrushPanel"; import EraserPanel from "@/components/annotator/panels/EraserPanel"; +import KeypointPanel from "@/components/annotator/panels/KeypointPanel" import { mapMutations } from "vuex"; @@ -239,7 +247,8 @@ export default { ModeButton, UndoButton, HideAllButton, - ShowAllButton + ShowAllButton, + KeypointPanel }, mixins: [toastrs, shortcuts], props: { From edfd7b0c562477056561d0f00eede008e404d6e4 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Thu, 21 Feb 2019 22:33:04 -0500 Subject: [PATCH 08/15] Setting keypoint edges --- .../src/components/annotator/Annotation.vue | 38 ++++++++++++------- client/src/components/annotator/Category.vue | 21 ++++++++++ .../annotator/panels/KeypointPanel.vue | 4 +- client/src/libs/keypoints.js | 29 ++++++++++++++ 4 files changed, 77 insertions(+), 15 deletions(-) diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index b8193a84..6987a731 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -244,13 +244,15 @@ export default { name: "", uuid: "", pervious: [], - edges: [[1, 3], [3, 5], [2, 4], [4, 5], [5, 6], [5, 7], [7, 8], [6, 8]], count: 0, currentKeypoint: null, keypoint: { tag: [], visibility: 0, - label: -1 + next: { + label: -1, + visibility: 0 + } }, tagRecomputeCounter: 0 }; @@ -300,7 +302,7 @@ export default { // Create new compoundpath this.compoundPath = new paper.CompoundPath(); - this.keypoints = new Keypoints(this.edges); + this.keypoints = new Keypoints(this.keypointEdges); let keypoints = this.annotation.keypoints; if (keypoints) { @@ -435,27 +437,26 @@ export default { this.compoundPath.fullySelected = this.isCurrent; }, addKeypoint(point, visibility, label) { - - visibility = visibility || this.keypoint.visibility; - label = label || parseInt(this.keypoint.label); + visibility = visibility || parseInt(this.keypoint.next.visibility); + label = label || parseInt(this.keypoint.next.label); let keypoint = new Keypoint(point.x, point.y, { visibility: visibility || 0, indexLabel: label || -1, onClick: event => { - // if (this.$parent.isCurrent) return; + if (this.$parent.isCurrent) return; let keypoint = event.target.keypoint; if (this.currentKeypoint && this.currentKeypoint != keypoint) { let i1 = this.currentKeypoint.indexLabel; let i2 = keypoint.indexLabel; if (this.keypoints && i1 > 0 && i2 > 0) { - - if (!this.keypoints.getLine([i1, i2])) { - this.keypoints.addEdge([i1, i2]); + let edge = [i1, i2]; + if (!this.keypoints.getLine(edge)) { + this.$parent.addKeypointEdge(edge); } else { - this.keypoints.removeLine([i1, i2]); + this.$parent.removeKeypointEdge(edge); } - + this.currentKeypoint = null; return; } @@ -464,7 +465,9 @@ export default { this.currentKeypoint = event.target.keypoint; }, onDoubleClick: event => { - // if (!this.$parent.isCurrent) return; + + if (!this.$parent.isCurrent) return; + this.currentKeypoint = event.target.keypoint; let id = `#keypointSettings${this.annotation.id}`; let indexLabel = this.currentKeypoint.indexLabel; @@ -608,13 +611,22 @@ export default { if (this.compoundPath == null) return; this.compoundPath.fullySelected = this.isCurrent; }, + currentKeypoint(point, old) { + if (old) old.selected = false; + if (point) point.selected = true; + }, "keypoint.tag"(newVal) { let id = newVal.length === 0 ? -1 : newVal[0]; this.keypoints.setKeypointIndex(this.currentKeypoint, id); this.tagRecomputeCounter++; }, "keypoint.visibility"(newVal) { + if (!this.currentKeypoint) return; this.currentKeypoint.visibility = newVal; + }, + "keypointEdges"(newEdges, oldEdges) { + newEdges.forEach(e => this.keypoints.addEdge(e)); + } }, computed: { diff --git a/client/src/components/annotator/Category.vue b/client/src/components/annotator/Category.vue index bc81475a..96164958 100755 --- a/client/src/components/annotator/Category.vue +++ b/client/src/components/annotator/Category.vue @@ -267,6 +267,27 @@ export default { return categoryData; }, + + addKeypointEdge(edge) { + this.keypoint.edges.push(edge); + }, + removeKeypointEdge(edge) { + let index = this.keypoint.edges.findIndex(e => { + let i1 = Math.min(edge[0], edge[1]) == Math.min(e[0], e[1]); + let i2 = Math.max(edge[0], edge[1]) == Math.max(e[0], e[1]); + + return i1 && i2; + }) + + if (index !== -1) { + let edge = this.keypoint.edges[index]; + this.keypoint.edges.splice(index, 1); + let annotations = this.$refs.annotation; + if (annotations) { + annotations.forEach(a => a.keypoints.removeLine(edge)) + } + } + }, /** * Event handler for visibility button */ diff --git a/client/src/components/annotator/panels/KeypointPanel.vue b/client/src/components/annotator/panels/KeypointPanel.vue index 3c6af9b5..621997e3 100755 --- a/client/src/components/annotator/panels/KeypointPanel.vue +++ b/client/src/components/annotator/panels/KeypointPanel.vue @@ -44,10 +44,10 @@ export default { } }, label(label) { - this.$parent.currentAnnotation.keypoint.label = label; + this.$parent.currentAnnotation.keypoint.next.label = label; }, visibility(visibility) { - this.$parent.currentAnnotation.keypoint.visibility = visibility; + this.$parent.currentAnnotation.keypoint.next.visibility = visibility; } }, computed: { diff --git a/client/src/libs/keypoints.js b/client/src/libs/keypoints.js index 119c4c1d..864e9139 100644 --- a/client/src/libs/keypoints.js +++ b/client/src/libs/keypoints.js @@ -149,6 +149,25 @@ export class Keypoints extends paper.Group { return array; } + edges() { + let edges = []; + let keys = Object.keys(this._edges); + + for (let i = 0; i < keys.length; i++) { + let i1 = parseInt(keys[i]); + let otherIndices = Array.from(this._edges[i1]); + + for (let j = 0; j < otherIndices.length; j++) { + let i2 = parseInt(otherIndices[j]); + + if (i2 < i1) continue; + edges.push([i1, i2]); + } + } + + return edges; + } + addEdge(edge) { if (edge.length !== 2) return; @@ -389,4 +408,14 @@ export class Keypoint extends paper.Point { get strokeColor() { return this.color; } + + set selected(val) { + this._selected = val; + if (val) this.radius *= 1.3; + else this.radius *= 0.7; + } + + get selected() { + return this._selected; + } } From aaee122ddb9ffd65ce79c2b5aa67a220aa9e0f58 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Thu, 21 Feb 2019 22:34:13 -0500 Subject: [PATCH 09/15] Fixed model --- app/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models.py b/app/models.py index 2fa280b4..31566c72 100644 --- a/app/models.py +++ b/app/models.py @@ -222,7 +222,7 @@ class AnnotationModel(db.DynamicDocument): color = db.StringField() - UNKNOWN = db.ListField(default=[]) + keypoints = db.ListField(default=[]) metadata = db.DictField(default={}) paper_object = db.ListField(default=[]) From 05d6cd999efab722300f51d11734ce8c7365148f Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Fri, 22 Feb 2019 22:05:20 -0500 Subject: [PATCH 10/15] Export keypoints and bug fixes --- app/util/coco_util.py | 49 ++++++++++++++++--- .../src/components/annotator/Annotation.vue | 44 +++++++++++++---- client/src/components/annotator/Category.vue | 14 +++++- .../annotator/tools/KeypointTool.vue | 2 +- client/src/libs/keypoints.js | 26 +++++----- client/src/views/Annotator.vue | 1 + 6 files changed, 103 insertions(+), 33 deletions(-) diff --git a/app/util/coco_util.py b/app/util/coco_util.py index 1ceec93c..6abf465e 100644 --- a/app/util/coco_util.py +++ b/app/util/coco_util.py @@ -114,24 +114,40 @@ def get_image_coco(image): annotations = [] for category in bulk_categories: - + category = category[1] category_annotations = AnnotationModel.objects( - deleted=False, category_id=category[1].id, image_id=image.get('id') + deleted=False, category_id=category.id, image_id=image.get('id') ).exclude('paper_object', 'deleted_date').all() - + if len(category_annotations) == 0: continue + + has_keypoints = len(category.keypoint_labels) > 0 for annotation in category_annotations: annotation = fix_ids(annotation) - if len(annotation.get('segmentation')) != 0: + if len(annotation.get('segmentation', [])) != 0 or \ + len(annotation.get('keypoints', [])) != 0: del annotation['deleted'] - del annotation['paper_object'] + + if not has_keypoints: + del annotation['keypoints'] + else: + arr = np.array(annotation.get('keypoints', [])) + arr = arr[2::3] + annotation['num_keypoints'] = len(arr[arr > 0]) + annotations.append(annotation) - category = fix_ids(category[1]) + category = fix_ids(category) del category['deleted'] + if has_keypoints: + category['keypoints'] = category.pop('keypoint_labels') + category['skeleton'] = category.pop('keypoint_edges') + else: + del category['keypoint_edges'] + del category['keypoint_labels'] categories.append(category) del image['deleted'] @@ -169,7 +185,15 @@ def get_dataset_coco(dataset): for category in categories: category = fix_ids(category[1]) + del category['deleted'] + if len(category.keypoint_labels) > 0: + category['keypoints'] = category.pop('keypoint_labels') + category['skeleton'] = category.pop('keypoint_edges') + else: + del category['keypoint_edges'] + del category['keypoint_labels'] + coco.get('categories').append(category) for image in images: @@ -180,8 +204,19 @@ def get_dataset_coco(dataset): annotations = fix_ids(annotations.all()) for annotation in annotations: - if len(annotation.get('segmentation', [])) != 0: + + has_keypoints = len(annotation.get('keypoints', [])) > 0 + has_segmentation = len(annotation.get('segmentation', [])) > 0 + + if has_keypoints or has_keypoints: del annotation['deleted'] + + if not has_keypoints: + del annotation['keypoints'] + else: + arr = np.array(annotation.get('keypoints', [])) + arr = arr[2::3] + annotation['num_keypoints'] = len(arr[arr > 0]) coco.get('annotations').append(annotation) image = fix_ids(image) diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index 6987a731..66ef0e83 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -1,10 +1,10 @@ @@ -173,6 +176,10 @@ export default { simplify: { type: Number, default: 1 + }, + activeTool: { + type: String, + required: true } }, data: function() { @@ -377,7 +384,12 @@ export default { let hsl = "hsl(" + h + "," + s + "%," + l + "%)"; this.group.strokeColor = hsl; - if (annotations) annotations.forEach(a => (a.keypoints.color = hsl)); + if (annotations) { + annotations.forEach(a => { + a.keypoints.color = hsl; + a.keypoints.bringToFront(); + }); + } } } }, diff --git a/client/src/components/annotator/tools/KeypointTool.vue b/client/src/components/annotator/tools/KeypointTool.vue index f85a896f..8b2f2cf7 100755 --- a/client/src/components/annotator/tools/KeypointTool.vue +++ b/client/src/components/annotator/tools/KeypointTool.vue @@ -27,7 +27,7 @@ export default { }, onMouseDown(event) { this.$parent.currentAnnotation.addKeypoint(event.point); - } + }, }, computed: { isDisabled() { diff --git a/client/src/libs/keypoints.js b/client/src/libs/keypoints.js index 864e9139..c321aed7 100644 --- a/client/src/libs/keypoints.js +++ b/client/src/libs/keypoints.js @@ -59,6 +59,7 @@ export class Keypoints extends paper.Group { this._keypoints.push(keypoint); this.addChild(keypoint.path); this._drawLines(keypoint); + keypoint.path.bringToFront(); } moveKeypoint(point, keypoint) { @@ -81,6 +82,7 @@ export class Keypoints extends paper.Group { }); } keypoint.move(point); + keypoint.path.bringToFront(); } set visible(val) { @@ -128,7 +130,7 @@ export class Keypoints extends paper.Group { let j = i * 3; array[j] = 0; array[j + 1] = 0; - array[j + 2] = 3; + array[j + 2] = 0; } this._keypoints.forEach(k => { @@ -149,6 +151,10 @@ export class Keypoints extends paper.Group { return array; } + contains(point) { + return this._keypoints.findIndex(k => k.path.contains(point)) > -1; + } + edges() { let edges = []; let keys = Object.keys(this._edges); @@ -301,6 +307,7 @@ export class Keypoint extends paper.Point { this.onClick = args.onClick; this.onDoubleClick = args.onDoubleClick; + this.onMouseDrag = args.onMouseDrag; this._draw(); this.color = args.color || "red"; @@ -353,17 +360,6 @@ export class Keypoint extends paper.Point { this.setFillColor(); } - onMouseDrag(event) { - let keypoint = event.target.keypoint; - let keypoints = event.target.keypoints; - - if (keypoints) { - keypoints.moveKeypoint(event.point, keypoint); - } else { - keypoint.move(event.point); - } - } - set visible(val) { this._visible = val; this.path.visible = val; @@ -393,7 +389,7 @@ export class Keypoint extends paper.Point { set color(val) { this._color = val; - this.path.strokeColor = val; + this.path.strokeColor = this.selected ? "white" : val; this.setFillColor(); } @@ -411,8 +407,8 @@ export class Keypoint extends paper.Point { set selected(val) { this._selected = val; - if (val) this.radius *= 1.3; - else this.radius *= 0.7; + this.path.strokeColor = val ? "white" : this.color; + this.path.bringToFront(); } get selected() { diff --git a/client/src/views/Annotator.vue b/client/src/views/Annotator.vue index a5070f20..a3b3c12b 100755 --- a/client/src/views/Annotator.vue +++ b/client/src/views/Annotator.vue @@ -121,6 +121,7 @@ :index="index" @click="onCategoryClick" :current="current" + :activeTool="activeTool" ref="category" /> From 2467ffeb747e2952671cfac3a57458f43185ab81 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Fri, 22 Feb 2019 22:06:26 -0500 Subject: [PATCH 11/15] Simplified clamp function --- app/util/coco_util.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/app/util/coco_util.py b/app/util/coco_util.py index 6abf465e..f71ca6e4 100644 --- a/app/util/coco_util.py +++ b/app/util/coco_util.py @@ -227,11 +227,4 @@ def get_dataset_coco(dataset): def _fit(value, max_value, min_value): - - if value > max_value: - return max_value - - if value < min_value: - return min_value - - return value + return max(min(value, max_value), min_value) From 6e383f9a94b41cc3bef9119b185b72851f47d9d1 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Fri, 22 Feb 2019 22:08:50 -0500 Subject: [PATCH 12/15] Fixed js lint --- client/src/components/PanelInputDropdown.vue | 2 +- .../src/components/annotator/Annotation.vue | 24 ++++++++----------- client/src/components/annotator/Category.vue | 6 ++--- .../annotator/panels/KeypointPanel.vue | 6 ++--- .../annotator/tools/KeypointTool.vue | 2 +- client/src/views/Annotator.vue | 6 ++--- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/client/src/components/PanelInputDropdown.vue b/client/src/components/PanelInputDropdown.vue index 0cee2df5..92927db2 100755 --- a/client/src/components/PanelInputDropdown.vue +++ b/client/src/components/PanelInputDropdown.vue @@ -4,7 +4,7 @@ {{ name }} diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index 66ef0e83..3db998f0 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -308,7 +308,7 @@ export default { // Create new compoundpath this.compoundPath = new paper.CompoundPath(); this.keypoints = new Keypoints(this.keypointEdges); - + let keypoints = this.annotation.keypoints; if (keypoints) { for (let i = 0; i < keypoints.length; i += 3) { @@ -443,8 +443,7 @@ export default { this.compoundPath.fullySelected = this.isCurrent; }, addKeypoint(point, visibility, label) { - if (label == null && this.keypoints.contains(point)) - return; + if (label == null && this.keypoints.contains(point)) return; visibility = visibility || parseInt(this.keypoint.next.visibility); label = label || parseInt(this.keypoint.next.label); @@ -461,13 +460,13 @@ export default { this.currentKeypoint = null; return; } - + if (this.currentKeypoint) { let i1 = this.currentKeypoint.indexLabel; let i2 = keypoint.indexLabel; if (this.keypoints && i1 > 0 && i2 > 0) { let edge = [i1, i2]; - + if (!this.keypoints.getLine(edge)) { this.$parent.addKeypointEdge(edge); } else { @@ -478,11 +477,11 @@ export default { return; } } - - this.currentKeypoint = event.target.keypoint; + + this.currentKeypoint = event.target.keypoint; }, onDoubleClick: event => { - if (!["Select", "Keypoints"].includes(this.activeTool)) return; + if (!["Select", "Keypoints"].includes(this.activeTool)) return; this.currentKeypoint = event.target.keypoint; let id = `#keypointSettings${this.annotation.id}`; let indexLabel = this.currentKeypoint.indexLabel; @@ -548,7 +547,6 @@ export default { this.$parent.setColor(); return; } - this.compoundPath.fillColor = this.color; let h = Math.round(this.compoundPath.fillColor.hue); @@ -616,7 +614,6 @@ export default { this.compoundPath.visible = newVisible; this.keypoints.visible = newVisible; - }, compoundPath() { if (this.compoundPath == null) return; @@ -649,9 +646,8 @@ export default { if (!this.currentKeypoint) return; this.currentKeypoint.visibility = newVal; }, - "keypointEdges"(newEdges, oldEdges) { + keypointEdges(newEdges) { newEdges.forEach(e => this.keypoints.addEdge(e)); - } }, computed: { @@ -747,9 +743,9 @@ export default { }, mounted() { this.initAnnotation(); - $(`#keypointSettings${this.annotation.id}`).on('hidden.bs.modal', () => { + $(`#keypointSettings${this.annotation.id}`).on("hidden.bs.modal", () => { this.currentKeypoint = null; - }) + }); } }; diff --git a/client/src/components/annotator/Category.vue b/client/src/components/annotator/Category.vue index d1b4894b..39921ff6 100755 --- a/client/src/components/annotator/Category.vue +++ b/client/src/components/annotator/Category.vue @@ -73,7 +73,7 @@ :keypoint-labels="keypoint.labels" ref="annotation" :hover="hover.annotation" - :activeTool="activeTool" + :active-tool="activeTool" @deleted="annotationDeleted" /> @@ -284,14 +284,14 @@ export default { let i2 = Math.max(edge[0], edge[1]) == Math.max(e[0], e[1]); return i1 && i2; - }) + }); if (index !== -1) { let edge = this.keypoint.edges[index]; this.keypoint.edges.splice(index, 1); let annotations = this.$refs.annotation; if (annotations) { - annotations.forEach(a => a.keypoints.removeLine(edge)) + annotations.forEach(a => a.keypoints.removeLine(edge)); } } }, diff --git a/client/src/components/annotator/panels/KeypointPanel.vue b/client/src/components/annotator/panels/KeypointPanel.vue index 621997e3..516488c1 100755 --- a/client/src/components/annotator/panels/KeypointPanel.vue +++ b/client/src/components/annotator/panels/KeypointPanel.vue @@ -33,12 +33,12 @@ export default { 1: "LABELED NOT VISIBLE", 2: "LABELED VISIBLE" } - } + }; }, watch: { notUsedLabels(notUsedLabels) { if (!notUsedLabels) return; - let values = Object.keys(notUsedLabels) + let values = Object.keys(notUsedLabels); if (values.length !== 0) { this.label = values[0]; } @@ -52,7 +52,7 @@ export default { }, computed: { notUsedLabels() { - if (!this.currentAnnotation) return { }; + if (!this.currentAnnotation) return {}; return this.currentAnnotation.notUsedKeypointLabels; } } diff --git a/client/src/components/annotator/tools/KeypointTool.vue b/client/src/components/annotator/tools/KeypointTool.vue index 8b2f2cf7..f85a896f 100755 --- a/client/src/components/annotator/tools/KeypointTool.vue +++ b/client/src/components/annotator/tools/KeypointTool.vue @@ -27,7 +27,7 @@ export default { }, onMouseDown(event) { this.$parent.currentAnnotation.addKeypoint(event.point); - }, + } }, computed: { isDisabled() { diff --git a/client/src/views/Annotator.vue b/client/src/views/Annotator.vue index a3b3c12b..120f5555 100755 --- a/client/src/views/Annotator.vue +++ b/client/src/views/Annotator.vue @@ -121,7 +121,7 @@ :index="index" @click="onCategoryClick" :current="current" - :activeTool="activeTool" + :active-tool="activeTool" ref="category" /> @@ -165,7 +165,7 @@
@@ -218,7 +218,7 @@ import SelectPanel from "@/components/annotator/panels/SelectPanel"; import MagicWandPanel from "@/components/annotator/panels/MagicWandPanel"; import BrushPanel from "@/components/annotator/panels/BrushPanel"; import EraserPanel from "@/components/annotator/panels/EraserPanel"; -import KeypointPanel from "@/components/annotator/panels/KeypointPanel" +import KeypointPanel from "@/components/annotator/panels/KeypointPanel"; import { mapMutations } from "vuex"; From cf2b090246ff757f7091d67e1f470cd642fc7db6 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Fri, 22 Feb 2019 22:36:48 -0500 Subject: [PATCH 13/15] Keypoint delete using delete key --- client/src/components/annotator/Annotation.vue | 13 +++++++++---- client/src/libs/keypoints.js | 13 +++++++++++++ client/src/mixins/shortcuts.js | 11 ++++++++++- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index 3db998f0..ef9fe187 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -316,9 +316,9 @@ export default { y = keypoints[i + 1] - height / 2, v = keypoints[i + 2]; - if (x !== 0 || y !== 0 || v !== 0) { - this.addKeypoint(new paper.Point(x, y), v, i / 3 + 1); - } + if (keypoints[i] === 0 && keypoints[i + 1] === 0 && v === 0) continue; + + this.addKeypoint(new paper.Point(x, y), v, i / 3 + 1); } } @@ -452,6 +452,7 @@ export default { visibility: visibility || 0, indexLabel: label || -1, onClick: event => { + if (!this.$parent.isCurrent) return; if (!["Select", "Keypoints"].includes(this.activeTool)) return; let keypoint = event.target.keypoint; @@ -481,6 +482,7 @@ export default { this.currentKeypoint = event.target.keypoint; }, onDoubleClick: event => { + if (!this.$parent.isCurrent) return; if (!["Select", "Keypoints"].includes(this.activeTool)) return; this.currentKeypoint = event.target.keypoint; let id = `#keypointSettings${this.annotation.id}`; @@ -504,6 +506,9 @@ export default { this.tagRecomputeCounter++; }, + deleteKeypoint(keypoint) { + this.keypoints.delete(keypoint); + }, /** * Unites current annotation path with anyother path. * @param {paper.CompoundPath} compound compound to unite current annotation path with @@ -687,7 +692,7 @@ export default { notUsedKeypointLabels() { this.tagRecomputeCounter; let tags = {}; - + console.log("Computing unsed " + this.tagRecomputeCounter) for (let i = 0; i < this.keypointLabels.length; i++) { // Include it tags if it is the current keypoint or not in use. if (this.keypoints && !this.keypoints._labelled[i + 1]) { diff --git a/client/src/libs/keypoints.js b/client/src/libs/keypoints.js index c321aed7..2b7a3693 100644 --- a/client/src/libs/keypoints.js +++ b/client/src/libs/keypoints.js @@ -62,6 +62,19 @@ export class Keypoints extends paper.Group { keypoint.path.bringToFront(); } + deleteKeypoint(keypoint) { + let indexLabel = keypoint.indexLabel; + if (this._labelled.hasOwnProperty(indexLabel)) { + delete this._labelled[indexLabel]; + } + if (this._edges.hasOwnProperty(indexLabel)) { + this._edges[indexLabel].forEach(e => this.removeLine([e, indexLabel])); + } + let index = this._keypoints.findIndex(k => k == keypoint); + if (index > -1) this._keypoints.splice(index, 1); + keypoint.path.remove(); + } + moveKeypoint(point, keypoint) { let indexLabel = keypoint.indexLabel; let edges = this._edges[indexLabel]; diff --git a/client/src/mixins/shortcuts.js b/client/src/mixins/shortcuts.js index 6d6b63aa..a0e5d2d7 100755 --- a/client/src/mixins/shortcuts.js +++ b/client/src/mixins/shortcuts.js @@ -44,7 +44,16 @@ export default { name: "Delete Current Annotation", function: () => { if (this.currentAnnotation) { - this.currentAnnotation.deleteAnnotation(); + let currentKeypoint = this.currentAnnotation.currentKeypoint; + if (currentKeypoint) { + this.currentAnnotation.keypoints.deleteKeypoint( + currentKeypoint + ); + this.currentAnnotation.tagRecomputeCounter++; + this.currentAnnotation.currentKeypoint = null; + } else { + this.currentAnnotation.deleteAnnotation(); + } } } }, From 367935b8f9b5a593fa93cb64a4281dc428dee1ef Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Fri, 22 Feb 2019 22:38:00 -0500 Subject: [PATCH 14/15] Keypoints shortcut --- client/src/mixins/shortcuts.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client/src/mixins/shortcuts.js b/client/src/mixins/shortcuts.js index a0e5d2d7..e5bb4091 100755 --- a/client/src/mixins/shortcuts.js +++ b/client/src/mixins/shortcuts.js @@ -84,6 +84,13 @@ export default { this.activeTool = "Magic Wand"; } }, + { + default: ["k"], + name: "Keypoints Tool", + function: () => { + if (!this.$refs.magicwand.isDisabled) this.activeTool = "Keypoints"; + } + }, { default: ["b"], name: "Brush Tool", From a449d95bf9fea584756cabe6c6e38118f8e0cd58 Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Fri, 22 Feb 2019 22:41:41 -0500 Subject: [PATCH 15/15] Fixed keypoint line error --- client/src/components/annotator/Annotation.vue | 4 ++-- client/src/libs/keypoints.js | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/client/src/components/annotator/Annotation.vue b/client/src/components/annotator/Annotation.vue index ef9fe187..009efb0b 100755 --- a/client/src/components/annotator/Annotation.vue +++ b/client/src/components/annotator/Annotation.vue @@ -317,7 +317,7 @@ export default { v = keypoints[i + 2]; if (keypoints[i] === 0 && keypoints[i + 1] === 0 && v === 0) continue; - + this.addKeypoint(new paper.Point(x, y), v, i / 3 + 1); } } @@ -692,7 +692,7 @@ export default { notUsedKeypointLabels() { this.tagRecomputeCounter; let tags = {}; - console.log("Computing unsed " + this.tagRecomputeCounter) + for (let i = 0; i < this.keypointLabels.length; i++) { // Include it tags if it is the current keypoint or not in use. if (this.keypoints && !this.keypoints._labelled[i + 1]) { diff --git a/client/src/libs/keypoints.js b/client/src/libs/keypoints.js index 2b7a3693..c347c93f 100644 --- a/client/src/libs/keypoints.js +++ b/client/src/libs/keypoints.js @@ -43,6 +43,11 @@ export class Keypoints extends paper.Group { } } + bringToFront() { + super.bringToFront(); + this._keypoints.forEach(k => k.path.bringToFront()); + } + addKeypoint(keypoint) { keypoint.keypoints = this; keypoint.path.keypoints = this;