feat(labeler): allow user fixing the question

docs/components/labeler.md

@@ -116,8 +116,10 @@ The view section collects the configs determining how frontend renders the quest
 - `Field`: represents the field of `Document` your question data come from.
  - `Tags Key`: when you select `Field` as `.tags`, this textbox will show up, asking you to further specify which `.tags` key your question data comes from.
 - `Content Type`: you need to select the right content type to have the correct rendering on the the question data.
-- `Examples/View`: The maximum number of labeling examples on the frontend.
-- `TopK/Examples`: The maximum number of results for each example on the frontend.
+- `Questions/session`: The maximum number of labeling examples on the frontend.
+- `TopK/Question`: The maximum number of results for each example on the frontend.
+- `Start question`: The starting index of the question
+- `Keep same question`: If set, then `Start question` and `Questions/session` are locked. You will always get the same questions for labeling. 
 
 ````{tip}
 If your question panel looks like the image below, this means rendering is not setup correctly. You need to change `Field`, `Content Type` and `Tags Key` to correct the render setup.
@@ -129,6 +131,9 @@ If your question panel looks like the image below, this means rendering is not s
 
 ````
 
+```{tip}
+You can use `Keep same question` to debug your model: by fixing the query and observing how the model behaves after learning from your new labels.
+```
 
 #### Progress
 

finetuner/labeler/ui/index.html

@@ -22,7 +22,8 @@
  </div>
  <sidebar :labeler-config="labeler_config" :view-template="view_template" :tags="tags" :is-busy="is_busy"
  :progress-stats="progress_stats" :positive-rate="positive_rate" :negative-rate="negative_rate"
- :advanced-config="advanced_config" :save-progress="saveProgress"></sidebar>
+ :advanced-config="advanced_config" :save-progress="saveProgress"
+ :next-batch="next_batch"></sidebar>
  <div class="b-example-divider"></div>
  <div class="flex-grow-1 p-1 overflow-hidden">
  <div class="d-flex flex-column h-100">

finetuner/labeler/ui/js/components/sidebar.vue.js

@@ -9,6 +9,7 @@ const sidebar = {
  negativeRate: Number,
  advancedConfig: Object,
  saveProgress: Function,
+ nextBatch: Function,
  },
  template: `
  <div class="d-flex flex-column flex-shrink-0 p-3 sidebar">
@@ -61,18 +62,34 @@ const sidebar = {
  </select>
  </div>
  </div>
+ <div class="row my-1">
+ <label class="col-sm-6 col-form-label">Start question</label>
+ <div class="col-sm-6">
+ <input class="form-control" type="number" min="0"
+ v-model.number="labelerConfig.start_idx" v-on:input="nextBatch()" 
+ :disabled="labelerConfig.same_question">
+ </div>
+ </div>
  <div class="row my-1">
  <label class="col-sm-6 col-form-label">Questions/Session</label>
  <div class="col-sm-6">
  <input class="form-control" type="number" min="1" max="9"
- v-model.number="labelerConfig.example_per_view">
+ v-model.number="labelerConfig.example_per_view" v-on:input="nextBatch()"
+ :disabled="labelerConfig.same_question">
+ </div>
+ </div>
+ <div class="row my-1">
+ <label class="col-sm-6 col-form-label">Keep same question</label>
+ <div class="col-sm-6 d-flex align-items-center justify-content-center">
+ <input class="form-check-input" type="checkbox"
+ v-model="labelerConfig.same_question" v-on:input="nextBatch(true, false)">
  </div>
  </div>
  <div class="row my-1">
  <label class="col-sm-6 col-form-label">TopK/Question</label>
  <div class="col-sm-6">
  <input class="form-control" type="number" min="1" max="9"
- v-model.number="labelerConfig.topk_per_example">
+ v-model.number="labelerConfig.topk_per_example" v-on:input="nextBatch()">
  </div>
  </div>
  </div>

finetuner/labeler/ui/js/main.js

@@ -22,7 +22,8 @@ const app = new Vue({
  labeler_config: {
  content: 'uri',
  style: 'image',
- example_per_view: 5,
+ example_per_view: 3,
+ same_question: false,
  topk_per_example: 9,
  tags: '',
  start_idx: 0,
@@ -122,7 +123,11 @@ const app = new Vue({
  });
  app.progress_stats.total.value++
  app.fit_doc(doc)
- app.next_batch()
+ if (app.labeler_config.same_question) {
+ app.next_batch(true, false)
+ } else {
+ app.next_batch(false)
+ }
  },
  fit_doc: function (doc) {
  app.is_busy = true
@@ -155,13 +160,15 @@ const app = new Vue({
  return doc.tags[app.labeler_config.tags]
  }
  },
- next_batch: function () {
- let end_idx = app.labeler_config.start_idx + (app.labeler_config.example_per_view - app.cur_batch.length)
- if (end_idx === app.labeler_config.start_idx) {
+ next_batch: function (clear_exist=true, update_start_idx=true) {
+ if (clear_exist) {
+ app.cur_batch = []
+ }
+ let new_examples_to_query = Math.max(0, app.labeler_config.example_per_view - app.cur_batch.length)
+ let end_idx = app.labeler_config.start_idx + new_examples_to_query
+ if (end_idx <= app.labeler_config.start_idx) {
  return
  }
- let start_idx = app.labeler_config.start_idx
- app.labeler_config.start_idx = end_idx
  app.is_busy = true
  app.is_conn_broken = false
  $.ajax({
@@ -170,7 +177,7 @@ const app = new Vue({
  data: JSON.stringify({
  data: [],
  parameters: {
- 'start': start_idx,
+ 'start': app.labeler_config.start_idx,
  'end': end_idx,
  'topk': app.labeler_config.topk_per_example,
  'sample_size': app.advanced_config.sample_size.value
@@ -179,8 +186,15 @@ const app = new Vue({
  contentType: "application/json; charset=utf-8",
  dataType: "json",
  }).success(function (data, textStatus, jqXHR) {
+ if (update_start_idx) {
+ app.labeler_config.start_idx = end_idx
+ }
+
  app.cur_batch.push(...data['data'].docs)
- app.tags = Object.keys(data.data.docs[0].tags)
+ try {
+ app.tags = Object.keys(data.data.docs[0].tags)
+ } catch (e) {}
+
  app.is_busy = false
  app.progress_stats.this_session.value = app.cur_batch.length
  }).fail(function () {

finetuner/labeler/ui/main.css

@@ -2,6 +2,7 @@ body {
  min-height: 100vh;
  min-height: -webkit-fill-available;
  font-family: 'Poppins', sans-serif;
+ font-weight: lighter;
 }
 
 html {