[WIP] implement match backend

.flake8

@@ -1,2 +1,2 @@
 [flake8]
-ignore = E111, E113, E114, E121, E125
+ignore = E111, E113, E114, E121, E125, E127

.travis.yml

@@ -1,7 +1,6 @@
 language: python
 python:
   - "2.7"
-  - "3.2"
   - "3.4"
   - "3.5"
 env:
@@ -10,8 +9,6 @@ env:
 matrix:
   # idaplugin is run with python2.7, we'll check against python3 to assert compatibility
   exclude:
-    - python: "3.2"
-      env: PROJECT=idaplugin
     - python: "3.4"
       env: PROJECT=idaplugin
 
@@ -28,6 +25,19 @@ before_install:
     - git submodule update --init --recursive
 
 install:
+  - if [ ${PROJECT} = "server" ]; then
+      set +vx;
+      if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+        wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+      else
+        wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+      fi;
+      bash ./miniconda.sh -b -p ${HOME}/miniconda;
+      export PATH=${HOME}/miniconda/bin:$PATH;
+      conda info -a;
+      conda update --yes conda;
+      conda install --yes python=${TRAVIS_PYTHON_VERSION} atlas numpy scipy scikit-learn;
+    fi;
   - if [ -f ${PROJECT}/requirements.txt ]; then pip install -r ${PROJECT}/requirements.txt ; fi
   - pip install flake8
 

idaplugin/rematch/actions/match.py

@@ -1,63 +1,178 @@
-import idaapi
-import idc
-from idautils import Functions
-
 from ..idasix import QtCore, QtWidgets
+import idautils
+
+from ..dialogs.match import MatchDialog
 
 from .. import instances
 from .. import network, netnode
 from . import base
 
 
-class MatchAllAction(base.BoundFileAction):
-  name = "&Match all"
-  group = "Match"
+class MatchAction(base.BoundFileAction):
+  name = "&Match"
+  dialog = MatchDialog
+
+  def __init__(self, *args, **kwargs):
+    super(MatchAction, self).__init__(*args, **kwargs)
+    self.function_gen = None
+    self.pbar = None
+    self.timer = None
+    self.task_id = None
+    self.instance_set = []
+
+    self.source = None
+    self.source_single = None
+    self.source_range = None
+    self.target = None
+    self.target_project = None
+    self.target_file = None
+    self.methods = None
+
+  def get_functions(self):
+    if self.source == 'idb':
+      return set(idautils.Functions())
+    elif self.source == 'user':
+      raise NotImplementedError("All user functions are not currently "
+                                "supported as source value.")
+    elif self.source == 'single':
+      return [self.source_single]
+    elif self.source == 'range':
+      return set(idautils.Functions(self.source_range[0],
+                                    self.source_range[1]))
+
+    raise ValueError("Invalid source value received from MatchDialog: {}"
+                     "".format(self.source))
+
+  def submit_handler(self, source, source_single, source_range, target,
+                     target_project, target_file, methods):
+    self.source = source
+    self.source_single = source_single
+    self.source_range = source_range
+    self.target = target
+    self.target_project = target_project if target == 'project' else None
+    self.target_file = target_file if target == 'file' else None
+    self.methods = methods
 
-  def activate(self, ctx):
-    self.file_id = netnode.bound_file_id
+    functions = self.get_functions()
+    if not functions:
+      return False
+
+    self.function_gen = enumerate(functions)
+    self.pbar = QtWidgets.QProgressDialog()
+    self.pbar.setLabelText("Processing IDB... You may continue working,\nbut "
+                           "please avoid making any ground-breaking changes.")
+    self.pbar.setRange(0, len(functions))
+    self.pbar.setValue(0)
+    self.pbar.canceled.connect(self.cancel_upload)
+    self.pbar.rejected.connect(self.reject_upload)
+    self.pbar.accepted.connect(self.accept_upload)
 
-    self.function_gen = enumerate(Functions())
-    pd = QtWidgets.QProgressDialog(labelText="Processing functions...",
-                                   minimum=0, maximum=len(list(Functions())))
-    self.progress = pd
-    self.progress.canceled.connect(self.cancel)
     self.timer = QtCore.QTimer()
-    self.timer.timeout.connect(self.perform)
-    self.timer.start()
+    self.timer.timeout.connect(self.perform_upload)
+    self.timer.start(0)
+
+    return True
 
-  def perform(self):
+  def perform_upload(self):
     try:
       i, offset = self.function_gen.next()
+    except StopIteration:
+      self.timer.stop()
+      return
 
-      func = instances.FunctionInstance(self.file_id, offset)
-      network.query("POST", "collab/instances/", params=func.serialize(),
-                    json=True)
+    try:
+      func = instances.FunctionInstance(netnode.bound_file_id, offset)
+      self.instance_set.append(func.serialize())
 
-      i = i + 1
-      self.progress.setValue(i)
-      if (i >= self.progress.maximum()):
-        self.timer.stop()
-    except:
-      self.timer.stop()
+      if len(self.instance_set) >= 100:
+        network.delayed_query("POST", "collab/instances/",
+                              params=self.instance_set, json=True,
+                              callback=self.progress_advance)
+        self.instance_set = []
+        self.pbar.setMaximum(self.pbar.maximum() + 1)
+      self.progress_advance()
+    except Exception:
+      self.cancel_upload()
       raise
 
-  def cancel(self):
+  def progress_advance(self, result=None):
+    del result
+    new_value = self.pbar.value() + 1
+    self.pbar.setValue(new_value)
+    if new_value >= self.pbar.maximum():
+      self.pbar.accept()
+
+  def cancel_upload(self):
     self.timer.stop()
+    self.timer = None
+    self.pbar = None
 
+  def reject_upload(self):
+    self.cancel_upload()
 
-class MatchFunctionAction(base.BoundFileAction):
-  name = "Match &Function"
-  group = "Match"
+  def accept_upload(self):
+    self.cancel_upload()
 
-  @staticmethod
-  def activate(ctx):
-    file_id = netnode.bound_file_id
+    if self.source == 'idb':
+      self.source_range = [None, None]
+    elif self.source == 'single':
+      self.source_range = [self.source_single, self.source_single]
+    elif self.source == 'range':
+      pass
+    else:
+      raise NotImplementedError("Unsupported source type encountered in task "
+                                "creation")
 
-    function = idaapi.choose_func("Choose function to match with database",
-                                  idc.ScreenEA())
-    if function is None:
-      return
+    params = {'source_file': netnode.bound_file_id,
+              'source_start': self.source_range[0],
+              'source_end': self.source_range[1],
+              'target_project': self.target_project,
+              'target_file': self.target_file,
+              'source': self.source, 'methods': self.methods}
+    r = network.query("POST", "collab/tasks/", params=params, json=True)
+    self.task_id = r['id']
+
+    self.pbar = QtWidgets.QProgressDialog()
+    self.pbar.setLabelText("Waiting for remote matching... You may continue "
+                           "working without any limitations.")
+    self.pbar.setRange(0, int(r['progress_max']) if r['progress_max'] else 0)
+    self.pbar.setValue(int(r['progress']))
+    self.pbar.canceled.connect(self.cancel_task)
+    self.pbar.rejected.connect(self.reject_task)
+    self.pbar.accepted.connect(self.accept_task)
+    self.pbar.show()
+
+    self.timer = QtCore.QTimer()
+    self.timer.timeout.connect(self.perform_task)
+    self.timer.start(1000)
+
+  def perform_task(self):
+    try:
+      r = network.query("GET", "collab/tasks/{}/".format(self.task_id),
+                        json=True)
+
+      progress_max = int(r['progress_max']) if r['progress_max'] else None
+      progress = int(r['progress'])
+      status = r['status']
+      if status == 'failed':
+        self.pbar.reject()
+      elif progress_max:
+        self.pbar.setMaximum(progress_max)
+        if progress >= progress_max:
+          self.pbar.accept()
+        else:
+          self.pbar.setValue(progress)
+    except Exception:
+      self.cancel_task()
+      raise
+
+  def cancel_task(self):
+    self.timer.stop()
+    self.timer = None
+    self.pbar = None
+
+  def reject_task(self):
+    self.cancel_task()
 
-    data = instances.FunctionInstance(file_id, function.startEA)
-    network.query("POST", "collab/instances/", params=data.serialize(),
-                  json=True)
+  def accept_task(self):
+    self.cancel_task()

idaplugin/rematch/dialogs/__init__.py

@@ -1,6 +1,7 @@
 import base
 import login
+import match
 import project
 import settings
 
-__all__ = [base, login, project, settings]
+__all__ = [base, login, match, project, settings]