Merge branch 'master' into nirizr/zestreleaser

Author: nirizr

.travis.yml

@@ -4,30 +4,36 @@ matrix:
   include:
     - python: "2.7"
       env: PROJECT=idaplugin ENV=conda
+    - python: "3.6"
+      env: PROJECT=idaplugin ENV=conda
     - python: "2.7"
       env: PROJECT=server ENV=data DJANGO_SETTINGS_MODULE='rematch.settings.debug'
       services:
-        - mysql
         - rabbitmq
-    - python: "3.5"
+    - python: "3.6"
       env: PROJECT=server ENV=data DJANGO_SETTINGS_MODULE='rematch.settings.production'
       services:
-        - mysql
         - rabbitmq
     - python: "2.7"
       env: PROJECT=server ENV=docker DJANGO_SETTINGS_MODULE='rematch.settings.docker'
-      services: docker
+      services:
+        - docker
     - python: "2.7"
       env: PROJECT=setup.py
     - python: "3.6"
       env: PROJECT=setup.py
     - python: "3.6"
       env: PROJECT=docs
+    - python: 2.7
+      env: PROJECT=interop ENV=conda DJANGO_SETTINGS_MODULE='rematch.settings.debug'
 
 branches:
   only:
     - master
 
+addons:
+  mariadb: '10.3'
+
 # Handle git submodules ourselves to replace ssh login with https login
 git:
     submodules: false
@@ -44,36 +50,43 @@ install:
       docker-compose -f ./${PROJECT}/docker-compose.yml build ;
       docker-compose -f ./${PROJECT}/docker-compose.yml up -d ;
       if [ -f ./tests/${PROJECT}/requirements.txt ]; then
-        docker-compose -f ./${PROJECT}/docker-compose.yml exec web pip install -r ./tests/requirements.txt ;
+        docker-compose -f ./${PROJECT}/docker-compose.yml exec web pip install --upgrade -r ./tests/server/requirements.txt ;
       fi ;
-      docker-compose -f ./${PROJECT}/docker-compose.yml exec web pip install coveralls pytest-cov ;
+      docker-compose -f ./${PROJECT}/docker-compose.yml exec web pip install --upgrade coveralls pytest-cov ;
+      pip install --upgrade flake8 ;
     fi ;
   - if [ "${ENV}" = "conda" ]; then
-      wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh ;
+      wget "https://repo.continuum.io/miniconda/Miniconda${TRAVIS_PYTHON_VERSION:0:1}-latest-Linux-x86_64.sh" -O miniconda.sh ;
       bash ./miniconda.sh -b -p "${HOME}/miniconda" ;
       export PATH=${HOME}/miniconda/bin:$PATH ;
       conda info -a ;
       conda install --yes python=${TRAVIS_PYTHON_VERSION} pyqt ;
     fi ;
   - if [ "${ENV}" = "data" ]; then
-      pip install --only-binary=numpy,scipy numpy scipy ;
-    fi ;
-  - if [ -f tests/${PROJECT}/requirements.txt ]; then
-      pip install -r tests/${PROJECT}/requirements.txt ;
+      pip install --upgrade --only-binary=numpy,scipy numpy scipy ;
     fi ;
-  - if [ -f "${PROJECT}/requirements.txt" ]; then
-      pip install -r "${PROJECT}/requirements.txt" ;
+  - if [ "${ENV}" != "docker" ]; then
+      if [ -f "tests/${PROJECT}/requirements.txt" ]; then
+        pip install --upgrade -r "tests/${PROJECT}/requirements.txt" ;
+      fi ;
+      if [ -f "${PROJECT}/requirements.txt" ]; then
+        pip install --upgrade -r "${PROJECT}/requirements.txt" ;
+      fi ;
+      pip install --upgrade flake8 coveralls pytest-cov ;
     fi ;
-  - pip install flake8 coveralls pytest-cov ;
 
 script:
   - flake8 ./${PROJECT} --show-source --statistics
   - if [ -d ./tests/${PROJECT} ]; then flake8 ./tests/${PROJECT} --show-source ; fi ;
   - if [ -d ./tests/${PROJECT} ]; then
       if [ "${ENV}" = "docker" ]; then
-        docker-compose -f ./${PROJECT}/docker-compose.yml exec web py.test -rapP ./ --verbose --cov-report= --cov=. ;
+        docker-compose -f ./${PROJECT}/docker-compose.yml exec web /bin/bash -c "PYTHONPATH=. py.test -rapP ./${PROJECT} ./tests/${PROJECT} -p tests.server.pymysql_mysqldb --verbose --cov-report= --cov=${PROJECT} ;" ;
+      elif [ "${PROJECT}" = "interop" ]; then
+        PYTHONPATH=. py.test -rapP ./server ./idaplugin ./tests/${PROJECT} -p tests.server.pymysql_mysqldb --verbose --cov-report= --cov=server --cov=idaplugin ;
+      elif [ "${PROJECT}" = "server" ]; then
+        PYTHONPATH=. py.test -rapP ./${PROJECT} ./tests/${PROJECT} -p tests.server.pymysql_mysqldb --verbose --cov-report= --cov=${PROJECT} ;
       else
-        PYTHONPATH=. py.test -rapP ./${PROJECT} ./tests/${PROJECT} --verbose --cov-report= --cov=${PROJECT} ;
+        py.test -rapP ./${PROJECT} ./tests/${PROJECT} --verbose --cov-report= --cov=${PROJECT} ;
       fi ;
     fi ;
   - if [ "${PROJECT}" = "setup.py" ]; then python ./setup.py server install ; fi ;

README.rst

@@ -35,8 +35,17 @@ the webservice's docker image by executing the following command:
 
 .. code-block:: console
 
-    $ docker-compose -f ./server/docker-compose.yml build ;
-    $ docker-compose -f ./server/docker-compose.yml up -d ;
+   $ docker-compose -f ./server/docker-compose.yml build ;
+   $ docker-compose -f ./server/docker-compose.yml up -d ;
+
+To create the rematch server administrator execute the following command:
+
+.. code-block:: console
+
+   $ docker-compose -f ./server/docker-compose.yml exec web ./server/manage.py createsuperuser
+
+Finally, point your browser to http://SERVER_IP:8000/admin/ to manage the
+service and add more users.
 
 .. |Read The Docs| image:: https://readthedocs.org/projects/rematch/badge/?version=latest
    :alt: Read The Docs

docs/installation.rst

@@ -41,6 +41,16 @@ directory of the rematch repository:
    $ docker-compose -f ./server/docker-compose.yml build ;
    $ docker-compose -f ./server/docker-compose.yml up -d ;
 
+Then execute the following command to set up the rematch server adminstrator
+account:
+
+.. code-block:: console
+
+   $ docker-compose -f ./server/docker-compose.yml exec web ./server/manage.py createsuperuser
+
+Finally, point your browser to http://SERVER_IP:8000/admin/ to manage the
+service and add more users.
+
 Installing Rematch IDA Plugin
 -----------------------------
 

idaplugin/plugin_rematch.py

@@ -1,4 +1,4 @@
-import rematch
+from . import rematch
 
 
 def PLUGIN_ENTRY():  # noqa: N802

idaplugin/rematch/actions/__init__.py

@@ -1,8 +1,8 @@
-import base
-import login
-import match
-import project
-import settings
+from . import base
+from . import login
+from . import match
+from . import project
+from . import settings
 
 
 __all__ = [base, login, match, project, settings]

idaplugin/rematch/actions/base.py

@@ -16,12 +16,13 @@ def __init__(self, ui_class):
     super(Action, self).__init__()
     self.ui_class = ui_class
     self.ui = None
+    self._running = False
 
   def __repr__(self):
     return "<Action: {}>".format(self.ui_class)
 
   def running(self):
-    return self.ui is not None
+    return self._running
 
 
 class IDAAction(Action, ida_kernwin.action_handler_t):
@@ -123,6 +124,7 @@ def activate(self, ctx):
     del ctx
     if self.running():
       return
+    self._running = True
 
     if callable(self.ui_class):
       self.ui = self.ui_class(accept_handler=self.accept_handler,
@@ -131,32 +133,15 @@ def activate(self, ctx):
                               submit_handler=self.submit_handler,
                               response_handler=self.response_handler,
                               exception_handler=self.exception_handler)
-      self.ui.finished.connect(self.close_dialog)
       self.ui.show()
     else:
       raise NotImplementedError("activation called on an action class with no "
                                 "ui_class defined")
 
-  def close_dialog(self):
-    """Destruction and cleanup of dialog bound to action on activation
-
-    This is called when dialog is finished for whatever reason, It is
-    guerenteed to be called after `self.finish_handler` due to slot execution
-    order since Qt4.6"""
-    log('action_base').info("Action finished: %s", self)
-
-    del self.ui
-    self.ui = None
-
-    self.force_update()
-
-  @staticmethod
-  def force_update():
-    """Forcefuly requests IDA kernel to update all widgets and views. Useful
-    for when delayed actions modify the program and/or plugin state without
-    IDA's awareness"""
-    iwid_all = 0xFFFFFFFF
-    ida_kernwin.request_refresh(iwid_all)
+  def finish_handler(self, status):
+    del status
+    log('actions').info("Action finished: %s", self)
+    self._running = False
 
 
 class IdbAction(IDAAction):

idaplugin/rematch/actions/match.py

@@ -18,7 +18,6 @@ class MatchAction(base.BoundFileAction):
 
   def __init__(self, *args, **kwargs):
     super(MatchAction, self).__init__(*args, **kwargs)
-    self._running = False
     self.functions = None
     self.results = None
     self.task_id = None
@@ -40,9 +39,6 @@ def __init__(self, *args, **kwargs):
     self.pbar = None
     self.timer = QtCore.QTimer()
 
-  def running(self):
-    return super(MatchAction, self).running() or self._running
-
   def clean(self):
     self.timer.stop()
     try:
@@ -58,15 +54,13 @@ def clean(self):
 
   def cancel_delayed(self):
     for delayed in self.delayed_queries:
-      log('match_action').info("async task cancelled: %s", repr(delayed))
       delayed.cancel()
     self.delayed_queries = []
 
   def cancel(self):
     log('match_action').info("match action cancelled")
     self.clean()
     self.cancel_delayed()
-    self._running = False
 
   @staticmethod
   def calc_file_version_hash():
@@ -146,6 +140,8 @@ def perform_upload(self):
       q = network.QueryWorker("POST", "collab/instances/",
                               params=self.instance_set, json=True)
       q.start(self.progress_advance)
+      self.delayed_queries.append(q)
+
       self.instance_set = []
       self.pbar.setMaximum(self.pbar.maximum() + 1)
     self.progress_advance()
@@ -162,7 +158,7 @@ def accept_upload(self):
     log('match_action').info("Data upload completed successfully")
 
     self.clean()
-    self.delayed_queries = []
+    self.cancel_delayed()
 
     self.start_task()
 
@@ -211,6 +207,7 @@ def perform_task(self):
       status = r['status']
       if status == 'failed':
         self.cancel()
+        # TODO: task failed, print a message box saying so
       elif progress_max:
         self.pbar.setMaximum(progress_max)
         if progress >= progress_max:
@@ -226,7 +223,7 @@ def accept_task(self):
     log('match_action').info("Remote task completed successfully")
 
     self.clean()
-    self.delayed_queries = []
+    self.cancel_delayed()
 
     self.start_results()
 
@@ -294,16 +291,16 @@ def handle_page(self, response, page_type):
       self.pbar.setValue(self.pbar.value() + 1)
 
     self.pbar.setValue(self.pbar.value() + len(response['results']))
-    log('match_action').info("result download progress: {} / {} with {}"
-                             "".format(self.pbar.value(), self.pbar.maximum(),
-                                       self.seen))
+    # log('match_action').info("result download progress: {} / {} with {}"
+    #                        "".format(self.pbar.value(), self.pbar.maximum(),
+    #                                  self.seen))
     if self.pbar.value() >= self.pbar.maximum():
       self.pbar.accept()
 
   def accept_results(self):
-    log('match_action').info("Result download completed successfully")
+    # log('match_action').info("Result download completed successfully")
 
     self.clean()
-    self.delayed_queries = []
+    self.cancel_delayed()
 
     self.results.show()

idaplugin/rematch/actions/settings.py

@@ -1,21 +1,24 @@
 import idaapi
 
 from . import base
-from .. import config
+from .. import config, netnode
 
 
 class SettingsAction(base.IDAAction):
   name = "&Settings"
 
   @staticmethod
-  def submit_handler(autocheck, autoupdate, autologin, autologout, debug):
+  def submit_handler(autocheck, autoupdate, autologin, autologout, debug,
+                     bound_file):
     config['settings']['update']['autocheck'] = autocheck
     config['settings']['update']['autoupdate'] = autoupdate
     config['settings']['login']['autologin'] = autologin
     config['settings']['login']['autologout'] = autologout
     config['debug'] = debug
     config.save()
 
+    netnode.bound_file_id = bound_file
+
     return True
 
   @staticmethod

idaplugin/rematch/collectors/__init__.py

@@ -6,16 +6,16 @@
 import json
 
 
-def collect(collectors, offset, instance_id=None):
-  for collector in collectors:
+def collect(collectors, offset):
+  for collector_cls in collectors:
     try:
-      collector_obj = collector(offset, instance_id)
-      r = collector_obj.collect()
+      r = collector_cls(offset).serialize()
       if r:
         yield r
     except UnicodeDecodeError:
       log('annotation').error("Unicode decoding error during serializion of "
-                              "type %s at offset %x", collector.type, offset)
+                              "type %s at offset %x", collector_cls.type,
+                              offset)
 
 
 def apply(offset, annotation):
@@ -27,7 +27,7 @@ def apply(offset, annotation):
 
     annotation_data = json.loads(annotation['data'])
 
-    if annotation_cls.data(offset) == annotation_data:
+    if annotation_cls(offset).data() == annotation_data:
       log('annotation_apply').info("Setting annotation %s skipped at %s with "
                                    "%s because value is already set",
                                    annotation_cls, offset, annotation_data)

idaplugin/rematch/collectors/annotations/annotation.py

@@ -1,12 +1,5 @@
-import json
+from .. import collector
 
 
-class Annotation(object):
-  @classmethod
-  def collect(cls, offset, instance_id=None):
-    data = cls.data(offset)
-    if not data:
-      return None
-
-    data = json.dumps(data)
-    return {"instance": instance_id, "type": cls.type, "data": data}
+class Annotation(collector.Collector):
+  pass

idaplugin/rematch/collectors/annotations/assembly.py

@@ -13,9 +13,8 @@
 class AssemblyAnnotation(annotation.Annotation):
   type = 'assembly'
 
-  @staticmethod
-  def data(offset):
-    func = ida_funcs.get_func(offset)
+  def data(self):
+    func = ida_funcs.get_func(self.offset)
 
     def clean(asm):
       """This removes markers of function offsets, including hidden variable
@@ -51,6 +50,5 @@ def node_contained(node):
     return nodes_data
 
   @staticmethod
-  def apply(offset, data):
-    del offset
+  def apply(data):
     del data