Merge pull request #2 from wufeifei/master

merge

README.md

@@ -5,20 +5,43 @@
 [![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/wufeifei/cobra/blob/master/LICENSE)
 
 当前版本非正式版本，正式版本正在做最后的内测中，建议等正式版本出来后再使用，敬请期待！
-[![asciicast](https://asciinema.org/a/132572.png)](https://asciinema.org/a/132572)
+[![asciicast](https://raw.githubusercontent.com/wufeifei/cobra/master/docs/report_03.jpg)](https://asciinema.org/a/132572)
 
 ## Introduction（介绍）
-Cobra是一款**源代码安全审计**工具，支持检测多种开发语言源代码中的数十种漏洞和风险点。
+Cobra是一款**源代码安全审计**工具，支持检测多种开发语言源代码中的大部分显著的安全问题和漏洞。
 
 ## Features（特点）
-#### [Multi-language support（支持多种开发语言）](https://github.com/wufeifei/cobra/blob/master/rules/languages.xml)
+#### Multi-language support（支持多种开发语言）
 > 支持PHP、Java等开发语言，并支持数十种类型文件。
 
-#### [Supported Multi-Vulnerabilities（支持多种漏洞类型）](https://github.com/wufeifei/cobra/blob/master/rules/vulnerabilities.xml)
+#### Supported Multi-Vulnerabilities（支持多种漏洞类型）
 > 首批开放数万条不安全的依赖检查规则和数十条代码安全扫描规则，后续将持续开放更多扫描规则。
 
-#### CLI、API（命令行模式和API模式）
-> 提供本地Server服务，可支持本地API接口，方便和其它系统（发布系统、CI等）对接扩展
+#### GUI、CLI、API（命令行模式和API模式）
+> 提供本地Web Server服务，可使用GUI可视化操作，也可支持本地API接口，方便和其它系统（发布系统、CI等）对接扩展。
+
+## Snapshot（截图）
+[![report01](https://raw.githubusercontent.com/wufeifei/cobra/master/docs/report_01.jpg)](https://wufeifei.github.io/cobra/api)
+[![report02](https://raw.githubusercontent.com/wufeifei/cobra/master/docs/report_02.jpg)](https://wufeifei.github.io/cobra/api)
 
 ## Documents（文档）
-- [Cobra文档](https://wufeifei.github.io/cobra/)
+- 安装
+    - [Cobra安装](https://wufeifei.github.io/cobra/installation)
+- 基础使用
+    - [CLI模式使用方法](https://wufeifei.github.io/cobra/cli)
+    - [API模式使用方法](https://wufeifei.github.io/cobra/api)
+- 进阶使用
+    - [高级功能配置](https://wufeifei.github.io/cobra/config)
+    - [升级框架和规则源](https://wufeifei.github.io/cobra/upgrade)
+- 规则开发规范
+    - [规则模板](https://wufeifei.github.io/cobra/rule_template)
+    - [规则样例](https://wufeifei.github.io/cobra/rule_demo)
+    - [规则开发流程](https://wufeifei.github.io/cobra/rule_flow)
+- 框架引擎
+    - [开发语言和文件类型定义](https://wufeifei.github.io/cobra/languages)
+    - [漏洞类型定义](https://wufeifei.github.io/cobra/labels)
+    - [危害等级定义](https://wufeifei.github.io/cobra/level)
+    - [程序目录结构](https://wufeifei.github.io/cobra/tree)
+- 贡献代码
+    - [单元测试](https://wufeifei.github.io/cobra/test)
+    - [贡献者](https://wufeifei.github.io/cobra/contributors)

cobra/__init__.py

@@ -16,10 +16,12 @@
 import time
 import argparse
 import logging
+import traceback
 from .log import logger
 from . import cli, api, config
 from .cli import get_sid
 from .engine import Running
+from .utils import unhandled_exception_message, create_github_issue
 
 from .__version__ import __title__, __introduction__, __url__, __version__
 from .__version__ import __author__, __author_email__, __license__
@@ -33,52 +35,58 @@
 
 
 def main():
-    # arg parse
-    t1 = time.clock()
-    parser = argparse.ArgumentParser(prog=__title__, description=__introduction__, epilog=__epilog__, formatter_class=argparse.RawDescriptionHelpFormatter)
-
-    parser_group_scan = parser.add_argument_group('Scan')
-    parser_group_scan.add_argument('-t', '--target', dest='target', action='store', default='', metavar='<target>', help='file, folder, compress, or repository address')
-    parser_group_scan.add_argument('-f', '--format', dest='format', action='store', default='json', metavar='<format>', choices=['html', 'json', 'csv', 'xml'], help='vulnerability output format (formats: %(choices)s)')
-    parser_group_scan.add_argument('-o', '--output', dest='output', action='store', default='', metavar='<output>', help='vulnerability output STREAM, FILE, HTTP API URL, MAIL')
-    parser_group_scan.add_argument('-r', '--rule', dest='special_rules', action='store', default=None, metavar='<rule_id>', help='specifies rules e.g: CVI-100001,cvi-190001')
-    parser_group_scan.add_argument('-d', '--debug', dest='debug', action='store_true', default=False, help='open debug mode')
-    parser_group_scan.add_argument('-sid', '--sid', dest='sid', action='store', default=None, help='scan id(API)')
-
-    parser_group_server = parser.add_argument_group('RESTful')
-    parser_group_server.add_argument('-H', '--host', dest='host', action='store', default=None, metavar='<host>', help='REST-JSON API Service Host')
-    parser_group_server.add_argument('-P', '--port', dest='port', action='store', default=None, metavar='<port>', help='REST-JSON API Service Port')
-
-    args = parser.parse_args()
-
-    if args.debug:
-        logger.setLevel(logging.DEBUG)
-        logger.debug('[INIT] set logging level: debug')
-
-    if args.host is None and args.port is None and args.target is '' and args.output is '':
-        parser.print_help()
-        exit()
-
-    if args.host is not None and args.port is not None:
-        logger.debug('[INIT] start RESTful Server...')
-        api.start(args.host, args.port, args.debug)
-    else:
-        logger.debug('[INIT] start scanning...')
-
-        # Native CLI mode
-        if args.sid is None:
-            a_sid = get_sid(args.target, True)
-            data = {
-                'status': 'running',
-                'report': ''
-            }
-            Running(a_sid).status(data)
+    try:
+        # arg parse
+        t1 = time.clock()
+        parser = argparse.ArgumentParser(prog=__title__, description=__introduction__, epilog=__epilog__, formatter_class=argparse.RawDescriptionHelpFormatter)
+
+        parser_group_scan = parser.add_argument_group('Scan')
+        parser_group_scan.add_argument('-t', '--target', dest='target', action='store', default='', metavar='<target>', help='file, folder, compress, or repository address')
+        parser_group_scan.add_argument('-f', '--format', dest='format', action='store', default='json', metavar='<format>', choices=['html', 'json', 'csv', 'xml'], help='vulnerability output format (formats: %(choices)s)')
+        parser_group_scan.add_argument('-o', '--output', dest='output', action='store', default='', metavar='<output>', help='vulnerability output STREAM, FILE, HTTP API URL, MAIL')
+        parser_group_scan.add_argument('-r', '--rule', dest='special_rules', action='store', default=None, metavar='<rule_id>', help='specifies rules e.g: CVI-100001,cvi-190001')
+        parser_group_scan.add_argument('-d', '--debug', dest='debug', action='store_true', default=False, help='open debug mode')
+        parser_group_scan.add_argument('-sid', '--sid', dest='sid', action='store', default=None, help='scan id(API)')
+
+        parser_group_server = parser.add_argument_group('RESTful')
+        parser_group_server.add_argument('-H', '--host', dest='host', action='store', default=None, metavar='<host>', help='REST-JSON API Service Host')
+        parser_group_server.add_argument('-P', '--port', dest='port', action='store', default=None, metavar='<port>', help='REST-JSON API Service Port')
+
+        args = parser.parse_args()
+
+        if args.debug:
+            logger.setLevel(logging.DEBUG)
+            logger.debug('[INIT] set logging level: debug')
+
+        if args.host is None and args.port is None and args.target is '' and args.output is '':
+            parser.print_help()
+            exit()
+
+        if args.host is not None and args.port is not None:
+            logger.debug('[INIT] start RESTful Server...')
+            api.start(args.host, args.port, args.debug)
         else:
-            # API call CLI mode
-            a_sid = args.sid
-        cli.start(args.target, args.format, args.output, args.special_rules, a_sid)
-    t2 = time.clock()
-    logger.info('[INIT] Done! Consume Time:{ct}s'.format(ct=t2 - t1))
+            logger.debug('[INIT] start scanning...')
+
+            # Native CLI mode
+            if args.sid is None:
+                a_sid = get_sid(args.target, True)
+                data = {
+                    'status': 'running',
+                    'report': ''
+                }
+                Running(a_sid).status(data)
+            else:
+                # API call CLI mode
+                a_sid = args.sid
+            cli.start(args.target, args.format, args.output, args.special_rules, a_sid)
+        t2 = time.clock()
+        logger.info('[INIT] Done! Consume Time:{ct}s'.format(ct=t2 - t1))
+    except Exception as e:
+        err_msg = unhandled_exception_message()
+        exc_msg = traceback.format_exc()
+        logger.warning(exc_msg)
+        create_github_issue(err_msg, exc_msg)
 
 
 if __name__ == '__main__':

cobra/__version__.py

@@ -1,7 +1,13 @@
+import sys
+import platform
+
 __title__ = 'cobra'
 __description__ = 'Code Security Audit'
 __url__ = 'https://github.com/wufeifei/cobra'
-__version__ = '2.0.0'
+__issue_page__ = 'https://github.com/wufeifei/cobra/issues/new'
+__python_version__ = sys.version.split()[0]
+__platform__ = platform.platform()
+__version__ = '2.0.0-beta.4'
 __author__ = 'Feei'
 __author_email__ = 'feei@feei.cn'
 __license__ = 'MIT License'
@@ -21,5 +27,5 @@
   {m} -t {td} -f json -o /tmp/report.json 
   {m} -t {tg} -f json -o feei@feei.cn 
   {m} -t {tg} -f json -o http://push.to.com/api 
-  {m} -H 127.0.0.1 -P 80
-""".format(m='./cobra.py', td='tests/vulnerabilities', tg='https://github.com/wufeifei/vc.git')
+  sudo {m} -H 127.0.0.1 -P 80
+""".format(m='./cobra.py', td='tests/vulnerabilities', tg='https://github.com/ethicalhack3r/DVWA')

cobra/api.py

@@ -11,26 +11,27 @@
     :license:   MIT, see LICENSE for more details.
     :copyright: Copyright (c) 2017 Feei. All rights reserved
 """
-import socket
 import errno
-import time
-import os
-import re
 import json
-import requests
 import multiprocessing
+import os
+import re
+import socket
 import subprocess
 import threading
+import time
 import traceback
+
+import requests
 from flask import Flask, request, render_template
 from flask_restful import Api, Resource
-from werkzeug.utils import secure_filename
+
 from . import cli
 from .cli import get_sid
+from .config import Config, running_path, code_path, package_path
 from .engine import Running
 from .log import logger
-from .config import Config, running_path, code_path, package_path
-from .utils import allowed_file
+from .utils import allowed_file, secure_filename, PY2
 
 try:
     # Python 3
@@ -88,7 +89,7 @@ def post():
         running = Running(a_sid)
 
         # Write a_sid running data
-        running.init_list()
+        running.init_list(data=target)
 
         # Write a_sid running status
         data = {
@@ -104,15 +105,17 @@ def post():
                 producer(task=arg)
 
             result = {
-                "msg": "Add scan job successfully.",
-                "sid": a_sid,
+                'msg': 'Add scan job successfully.',
+                'sid': a_sid,
+                'total_target_num': len(target),
             }
         else:
             arg = (target, formatter, output, rule, a_sid)
             producer(task=arg)
             result = {
-                "msg": "Add scan job successfully.",
-                "sid": a_sid,
+                'msg': 'Add scan job successfully.',
+                'sid': a_sid,
+                'total_target_num': 1,
             }
 
         return {"code": 1001, "result": result}
@@ -147,8 +150,8 @@ def post():
             return data
         else:
             result = running.status()
+            r_data = running.list()
             if result['status'] == 'running':
-                r_data = running.list()
                 ret = True
                 result['still_running'] = dict()
                 for s_sid, git in r_data['sids'].items():
@@ -163,7 +166,10 @@ def post():
                 'sid': sid,
                 'status': result.get('status'),
                 'report': result.get('report'),
-                'still_running': result.get('still_running')
+                'still_running': result.get('still_running'),
+                'total_target_num': r_data.get('total_target_num'),
+                'not_finished': int(r_data.get('total_target_num')) - len(r_data.get('sids'))
+                                + len(result.get('still_running')),
             }
         return {"code": 1001, "result": data}
 
@@ -258,20 +264,29 @@ def post():
         file_path = data.get('file_path')
 
         if target.startswith('http'):
-            target = re.findall(r'(.*?\.git)', target)[0]
             repo_user = target.split('/')[-2]
-            repo_name = target.split('/')[-1].replace('.git', '')
-            # repo_directory = os.path.join(os.path.join(os.path.join(code_path, 'git'), repo_user), repo_name)
+
+            split_target = target.split(':')
+            if len(split_target) == 3:
+                repo_name = split_target[1].split('/')[-1].replace('.git', '')
+            elif len(split_target) == 2:
+                repo_name = split_target[1].split('/')[-1].replace('.git', '')
+            else:
+                return {'code': 1002, 'msg': 'Invalid target.'}
             repo_directory = os.path.join(code_path, 'git', repo_user, repo_name)
 
-            file_path = map(secure_filename, file_path.split('/'))
+            if PY2:
+                file_path = map(secure_filename, [path.decode('utf-8') for path in file_path.split('/')])
+            else:
+                file_path = map(secure_filename, [path for path in file_path.split('/')])
+
 
             # 循环生成路径，避免文件越级读取
             file_name = repo_directory
             for _dir in file_path:
                 file_name = os.path.join(file_name, _dir)
-            print(file_name)
             if os.path.exists(file_name):
+                extension = guess_type(file_name)
                 if is_text(file_name):
                     with open(file_name, 'r') as f:
                         file_content = f.read()
@@ -280,7 +295,8 @@ def post():
             else:
                 return {'code': 1002, 'msg': 'No such file.'}
 
-            return {'code': 1001, 'result': file_content}
+            return {'code': 1001, 'result': {'file_content': file_content,
+                                             'extension': extension}}
 
 
 @app.route('/', methods=['GET', 'POST'])
@@ -328,11 +344,24 @@ def summary():
             total_vul_number, critical_vul_number, high_vul_number, medium_vul_number, low_vul_number = 0, 0, 0, 0, 0
             rule_filter = dict()
             targets = list()
-            for s_sid in scan_list.keys():
+
+            for s_sid, target_str in scan_list.items():
                 target_info = dict()
+
+                # 分割项目地址与分支，默认 master
+                split_target = target_str.split(':')
+                if len(split_target) == 3:
+                    target, branch = '{p}:{u}'.format(p=split_target[0], u=split_target[1]), split_target[-1]
+                elif len(split_target) == 2:
+                    target, branch = target_str, 'master'
+                else:
+                    logger.critical('Target url exception: {u}'.format(u=target_str))
+                    target, branch = target_str, 'master'
+
                 target_info.update({
                     'sid': s_sid,
-                    'target': scan_list.get(s_sid),
+                    'target': target,
+                    'branch': branch,
                 })
                 s_sid_file = os.path.join(running_path, '{sid}_data'.format(sid=s_sid))
                 with open(s_sid_file, 'r') as f:
@@ -395,6 +424,26 @@ def is_text(fn):
     return 'text' in msg.decode('utf-8')
 
 
+def guess_type(fn):
+    import mimetypes
+    extension = mimetypes.guess_type(fn)[0]
+    if extension:
+        """text/x-python or text/x-java-source"""
+        # extension = extension.split('/')[1]
+        extension = extension.replace('-source', '')
+    else:
+        extension = fn.split('/')[-1].split('.')[-1]
+
+    custom_ext = {
+        'html': 'htmlmixed',
+        'md': 'markdown',
+    }
+    if custom_ext.get(extension) is not None:
+        extension = custom_ext.get(extension)
+
+    return extension.lower()
+
+
 def start(host, port, debug):
     logger.info('Start {host}:{port}'.format(host=host, port=port))
     api = Api(app)

cobra/cast.py

@@ -283,7 +283,7 @@ def is_controllable_param(self):
                         return False, self.data
                     logger.debug("[AST] Is assign out data: `No`")
                     return True, self.data
-                logger.critical("[AST] Not Java/PHP, can't parse")
+                logger.debug("[AST] Not Java/PHP, can't parse ({l})".format(l=self.language))
                 return False, self.data
         else:
             logger.warning("[AST] Can't get `param`, check built-in rule")