optimal performance (#11)

* optimal performance

* change gzip to stream

* add coverage duration limit

* change interval variable name

* delete useless reference

* Add response headers

lyrebird_api_coverage/app_ui.py

@@ -2,7 +2,7 @@
 
 import lyrebird
 import os
-from flask import request, jsonify, Response
+from flask import request, jsonify, Response, stream_with_context
 from lyrebird_api_coverage.client.context import app_context
 
 from lyrebird_api_coverage.client import context
@@ -13,6 +13,7 @@
 from lyrebird_api_coverage.handlers.import_file_handler import ImportHandler
 from lyrebird_api_coverage.handlers.result_handler import ResultHandler, PLUGINS_DUMP_DIR
 from lyrebird import context
+import time
 
 
 class AppUI(lyrebird.PluginView):
@@ -22,61 +23,15 @@ def index(self):
  """
  return self.render_template('index.html')
 
- def get_coverage_data(self):
- # 获取app_context里面缓存的测试数据
- # 如果内存为空，则视为首次进入该页面
- data = app_context.merge_list
- if not data:
- # 获取base_data_config文件信息
- base_dict = BaseDataHandler().get_base_source()
- # 如果import的文件异常
- if isinstance(base_dict, Response):
- resp = base_dict
- else:
- mergeAlgorithm.first_result_handler(base_dict)
- mergeAlgorithm.coverage_arithmetic(base_dict)
- resp = jsonify({'test_data': app_context.merge_list, 'coverage': app_context.coverage})
- # 若不为空，则视为有测试缓存
- else:
- resp = jsonify({'test_data': app_context.merge_list, 'coverage': app_context.coverage})
- return resp
+ def generate(self, data):
+ rtext = json.dumps(data)
+ yield rtext
 
- # 获取init base数据 以及 测试缓存数据 API
  def get_test_data(self):
- # 获取app_context里面缓存的测试数据
- # 如果内存为空，则视为首次进入该页面
- data = app_context.merge_list
- if not data:
- # 获取base_data_config文件信息
- base_dict = BaseDataHandler().get_base_source()
- # 如果import的文件异常
- if isinstance(base_dict, Response):
- resp = base_dict
- else:
- mergeAlgorithm.first_result_handler(base_dict)
- resp = jsonify({'test_data': app_context.merge_list})
- # 若不为空，则视为有测试缓存
- else:
- resp = jsonify({'test_data': app_context.merge_list})
- return resp
+ return Response(stream_with_context(self.generate({'test_data': app_context.merge_list})), content_type='application/json')
 
  def get_coverage(self):
- # 获取app_context里面缓存的测试数据
- # 如果内存为空，则视为首次进入该页面
- data = app_context.coverage
- if not data:
- # 获取base_data_config文件信息
- base_dict = BaseDataHandler().get_base_source()
- # 如果import的文件异常
- if isinstance(base_dict, Response):
- resp = base_dict
- else:
- mergeAlgorithm.coverage_arithmetic(base_dict)
- resp = jsonify(app_context.coverage)
- # 若不为空，则视为有测试缓存
- else:
- resp = jsonify(app_context.coverage)
- return resp
+ return Response(stream_with_context(self.generate(app_context.coverage)), content_type='application/json')
 
  def save_result(self):
  # 传入文件名
@@ -103,7 +58,7 @@ def import_base(self):
  def get_filter_conf(self):
  msg = FilterHandler().get_filer_conf()
  # 如果返回的string包含报错信息，则是报错
- if "筛除配置文件有误" in msg:
+ if isinstance(msg, str):
  return context.make_fail_response(msg)
  else:
  return jsonify(msg)
@@ -129,6 +84,14 @@ def get_base_info(self):
  'version_code': app_context.version_code})
 
  def on_create(self):
+
+ # 获取base_data_config文件信息
+ base_dict = BaseDataHandler().get_base_source()
+ # 如果import的文件异常
+ if not isinstance(base_dict, Response):
+ mergeAlgorithm.first_result_handler(base_dict)
+ mergeAlgorithm.coverage_arithmetic(base_dict)
+
  # 设置模板目录（可选，设置模板文件目录。默认值templates）
  self.set_template_root('lyrebird_api_coverage')
  # 设置静态文件目录（可选，设置静态文件目录。默认值static）
@@ -143,9 +106,6 @@ def on_create(self):
  self.add_url_rule('/getTest', view_func=self.get_test_data)
  # 获取内存里保存的测试覆盖率信息
  self.add_url_rule('/getCoverage', view_func=self.get_coverage)
-
- self.add_url_rule('/getCoverageData', view_func=self.get_coverage_data)
-
  # 保存测试数据在本地
  self.add_url_rule('/saveResult', view_func=self.save_result, methods=['POST'])
  # 续传测试结果

lyrebird_api_coverage/client/context.py

@@ -31,6 +31,12 @@ def __init__(self):
  self.version_code = None
  # user_info
  self.user_info = {}
+ # 记录请求最后的时间，避免频繁emit io消息
+ self.endtime = 0
+ # 记录上次coverage变化的时间，避免频繁emit io消息
+ self.covtime = 0
+ # 时间间隔，每隔指定时间触发1次socket io消息，防止刷新频繁
+ self.SOCKET_PUSH_INTERVAL = 1
 
 
 # 单例模式

lyrebird_api_coverage/client/merge_algorithm.py

@@ -1,8 +1,7 @@
 import lyrebird
-
 from lyrebird_api_coverage.client import format_url
-
 from lyrebird_api_coverage.client.context import app_context
+import time
 
 
 class MergeAlgorithm:
@@ -103,10 +102,13 @@ def coverage_handler(self):
  coverage = round(test_len / app_context.coverage['len'] * 100, 2)
  # 为了传给Overbridge的socket信息format数据格式
  app_context.coverage['total'] = coverage
+ # 覆盖率有变化才emit & publish 覆盖率的变化消息给API-Coverage前端，overbridge前端，和消息总线
  if not history_coverage == coverage:
- # 覆盖率有变化才emit & publish 覆盖率的变化消息给API-Coverage前端，overbridge前端，和消息总线
- lyrebird.emit('update', app_context.coverage, namespace='/charts')
- lyrebird.emit('coverage message', app_context.coverage.get('total'), namespace='/api_coverage')
+ handler_time = time.time()
+ # 限制频繁emit io msg，在两次之间大于指定时间间隔才会emit
+ if handler_time - app_context.covtime > app_context.SOCKET_PUSH_INTERVAL:
+ lyrebird.emit('coverage message', app_context.coverage.get('total'), namespace='/api_coverage')
+ app_context.covtime = handler_time
  by_priority = [p.get('value') for p in app_context.coverage['priorities']]
  lyrebird.publish('coverage', 
  dict(

lyrebird_api_coverage/client/report.py

@@ -8,12 +8,13 @@
 
 class ReportHandler:
  def check_url_info(self, url, device_ip):
- specific_data = list(filter(lambda x: x.get('url') == url, app_context.merge_list))[0]
- desc = specific_data.get('desc')
- if specific_data.get('status') == 1:
+ specific_list = list(filter(lambda x: x.get('url') == url, app_context.merge_list))
+ if specific_list and specific_list[0].get('status') == 1:
+  desc = specific_list[0].get('desc')
  count_flag = 1
- priority = specific_data.get('priority')
+ priority = specific_list[0].get('priority')
  else:
+ desc = 'N/A'
  count_flag = -1
  priority = -1
  info_dict = {'url': url, 'desc': desc, 'priority': priority, 'count_flag': count_flag,

lyrebird_api_coverage/proxy/proxy_handler.py

@@ -10,6 +10,7 @@
 from lyrebird_api_coverage.client.report import report_worker
 from lyrebird_api_coverage.client.url_compare import compare_query
 from lyrebird_api_coverage.handlers.base_source_handler import BaseDataHandler
+import time
 
 
 """
@@ -23,6 +24,7 @@ class MyDataHandler:
  # self.user_list = []
 
  def handle(self, handler_context: HandlerContext):
+ req_starttime = time.time()
  # UI自动化等需要mock的手段orgin url 需要调用这个方法
  org_url = handler_context.get_origin_url()
  if not org_url:
@@ -34,36 +36,24 @@ def handle(self, handler_context: HandlerContext):
  path_id = handler_context.id
  device_ip = handler_context.client_address
 
- # 当请求过来的时候，base还没有init，需要进行init处理
- if not app_context.base_list:
- base_dict = BaseDataHandler().get_base_source()
- if isinstance(base_dict, Response):
- get_logger().error('API-Coverage base file is None.')
- else:
- mergeAlgorithm.first_result_handler(base_dict)
-
  if path in app_context.base_list:
  # merge到 context merge list中
  mergeAlgorithm.merge_handler_new(path, path_id)
  # 在base里的需要去计算下覆盖率
  mergeAlgorithm.coverage_handler()
- # path传给覆盖详情表格
- lyrebird.emit('test_data message', path, namespace='/api_coverage')
- # 如果设备信息抓取到,进行上报
- # if device_ip in app_context.info:
+ # 进行上报
  report_worker(path, device_ip)
+ # 计算差值，指定时间间隔内只发1次io msg，限制刷新频率
+ self.emit(req_starttime, path)
 
  # 如果path配置了对应的参数
- if path in list(app_context.path_param_dic.keys()):
+ elif path in app_context.path_param_dic:
  ulr_list = app_context.path_param_dic[path]
  flag = 0
  for item in ulr_list:
  if compare_query(item['url'], handler_context.request.url):
  mergeAlgorithm.merge_handler_new(item['url_base'], path_id)
  mergeAlgorithm.coverage_handler()
- lyrebird.emit('test_data message', item['url_base'], namespace='/api_coverage')
- # 如果设备信息抓取到,进行上报
- # if device_ip in app_context.info:
  report_worker(item['url_base'], device_ip)
  flag = 1
  # 如果参数组合不存在，提取关注的字段
@@ -82,21 +72,21 @@ def handle(self, handler_context: HandlerContext):
 
  mergeAlgorithm.merge_handler_new(url_pgroup, path_id)
  mergeAlgorithm.coverage_handler()
- lyrebird.emit('test_data message', url_pgroup, namespace='/api_coverage')
- # 如果设备信息抓取到,进行上报
- # if device_ip in app_context.info:
  report_worker(url_pgroup, device_ip)
+
+ # 计算差值，指定时间间隔内只发1次io msg，限制刷新频率
+ self.emit(req_starttime, path)
+
 
- # 如果不在base里，需要判断这些API是否被筛掉
+ # 如果不在base里，不需要merge到数据中
  else:
- # 如果不在筛除列表内，才进行merge等一系列算法
- if not Filter().filter_all(path):
- # merge到 context merge list中
- mergeAlgorithm.merge_handler_new(path, path_id)
- # 传给api_coverage前端的socket信息
- lyrebird.emit('test_data message', path, namespace='/api_coverage')
- # 如果设备信息抓取到,进行上报
- # if device_ip in app_context.info:
- report_worker(path, device_ip)
- else:
- pass
+ # mergeAlgorithm.merge_handler_new(path, path_id)
+ # 进行上报
+ report_worker(path, device_ip)
+
+
+ def emit(self, starttime, path):
+ duration = starttime - app_context.endtime
+ if duration > app_context.SOCKET_PUSH_INTERVAL:
+ app_context.endtime = starttime
+ lyrebird.emit('test_data message', path, namespace='/api_coverage')

lyrebird_api_coverage/static/component/main.vue

@@ -16,7 +16,6 @@
 <script>
 //websocket namespace /api_coverage
 let apicoverageIO = io("/api_coverage");
-let timer = null;
 
 module.exports = {
  mounted: function() {
@@ -28,15 +27,7 @@ module.exports = {
  loadCoverageData();
  });
  apicoverageIO.on("test_data message", function(msg) {
- console.log(msg);
  loadDetailData();
- if (!timer) {
- timer = setTimeout(() => {
- loadDetailData();
- clearTimeout(timer);
- timer = null;
- }, 1000);
- }
  });
  },
  data: function() {
@@ -47,9 +38,6 @@ module.exports = {
  targetContext: null
  };
  },
- updated: function() {
- console.log("init");
- },
  methods: {
  loadCoverageData: function() {
  this.$http.get("/ui/plugin/api_coverage/getCoverage").then(

lyrebird_api_coverage/templates/index.html

@@ -2,8 +2,4 @@
  <apicoverage></apicoverage>
 </div>
 
-
-<link href="https://cdn.bootcss.com/iview/2.14.0/styles/iview.css" rel="stylesheet">
-<script src="https://cdn.bootcss.com/iview/2.14.0/iview.min.js"></script>
-
 <script src="/ui/plugin/api_coverage/static/js/main.js"></script>

setup.py

@@ -8,7 +8,7 @@
 
 setup(
  name='lyrebird-api-coverage',
- version='0.2.6',
+ version='0.2.7',
  packages=['lyrebird_api_coverage'],
  url='https://github.com/meituan/lyrebird-api-coverage',
  author='HBQA',