diff --git a/.gitignore b/.gitignore index 894a44c..6cf2103 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +*.log +*.log.* +config.py +*.pth +.idea/ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/GiaoRobot.py b/GiaoRobot.py new file mode 100644 index 0000000..c7d0ebb --- /dev/null +++ b/GiaoRobot.py @@ -0,0 +1,54 @@ +import cv2 +import time +import config +from logger import Logger +from push import Push +from storage import Storage +from camera import Camera + +log = Logger.getLogger("Giao") + + +def main(): + # open camera + cap = cv2.VideoCapture(0) + + last_send_time = 0 + + camera = Camera(config.TinyFaceModelPath, config.TinyFaceModelDevice) + push = Push(config.dingTalkWebhookAccessToken) + storage = Storage(config.SinaSCSAccessKey, config.SinaSCSSecretKey, config.SinaSCSBucketName, + config.SinaSCSBucketUrl) + + while (cap.isOpened()): + try: + ret, img = cap.read() + if ret == True: + # img = img[config.MonitorRegion] + imgRegion = img[185:305, 270:490] + diff = camera.detectDiff(imgRegion) + if diff < 150: + log.info("different < 150") + time.sleep(0.2) + continue + bboxes = camera.detectFaces(imgRegion) + log.info("bboxes len %d", len(bboxes)) + if len(bboxes) > 0 and (time.time() - last_send_time) > 30: + last_send_time = time.time() + saveName = str(time.strftime('images/%Y%m%d_%H_%M_%S.png')) + cv2.imwrite(saveName, img) + saveName = str(time.strftime('images/%Y%m%d_%H_%M_%S.jpg')) + cv2.imwrite(saveName, img) + url = storage.saveImage(saveName) + log.info("send giao! %s" % url) + push.sendImage(config.PushTitle, url) + except Exception as e: + log.error("error: %s", e) + + time.sleep(0.5) + + cap.release() + + +if __name__ == '__main__': + main() diff --git a/camera.py b/camera.py new file mode 100644 index 0000000..f5d82bc --- /dev/null +++ b/camera.py @@ -0,0 +1,39 @@ +from tinyface import TinyFace +import cv2, time +import numpy as np +from logger import Logger + +log = Logger.getLogger('camera') + + +class Camera(object): + + def __init__(self, modelPath, device='cpu'): + self.TF = TinyFace(modelPath, device=device) + self.lastImg = None + + def detectDiff(self, img): + tStart = time.time() + if self.lastImg is None: + ret, binary_img = cv2.threshold(img, 127, 255, cv2.THRESH_BINARY) + binary_img = binary_img / 255 + self.lastImg = binary_img + return 99999999 + diff = 0 + ret, binary_img = cv2.threshold(img, 127, 255, cv2.THRESH_BINARY) + if self.lastImg is not None: + binary_img = binary_img / 255 + diff = binary_img - self.lastImg + diff = np.abs(np.sum(diff)) + self.lastImg = binary_img + else: + self.lastImg = binary_img / 255 + + log.info("diff: %d, using %.6f sec" % (diff, time.time() - tStart)) + + return diff + + def detectFaces(self, img): + imgRegion = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + bboxes = self.TF.detect_faces(imgRegion, conf_th=0.9, scales=[1]) + return bboxes diff --git a/config.py.example b/config.py.example new file mode 100644 index 0000000..cd9da67 --- /dev/null +++ b/config.py.example @@ -0,0 +1,68 @@ +# dingTalk webhook access token +dingTalkWebhookAccessToken = 'xxx' + +# the region size need to monitor +# [x, y, w, h] +MonitorRegion = [-1, -1, -1, -1] + +# TinyFace model path +TinyFaceModelPath = './checkpoint_50.pth' +# cpu or cuda +TinyFaceModelDevice = 'cpu' + +# Sina SCS access key, secret key and bucket name +SinaSCSAccessKey = 'xxx' +SinaSCSSecretKey = 'xxx' +SinaSCSBucketName = 'xxx' +SinaSCSBucketUrl = 'http://xxx.xxx/xxx/%s' + +# push title +PushTitle = '[进入预警]' + +# Logger config json text +LoggerJsonConfig = '''{ + "version":1, + "disable_existing_loggers":false, + "formatters":{ + "simple":{ + "format":"[%(asctime)s][%(funcName)-15s:%(lineno)-4s] - %(name)-10s - %(levelname)-5s : %(message)s" + } + }, + "handlers":{ + "console":{ + "class":"logging.StreamHandler", + "level":"DEBUG", + "formatter":"simple", + "stream":"ext://sys.stdout" + }, + "info_file_handler":{ + "class":"logging.handlers.RotatingFileHandler", + "level":"INFO", + "formatter":"simple", + "filename":"info.log", + "maxBytes":10485760, + "backupCount":20, + "encoding":"utf8" + }, + "error_file_handler":{ + "class":"logging.handlers.RotatingFileHandler", + "level":"ERROR", + "formatter":"simple", + "filename":"errors.log", + "maxBytes":10485760, + "backupCount":20, + "encoding":"utf8" + } + }, + "loggers":{ + "my_module":{ + "level":"ERROR", + "handlers":["info_file_handler"], + "propagate":"no" + } + }, + "root":{ + "level":"INFO", + "handlers":["console","info_file_handler","error_file_handler"] + } +}''' diff --git a/images/.gitignore b/images/.gitignore new file mode 100644 index 0000000..f399bde --- /dev/null +++ b/images/.gitignore @@ -0,0 +1,2 @@ +*.png +*.jpg \ No newline at end of file diff --git a/logger.py b/logger.py new file mode 100644 index 0000000..a9e0938 --- /dev/null +++ b/logger.py @@ -0,0 +1,21 @@ +import logging, logging.config +import json, config + +class Logger(object): + + @staticmethod + def getLogger(name): + conf = json.loads(config.LoggerJsonConfig) + logging.config.dictConfig(conf) + return logging.getLogger(name) + +if __name__ == '__main__': + l = Logger.getLogger("test") + l.info("aaa") + l = Logger.getLogger("testa") + l.info("aaa") + l = Logger.getLogger("test.a") + l.info("aaa") + l.debug("ddd") + l.warning("w") + l.error('e') \ No newline at end of file diff --git a/push.py b/push.py new file mode 100644 index 0000000..4bb0efb --- /dev/null +++ b/push.py @@ -0,0 +1,19 @@ +from dingtalkchatbot.chatbot import DingtalkChatbot + + +class Push(object): + + def __init__(self, token): + self.d = DingtalkChatbot( + 'https://oapi.dingtalk.com/robot/send?access_token=%s' % token) + + def sendImage(self, title, url, is_at_all=False): + self.d.send_markdown(title=title, text='Giao, Giao, Giao!\n![Giao](' + url + ')\n', is_at_all=is_at_all) + + def sendMessage(self, msg, is_at_all=False): + self.d.send_text(msg=msg, is_at_all=is_at_all) + + +if __name__ == '__main__': + push = Push("a") + push.sendImage("a", "b") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5f915d7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +torch>=0.4.1 +torchvision>=0.2.1 +DingtalkChatbot>=1.3.0 +opencv-python>=3.4.2 \ No newline at end of file diff --git a/storage.py b/storage.py new file mode 100644 index 0000000..ec9d847 --- /dev/null +++ b/storage.py @@ -0,0 +1,19 @@ +from sinastorage.bucket import SCSBucket +import sinastorage +import time + + +class Storage(object): + + def __init__(self, accessKey, secretKey, bucketName, bucketUrl): + sinastorage.setDefaultAppInfo(accessKey, secretKey) + self.s = SCSBucket(bucketName) + self.bucketUrl = bucketUrl + + def saveImage(self, path): + name = str(time.strftime('%Y%m%d/%H:%M:%S.jpg')) + self.s.putFile(name, path, acl='public-read') + return self.bucketUrl % name + +# def putFileCallback(size, progress): +# print(size, progress) diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..65c0dcf --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +test.png \ No newline at end of file diff --git a/test/cameraTest.py b/test/cameraTest.py new file mode 100644 index 0000000..3779f5b --- /dev/null +++ b/test/cameraTest.py @@ -0,0 +1,23 @@ +import cv2 +import time + +cap = cv2.VideoCapture(0) + +cv2.namedWindow("Image") +while(cap.isOpened()): + ret,img = cap.read() + if ret == True: + cv2.imshow('Image',img) + k = cv2.waitKey(100) + if k == ord('a') or k == ord('A'): + + cv2.imwrite(str(time.strftime('%Y%m%d_%H_%M_%S.png')),img) + break +cv2.waitKey(0) +# if cap.isOpened(): +# time.sleep(15) +# ret,img = cap.read() +# if ret == True: +# cv2.imwrite('test2.png',img) + +cap.release() diff --git a/test/dingTalkChatbotTest.py b/test/dingTalkChatbotTest.py new file mode 100644 index 0000000..628530d --- /dev/null +++ b/test/dingTalkChatbotTest.py @@ -0,0 +1,12 @@ +from dingtalkchatbot.chatbot import DingtalkChatbot +import sys +sys.path.append("../") +import config + +webhook = 'https://oapi.dingtalk.com/robot/send?access_token=' + config.dingTalkWebhookAccessToken +xiaoding = DingtalkChatbot(webhook) +# Text MSG and at all +# xiaoding.send_text(msg='Giao!看我头像') + +xiaoding.send_markdown(title='x', text='![Giao](http://xxx/20191205/11:08:13.jpg)\n', + is_at_all=True) \ No newline at end of file diff --git a/test/selfie.jpg b/test/selfie.jpg new file mode 100644 index 0000000..1a1da0c Binary files /dev/null and b/test/selfie.jpg differ diff --git a/test/sinaSCSTest.py b/test/sinaSCSTest.py new file mode 100644 index 0000000..48c2d8b --- /dev/null +++ b/test/sinaSCSTest.py @@ -0,0 +1,19 @@ +from sinastorage.bucket import SCSBucket +import sinastorage +import sys +sys.path.append("../") +import config +import time + +uploadedAmount = 0.0 +def putFileCallback(size, progress): + print(size, progress) + +sinastorage.setDefaultAppInfo(config.SinaSCSAccessKey, config.SinaSCSSecretKey) + +s = SCSBucket(config.SinaSCSBucketName) +# s.putFile('1.jpg', 'selfie.jpg', putFileCallback) +ret = s.putFile(str(time.strftime('%Y%m%d/%H:%M:%S')) + '.jpg', 'selfie.jpg', acl='public-read') +print('complete!') +print(ret.info()) + diff --git a/test/tinyFaceTest.py b/test/tinyFaceTest.py new file mode 100644 index 0000000..5ace9d9 --- /dev/null +++ b/test/tinyFaceTest.py @@ -0,0 +1,24 @@ +import sys +sys.path.append("../") +import time, cv2 +from tinyface import TinyFace + +# load image with cv in RGB. +# IMAGE_PATH = 'selfie.jpg' +IMAGE_PATH = 'selfie.jpg' +img = cv2.imread(IMAGE_PATH) +img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + +# load detectors. +DET3 = TinyFace('../checkpoint_50.pth', device='cpu') + +# Tiny Face returns bboxes. +t = time.time() +bboxes = DET3.detect_faces(img, conf_th=0.9, scales=[1]) +print('Tiny Face : %d faces in %.4f seconds.' % (len(bboxes), time.time() - t)) +print("bboxes: ", bboxes) +sizes = [] +for box in bboxes: + sizes.append((box[2] - box[0]) * (box[3] - box[1])) +print(min(sizes)) +print(max(sizes)) \ No newline at end of file diff --git a/tinyface/.gitignore b/tinyface/.gitignore new file mode 100644 index 0000000..b331dac --- /dev/null +++ b/tinyface/.gitignore @@ -0,0 +1 @@ +*.pth \ No newline at end of file diff --git a/tinyface/__init__.py b/tinyface/__init__.py new file mode 100644 index 0000000..97f4411 --- /dev/null +++ b/tinyface/__init__.py @@ -0,0 +1,130 @@ +""" +Finding Tiny Faces +""" + +import time +from collections import OrderedDict +import cv2 +import numpy as np +import torch +from torchvision import transforms +from .nets import TFNet +from .box_utils import nms, templates +from logger import Logger + +log = Logger.getLogger("TinyFace") + +class TinyFace(): + + def __init__(self, model_path, device='cuda'): + tstamp = time.time() + self.device = device + + log.info('[Tiny Face] loading with %s', self.device) + self.net = TFNet().to(device) + + state_dict = torch.load(model_path, map_location=self.device)['model'] + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + name = k[7:] + new_state_dict[name] = v + self.net.load_state_dict(new_state_dict) + + self.net.eval() + log.info('[Tiny Face] finished loading (%.4f sec)' % (time.time() - tstamp)) + + + def detect_faces(self, image, conf_th=0.8, scales=[1]): + """ + input + image: cv image in RGB. + conf_th: confidence threshold. + """ + + # image pyramid + # scales = [0.25, 0.5, 1, 2] + + trans_norm = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + ]) + + # initialize output + bboxes = np.empty(shape=(0, 5)) + + tstamp = time.time() + + with torch.no_grad(): + for s in scales: + + # input image + scaled_img = cv2.resize(image, dsize=(0, 0), fx=s, fy=s, interpolation=cv2.INTER_LINEAR) + img = trans_norm(scaled_img) + img.unsqueeze_(0) + + # run + x = img.float().to(self.device) + y = self.net(x) + + # collect scores + score_cls = y[:, :25, :, :] + prob_cls = torch.sigmoid(score_cls) + score_reg = y[:, 25:125, :, :] + score_cls = score_cls.data.cpu().numpy().transpose((0, 2, 3, 1)) + prob_cls = prob_cls.data.cpu().numpy().transpose((0, 2, 3, 1)) + score_reg = score_reg.data.cpu().numpy().transpose((0, 2, 3, 1)) + + # ignore templates by scale + tids = list(range(4, 12)) + ([] if s <= 1.0 else list(range(18, 25))) + ignored_tids = list(set(range(0, 25)) - set(tids)) + try: + prob_cls[:, :, ignored_tids] = 0.0 + except IndexError: + pass + + # threshold for detection + indices = np.where(prob_cls > conf_th) + fb, fy, fx, fc = indices + scores = prob_cls[fb, fy, fx, fc] + scores = scores.reshape((scores.shape[0], 1)) + + # interpret heatmap into bounding boxes + cx = fx * 8 - 1 + cy = fy * 8 - 1 + cw = templates[fc, 2] - templates[fc, 0] + 1 + ch = templates[fc, 3] - templates[fc, 1] + 1 + + # extract bounding box refinement + tx = score_reg[0, :, :, 0:25] + ty = score_reg[0, :, :, 25:50] + tw = score_reg[0, :, :, 50:75] + th = score_reg[0, :, :, 75:100] + + # refine bounding boxes + dcx = cw * tx[fy, fx, fc] + dcy = ch * ty[fy, fx, fc] + rcx = cx + dcx + rcy = cy + dcy + rcw = cw * np.exp(tw[fy, fx, fc]) + rch = ch * np.exp(th[fy, fx, fc]) + + # create bbox array + rcx = rcx.reshape((rcx.shape[0], 1)) + rcy = rcy.reshape((rcy.shape[0], 1)) + rcw = rcw.reshape((rcw.shape[0], 1)) + rch = rch.reshape((rch.shape[0], 1)) + tmp_bboxes = np.array([rcx - rcw / 2, rcy - rch / 2, rcx + rcw / 2, rcy + rch / 2]) + tmp_bboxes = tmp_bboxes * (1 / s) + + # bboxes with confidences + for i in range(tmp_bboxes.shape[1]): + bbox = (tmp_bboxes[0][i][0], tmp_bboxes[1][i][0], tmp_bboxes[2][i][0], tmp_bboxes[3][i][0], scores[i][0]) + bboxes = np.vstack((bboxes, bbox)) + + # nms + keep = nms(bboxes, 0.1) + bboxes = bboxes[keep] + + log.info('detect faces using %.6f sec' % (time.time() - tstamp)) + + return bboxes diff --git a/tinyface/box_utils.py b/tinyface/box_utils.py new file mode 100644 index 0000000..ef0c1b1 --- /dev/null +++ b/tinyface/box_utils.py @@ -0,0 +1,64 @@ +import numpy as np + + +templates = np.array([ + [-86.3382352941177, -113.444117647059, 86.3382352941177, 113.444117647059, 0.500000000000000], + [-48.7500000000000, -65.2500000000000, 48.7500000000000, 65.2500000000000, 0.500000000000000], + [-33.2500000000000, -43.7500000000000, 33.2500000000000, 43.7500000000000, 0.500000000000000], + [-25.7500000000000, -33.7500000000000, 25.7500000000000, 33.7500000000000, 0.500000000000000], + [-40.5000000000000, -54.5000000000000, 40.5000000000000, 54.5000000000000, 1], + [-34.5000000000000, -43.5000000000000, 34.5000000000000, 43.5000000000000, 1], + [-28.5000000000000, -38, 28.5000000000000, 38, 1], + [-25.6589050000000, -31.3221500000000, 25.6589050000000, 31.3221500000000, 1], + [-21.6137000000000, -27.5976700000000, 21.6137000000000, 27.5976700000000, 1], + [-20, -22.5000000000000, 20, 22.5000000000000, 1], + [-17.5000000000000, -25.5000000000000, 17.5000000000000, 25.5000000000000, 1], + [-16.3279854000000, -20.8855500000000, 16.3279854000000, 20.8855500000000, 1], + [-29.4755000000000, -34.4803000000000, 29.4755000000000, 34.4803000000000, 2], + [-25.4202000000000, -37.1060900000000, 25.4202000000000, 37.1060900000000, 2], + [-24.2118000000000, -30.2145600000000, 24.2118000000000, 30.2145600000000, 2], + [-22.0129000000000, -24.7059000000000, 22.0129000000000, 24.7059000000000, 2], + [-19.3142000000000, -28.0202000000000, 19.3142000000000, 28.0202000000000, 2], + [-17.9677000000000, -22.7849000000000, 17.9677000000000, 22.7849000000000, 2], + [-15.6123000000000, -19.5907000000000, 15.6123000000000, 19.5907000000000, 2], + [-13.1842000000000, -18.3421000000000, 13.1842000000000, 18.3421000000000, 2], + [-13.5027700000000, -15.0792000000000, 13.5027700000000, 15.0792000000000, 2], + [-11.1091000000000, -16.4909000000000, 11.1091000000000, 16.4909000000000, 2], + [-10.9768600000000, -14.0478000000000, 10.9768600000000, 14.0478000000000, 2], + [-9.97219999999993, -12.4105000000000, 9.97219999999993, 12.4105000000000, 2], + [-9.66129999999998, -10.6161000000000, 9.66129999999998, 10.6161000000000, 2] +]) + + +def nms(dets, thresh): + """ + Courtesy of Ross Girshick + [https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/nms/py_cpu_nms.py] + """ + x1 = dets[:, 0] + y1 = dets[:, 1] + x2 = dets[:, 2] + y2 = dets[:, 3] + scores = dets[:, 4] + + areas = (x2 - x1) * (y2 - y1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(int(i)) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1) + h = np.maximum(0.0, yy2 - yy1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= thresh)[0] + order = order[inds + 1] + + return np.array(keep).astype(np.int) diff --git a/tinyface/nets.py b/tinyface/nets.py new file mode 100644 index 0000000..5b73c19 --- /dev/null +++ b/tinyface/nets.py @@ -0,0 +1,67 @@ +import torch +import torch.nn as nn +import numpy as np +from torchvision.models import resnet101 + + +class TFNet(nn.Module): + + def __init__(self): + + super(TFNet, self).__init__() + + self.model = resnet101(pretrained=True) + del self.model.layer4 + + self.score_res3 = nn.Conv2d(512, 125, 1, 1) + self.score_res4 = nn.Conv2d(1024, 125, 1, 1) + + self.score4_upsample = nn.ConvTranspose2d(125, 125, 4, 2, padding=1, bias=False) + + self._init_bilinear() + + def _init_bilinear(self): + k_size = self.score4_upsample.kernel_size[0] + ch_in = self.score4_upsample.in_channels + ch_out = self.score4_upsample.out_channels + + factor = np.floor((k_size + 1) / 2) + if k_size % 2 == 1: + center = factor + else: + center = factor + 0.5 + + f = np.zeros((ch_in, ch_out, k_size, k_size)) + C = np.array([1, 2, 3, 4]) + + temp = (np.ones((1, k_size)) - (np.abs(C - center) / factor)) + for i in range(ch_out): + f[i, i, :, :] = temp.T @ temp + + self.score4_upsample.weight = torch.nn.Parameter(data=torch.Tensor(f)) + + def forward(self, x): + x = self.model.conv1(x) + x = self.model.bn1(x) + x = self.model.relu(x) + x = self.model.maxpool(x) + x = self.model.layer1(x) + x = self.model.layer2(x) + score_res3 = self.score_res3(x) + x = self.model.layer3(x) + score_res4 = self.score_res4(x) + score4 = self.score4_upsample(score_res4) + + cropv = score4.size(2) - score_res3.size(2) + cropu = score4.size(3) - score_res3.size(3) + + if cropv == 0: + cropv = -score4.size(2) + if cropu == 0: + cropu = -score4.size(3) + + score4 = score4[:, :, 0:-cropv, 0:-cropu] + + score = score_res3 + score4 + + return score