Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

删掉自己写的protobuf；extrabuf提速 #270

Merged
merged 7 commits into from
Jan 5, 2024
Merged

删掉自己写的protobuf；extrabuf提速 #270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ca491c3
去除parseBytes中的字段名称，返回结果更简洁；把对protobuf的依赖去除
STDquantum Jan 5, 2024
4416fea
Revert "去除parseBytes中的字段名称，返回结果更简洁；把对protobuf的依赖去除"
STDquantum Jan 5, 2024
f6ef6a8
删掉自己写的protobuf；extrabuf提速
STDquantum Jan 5, 2024
2ae054b
Merge branch 'LC044:master' into master
STDquantum Jan 5, 2024
ce3e130
似乎现在并不需要extrabuf那个
STDquantum Jan 5, 2024
7406f71
删掉
STDquantum Jan 5, 2024
51b5b63
.avatar user-select:none
STDquantum Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
262 changes: 45 additions & 217 deletions app/DataBase/hard_link.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import xml.etree.ElementTree as ET

from app.log import log
from app.util.protocbuf.msg_pb2 import MessageBytesExtra

image_db_lock = threading.Lock()
video_db_lock = threading.Lock()
Expand All @@ -30,224 +31,47 @@ def get_md5_from_xml(content, type_="img"):
return None


class tencent_struct:
def __setVals__(self, data, off):
if data:
self.__data = data
if self.__data:
self.__size = len(self.__data)
self.__off = off

def __readString(self):
try:
length = self.__readUleb()
res = self.__data[self.__off: self.__off + length]
self.__add(length)
except:
raise
return res.decode("utf-8")

def __readUleb(self):
try:
i = self.__data[self.__off]
self.__add()
if i & 0x80:
j = self.__data[self.__off]
i = i & 0x7F
i = i | (j << 7)
self.__add()
if i & 0x4000:
j = self.__data[self.__off]
i = i & 0x3FFF
i = i | (j << 14)
self.__add()
if i & 0x200000:
j = self.__data[self.__off]
i = i & 0x1FFFFF
i = i | (j << 21)
self.__add()
if i & 0x10000000:
j = self.__data[self.__off]
i = i & 0xFFFFFFF
i = i | (j << 28)
self.__add()
return i
except:
raise

def __readData(self):
try:
length = self.__readUleb()
data = self.__data[self.__off: self.__off + length]
self.__add(length)
return data
except:
raise

def __init__(self, data=None, off=0):
self.__data = data
self.__off = off
if self.__data:
self.__size = len(self.__data)
else:
self.__size = 0

def __add(self, value=1):
self.__off += value
if self.__off > self.__size:
raise "偏移量超出size"

def readStruct(self, struct_type):
current_dict = None
if isinstance(struct_type, str):
current_dict = getattr(self, struct_type)
else:
current_dict = struct_type
res = {}
try:
while self.__off < self.__size:
key = self.__readUleb()
key = key >> 3
if key == 0:
break
op = None
fieldName = ""
if key in current_dict:
op = current_dict[key][1]
fieldName = current_dict[key][0]
else:
break
if isinstance(op, dict):
if not key in res:
res[key] = []
current_struct = self.__readData()
recursion = tencent_struct(current_struct)
res[key].append((fieldName, recursion.readStruct(op)))
elif op != "":
res[key] = (fieldName, self.__contenttype__[op](self))
else:
break
except:
raise
return res

__struct1__ = {1: ("", "I"), 2: ("", "I")}

__msgInfo__ = {1: ("", "I"), 2: ("msg_info", "s")}

__bytesExtra__ = {
1: ("", __struct1__),
3: ("msg_info_struct", __msgInfo__),
}

__struct2__ = {1: ("", "s"), 2: ("", "s")}

__extraBuf__ = {
1: ("", __struct2__),
}

def get_bytesExta_Content(self, data=None, off=0):
self.__setVals__(data, off)
try:
return self.readStruct("__bytesExtra__")
except:
raise

def get_extraBuf_Content(self, data=None, off=0):
self.__setVals__(data, off)
try:
return self.readStruct("__extraBuf__")
except:
raise

__contenttype__ = {
"s": __readString,
"I": __readUleb,
"P": __readData,
}


def parseBytes(content: bytes):
try:
bytesExtra = tencent_struct().get_bytesExta_Content(content)
return bytesExtra
except:
pass


def parseExtraBuf(content: bytes):
try:
extraBuf = tencent_struct().get_extraBuf_Content(content)
return extraBuf
except:
pass


def decodeExtraBuf(extra_buf_content: bytes):
off = 0
types = [b"\x04", b"\x18", b"\x17", b"\x02", b"\x05"]
trunkName = {
"46CF10C4": "个性签名",
"A4D9024A": "国家",
"E2EAA8D1": "省份",
"1D025BBF": "市",
"81AE19B4": "朋友圈背景url",
"F917BCC0": "公司名称",
"4EB96D85": "企业微信属性",
"0E719F13": "备注图片",
"759378AD": "手机号",
"74752C06": "性别",
b"\x46\xCF\x10\xC4": "个性签名",
b"\xA4\xD9\x02\x4A": "国家",
b"\xE2\xEA\xA8\xD1": "省份",
b"\x1D\x02\x5B\xBF": "市",
# b"\x81\xAE\x19\xB4": "朋友圈背景url",
# b"\xF9\x17\xBC\xC0": "公司名称",
# b"\x4E\xB9\x6D\x85": "企业微信属性",
# b"\x0E\x71\x9F\x13": "备注图片",
b"\x75\x93\x78\xAD": "手机号",
b"\x74\x75\x2C\x06": "性别",
}
res = {'手机号': {'18': ''}}
while off < len(extra_buf_content):
length = 4 # 块头
trunk_head = extra_buf_content[off: off + length]
off += length
trunk_head = binascii.hexlify(trunk_head).decode().upper()
if trunk_head in trunkName:
trunk_head = trunkName[trunk_head]
res[trunk_head] = {}
char = extra_buf_content[off: off + 1]
res = {"手机号": ""}
off = 0
for key in trunkName:
trunk_head = trunkName[key]
try:
off = extra_buf_content.index(key) + 4
except:
pass
char = extra_buf_content[off : off + 1]
off += 1
field = binascii.hexlify(char).decode()
if char == b"\x04": # 四个字节的int，小端序
length = 4
intContent = extra_buf_content[off: off + length]
intContent = extra_buf_content[off : off + 4]
off += 4
intContent = int.from_bytes(intContent, "little")
res[trunk_head][field] = intContent
res[trunk_head] = intContent
elif char == b"\x18": # utf-16字符串
length = 4
lengthContent = extra_buf_content[off: off + length]
off += 4
lengthContent = int.from_bytes(lengthContent, "little")
strContent = extra_buf_content[off: off + lengthContent]
off += lengthContent
res[trunk_head][field] = strContent.decode("utf-16").rstrip("\x00")
elif char == b"\x17": # utf-8 protobuf
length = 4
lengthContent = extra_buf_content[off: off + length]
lengthContent = extra_buf_content[off : off + 4]
off += 4
lengthContent = int.from_bytes(lengthContent, "little")
strContent = extra_buf_content[off: off + lengthContent]
strContent = extra_buf_content[off : off + lengthContent]
off += lengthContent
res[trunk_head][field] = parseExtraBuf(strContent)
elif char == b"\x02": # 一个字节的int
content = extra_buf_content[off: off + 1]
off += 1
res[trunk_head][field] = int.from_bytes(content, "little")
elif char == b"\x05": # 暂时不知道有啥用，固定8个字节，先当int处理
length = 8
content = extra_buf_content[off: off + length]
off += length
res[trunk_head][field] = int.from_bytes(content, "little")
# print(res)
res[trunk_head] = strContent.decode("utf-16").rstrip("\x00")

return {
'region': (res['国家']['18'], res['省份']['18'], res['市']['18']),
'signature': res['个性签名']['18'],
'telephone': res['手机号']['18'],
'gender': res['性别']['04']
"region": (res["国家"], res["省份"], res["市"]),
"signature": res["个性签名"],
"telephone": res["手机号"],
"gender": res["性别"],
}


Expand Down Expand Up @@ -337,12 +161,14 @@ def get_video_by_md5(self, md5: bytes):
video_db_lock.release()

def get_image(self, content, bytesExtra, thumb=False):
bytesDict = parseBytes(bytesExtra)
for msginfo in bytesDict[3]:
if msginfo[1][1][1] == (3 if thumb else 4):
pathh = msginfo[1][2][1] # wxid\FileStorage\...
pathh = "\\".join(pathh.split("\\")[1:])
return pathh
msg_bytes = MessageBytesExtra()
msg_bytes.ParseFromString(bytesExtra)
for tmp in msg_bytes.message2:
if tmp.field1 != (3 if thumb else 4):
continue
pathh = tmp.field2 # wxid\FileStorage\...
pathh = "\\".join(pathh.split("\\")[1:])
return pathh
md5 = get_md5_from_xml(content)
if not md5:
return None
Expand All @@ -357,12 +183,14 @@ def get_image(self, content, bytesExtra, thumb=False):
return dat_image

def get_video(self, content, bytesExtra, thumb=False):
bytesDict = parseBytes(bytesExtra)
for msginfo in bytesDict[3]:
if msginfo[1][1][1] == (3 if thumb else 4):
pathh = msginfo[1][2][1] # wxid\FileStorage\...
pathh = "\\".join(pathh.split("\\")[1:])
return pathh
msg_bytes = MessageBytesExtra()
msg_bytes.ParseFromString(bytesExtra)
for tmp in msg_bytes.message2:
if tmp.field1 != (3 if thumb else 4):
continue
pathh = tmp.field2 # wxid\FileStorage\...
pathh = "\\".join(pathh.split("\\")[1:])
return pathh
md5 = get_md5_from_xml(content, type_="video")
if not md5:
return None
Expand Down
15 changes: 7 additions & 8 deletions app/DataBase/msg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import threading
import traceback

from app.DataBase.hard_link import parseBytes
from app.log import logger
from app.util.compress_content import parser_reply
from app.util.protocbuf.msg_pb2 import MessageBytesExtra
Expand Down Expand Up @@ -651,12 +650,12 @@ def __del__(self):
else:
show_display_name = appinfo.find('appname').text
print(title, des, url, show_display_name)
bytesDict = parseBytes(msg[10])
for msginfo in bytesDict[3]:
print(msginfo)
if msginfo[1][1][1] == 3:
thumb = msginfo[1][2][1]
msg_bytes = MessageBytesExtra()
msg_bytes.ParseFromString(msg[10])
for tmp in msg_bytes.message2:
if tmp.field1 == 3:
thumb = tmp.field2
print(thumb)
if msginfo[1][1][1] == 4:
app_logo = msginfo[1][2][1]
if tmp.field2 == 4:
app_logo = tmp.field2
print('logo',app_logo)
11 changes: 9 additions & 2 deletions app/resources/data/template.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@
width: 42px;
height: 42px;
border-radius: 50%;
user-select: none;
}
.chat-video video{
margin-right: 18px;
Expand Down Expand Up @@ -465,6 +466,12 @@
margin-left: 5px;
}
}

.system-msg>.emoji_img {
width: 18px;
height: 18px;
}

.emoji_img {
width: 22px;
height: 22px;
Expand Down Expand Up @@ -802,7 +809,7 @@
// 从数据列表中取出对应范围的元素并添加到容器中
for (let i = startIndex; i < endIndex && i < chatMessages.length; i++) {
const message = chatMessages[i];
if (i == startIndex) { // 判断一下在页面顶部多加一个时间
if (i == startIndex && (reachedBottom ? !/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(chatMessages[i - 1].text) : 1)) { // 判断一下在页面顶部多加一个时间
if (!/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(message.text)) {
// 时间戳转成时间
function timestampToTime(timestamp) {
Expand Down Expand Up @@ -843,7 +850,7 @@
}
else if (message.type == 0) {
messageElement.className = "item item-center";
messageElement.innerHTML = `<span>${message.text}</span>`;
messageElement.innerHTML = `<span class="system-msg">${replaceEmoji(message.text)}</span>`;
}
else if (message.type == 3) {
// displayname 和 img
Expand Down
Loading