From f6ef6a8357d805fa2418ed330470a464136ca04e Mon Sep 17 00:00:00 2001
From: STDquantum <405720329@qq.com>
Date: Fri, 5 Jan 2024 20:06:23 +0800
Subject: [PATCH] =?UTF-8?q?=E5=88=A0=E6=8E=89=E8=87=AA=E5=B7=B1=E5=86=99?=
=?UTF-8?q?=E7=9A=84protobuf=EF=BC=9Bextrabuf=E6=8F=90=E9=80=9F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
app/DataBase/hard_link.py | 263 +++++------------------------
app/DataBase/msg.py | 15 +-
app/resources/data/template.html | 10 +-
app/util/compress_content.py | 15 +-
app/util/protocbuf/extrabuf.proto | 11 ++
app/util/protocbuf/extrabuf_pb2.py | 29 ++++
6 files changed, 109 insertions(+), 234 deletions(-)
create mode 100644 app/util/protocbuf/extrabuf.proto
create mode 100644 app/util/protocbuf/extrabuf_pb2.py
diff --git a/app/DataBase/hard_link.py b/app/DataBase/hard_link.py
index 22e93f8..a7bf1b9 100644
--- a/app/DataBase/hard_link.py
+++ b/app/DataBase/hard_link.py
@@ -5,6 +5,8 @@ import threading
import xml.etree.ElementTree as ET
from app.log import log
+from app.util.protocbuf.extrabuf_pb2 import ExtraBuf
+from app.util.protocbuf.msg_pb2 import MessageBytesExtra
image_db_lock = threading.Lock()
video_db_lock = threading.Lock()
@@ -30,224 +32,47 @@ def get_md5_from_xml(content, type_="img"):
return None
-class tencent_struct:
- def __setVals__(self, data, off):
- if data:
- self.__data = data
- if self.__data:
- self.__size = len(self.__data)
- self.__off = off
-
- def __readString(self):
- try:
- length = self.__readUleb()
- res = self.__data[self.__off: self.__off + length]
- self.__add(length)
- except:
- raise
- return res.decode("utf-8")
-
- def __readUleb(self):
- try:
- i = self.__data[self.__off]
- self.__add()
- if i & 0x80:
- j = self.__data[self.__off]
- i = i & 0x7F
- i = i | (j << 7)
- self.__add()
- if i & 0x4000:
- j = self.__data[self.__off]
- i = i & 0x3FFF
- i = i | (j << 14)
- self.__add()
- if i & 0x200000:
- j = self.__data[self.__off]
- i = i & 0x1FFFFF
- i = i | (j << 21)
- self.__add()
- if i & 0x10000000:
- j = self.__data[self.__off]
- i = i & 0xFFFFFFF
- i = i | (j << 28)
- self.__add()
- return i
- except:
- raise
-
- def __readData(self):
- try:
- length = self.__readUleb()
- data = self.__data[self.__off: self.__off + length]
- self.__add(length)
- return data
- except:
- raise
-
- def __init__(self, data=None, off=0):
- self.__data = data
- self.__off = off
- if self.__data:
- self.__size = len(self.__data)
- else:
- self.__size = 0
-
- def __add(self, value=1):
- self.__off += value
- if self.__off > self.__size:
- raise "偏移量超出size"
-
- def readStruct(self, struct_type):
- current_dict = None
- if isinstance(struct_type, str):
- current_dict = getattr(self, struct_type)
- else:
- current_dict = struct_type
- res = {}
- try:
- while self.__off < self.__size:
- key = self.__readUleb()
- key = key >> 3
- if key == 0:
- break
- op = None
- fieldName = ""
- if key in current_dict:
- op = current_dict[key][1]
- fieldName = current_dict[key][0]
- else:
- break
- if isinstance(op, dict):
- if not key in res:
- res[key] = []
- current_struct = self.__readData()
- recursion = tencent_struct(current_struct)
- res[key].append((fieldName, recursion.readStruct(op)))
- elif op != "":
- res[key] = (fieldName, self.__contenttype__[op](self))
- else:
- break
- except:
- raise
- return res
-
- __struct1__ = {1: ("", "I"), 2: ("", "I")}
-
- __msgInfo__ = {1: ("", "I"), 2: ("msg_info", "s")}
-
- __bytesExtra__ = {
- 1: ("", __struct1__),
- 3: ("msg_info_struct", __msgInfo__),
- }
-
- __struct2__ = {1: ("", "s"), 2: ("", "s")}
-
- __extraBuf__ = {
- 1: ("", __struct2__),
- }
-
- def get_bytesExta_Content(self, data=None, off=0):
- self.__setVals__(data, off)
- try:
- return self.readStruct("__bytesExtra__")
- except:
- raise
-
- def get_extraBuf_Content(self, data=None, off=0):
- self.__setVals__(data, off)
- try:
- return self.readStruct("__extraBuf__")
- except:
- raise
-
- __contenttype__ = {
- "s": __readString,
- "I": __readUleb,
- "P": __readData,
- }
-
-
-def parseBytes(content: bytes):
- try:
- bytesExtra = tencent_struct().get_bytesExta_Content(content)
- return bytesExtra
- except:
- pass
-
-
-def parseExtraBuf(content: bytes):
- try:
- extraBuf = tencent_struct().get_extraBuf_Content(content)
- return extraBuf
- except:
- pass
-
-
def decodeExtraBuf(extra_buf_content: bytes):
- off = 0
- types = [b"\x04", b"\x18", b"\x17", b"\x02", b"\x05"]
trunkName = {
- "46CF10C4": "个性签名",
- "A4D9024A": "国家",
- "E2EAA8D1": "省份",
- "1D025BBF": "市",
- "81AE19B4": "朋友圈背景url",
- "F917BCC0": "公司名称",
- "4EB96D85": "企业微信属性",
- "0E719F13": "备注图片",
- "759378AD": "手机号",
- "74752C06": "性别",
+ b"\x46\xCF\x10\xC4": "个性签名",
+ b"\xA4\xD9\x02\x4A": "国家",
+ b"\xE2\xEA\xA8\xD1": "省份",
+ b"\x1D\x02\x5B\xBF": "市",
+ # b"\x81\xAE\x19\xB4": "朋友圈背景url",
+ # b"\xF9\x17\xBC\xC0": "公司名称",
+ # b"\x4E\xB9\x6D\x85": "企业微信属性",
+ # b"\x0E\x71\x9F\x13": "备注图片",
+ b"\x75\x93\x78\xAD": "手机号",
+ b"\x74\x75\x2C\x06": "性别",
}
- res = {'手机号': {'18': ''}}
- while off < len(extra_buf_content):
- length = 4 # 块头
- trunk_head = extra_buf_content[off: off + length]
- off += length
- trunk_head = binascii.hexlify(trunk_head).decode().upper()
- if trunk_head in trunkName:
- trunk_head = trunkName[trunk_head]
- res[trunk_head] = {}
- char = extra_buf_content[off: off + 1]
+ res = {"手机号": ""}
+ off = 0
+ for key in trunkName:
+ trunk_head = trunkName[key]
+ try:
+ off = extra_buf_content.index(key) + 4
+ except:
+ pass
+ char = extra_buf_content[off : off + 1]
off += 1
- field = binascii.hexlify(char).decode()
if char == b"\x04": # 四个字节的int，小端序
- length = 4
- intContent = extra_buf_content[off: off + length]
+ intContent = extra_buf_content[off : off + 4]
off += 4
intContent = int.from_bytes(intContent, "little")
- res[trunk_head][field] = intContent
+ res[trunk_head] = intContent
elif char == b"\x18": # utf-16字符串
- length = 4
- lengthContent = extra_buf_content[off: off + length]
+ lengthContent = extra_buf_content[off : off + 4]
off += 4
lengthContent = int.from_bytes(lengthContent, "little")
- strContent = extra_buf_content[off: off + lengthContent]
+ strContent = extra_buf_content[off : off + lengthContent]
off += lengthContent
- res[trunk_head][field] = strContent.decode("utf-16").rstrip("\x00")
- elif char == b"\x17": # utf-8 protobuf
- length = 4
- lengthContent = extra_buf_content[off: off + length]
- off += 4
- lengthContent = int.from_bytes(lengthContent, "little")
- strContent = extra_buf_content[off: off + lengthContent]
- off += lengthContent
- res[trunk_head][field] = parseExtraBuf(strContent)
- elif char == b"\x02": # 一个字节的int
- content = extra_buf_content[off: off + 1]
- off += 1
- res[trunk_head][field] = int.from_bytes(content, "little")
- elif char == b"\x05": # 暂时不知道有啥用，固定8个字节，先当int处理
- length = 8
- content = extra_buf_content[off: off + length]
- off += length
- res[trunk_head][field] = int.from_bytes(content, "little")
- # print(res)
+ res[trunk_head] = strContent.decode("utf-16").rstrip("\x00")
return {
- 'region': (res['国家']['18'], res['省份']['18'], res['市']['18']),
- 'signature': res['个性签名']['18'],
- 'telephone': res['手机号']['18'],
- 'gender': res['性别']['04']
+ "region": (res["国家"], res["省份"], res["市"]),
+ "signature": res["个性签名"],
+ "telephone": res["手机号"],
+ "gender": res["性别"],
}
@@ -337,12 +162,14 @@ class HardLink:
video_db_lock.release()
def get_image(self, content, bytesExtra, thumb=False):
- bytesDict = parseBytes(bytesExtra)
- for msginfo in bytesDict[3]:
- if msginfo[1][1][1] == (3 if thumb else 4):
- pathh = msginfo[1][2][1] # wxid\FileStorage\...
- pathh = "\\".join(pathh.split("\\")[1:])
- return pathh
+ msg_bytes = MessageBytesExtra()
+ msg_bytes.ParseFromString(bytesExtra)
+ for tmp in msg_bytes.message2:
+ if tmp.field1 != (3 if thumb else 4):
+ continue
+ pathh = tmp.field2 # wxid\FileStorage\...
+ pathh = "\\".join(pathh.split("\\")[1:])
+ return pathh
md5 = get_md5_from_xml(content)
if not md5:
return None
@@ -357,12 +184,14 @@ class HardLink:
return dat_image
def get_video(self, content, bytesExtra, thumb=False):
- bytesDict = parseBytes(bytesExtra)
- for msginfo in bytesDict[3]:
- if msginfo[1][1][1] == (3 if thumb else 4):
- pathh = msginfo[1][2][1] # wxid\FileStorage\...
- pathh = "\\".join(pathh.split("\\")[1:])
- return pathh
+ msg_bytes = MessageBytesExtra()
+ msg_bytes.ParseFromString(bytesExtra)
+ for tmp in msg_bytes.message2:
+ if tmp.field1 != (3 if thumb else 4):
+ continue
+ pathh = tmp.field2 # wxid\FileStorage\...
+ pathh = "\\".join(pathh.split("\\")[1:])
+ return pathh
md5 = get_md5_from_xml(content, type_="video")
if not md5:
return None
diff --git a/app/DataBase/msg.py b/app/DataBase/msg.py
index d1a7c44..4c67639 100644
--- a/app/DataBase/msg.py
+++ b/app/DataBase/msg.py
@@ -4,7 +4,6 @@ import sqlite3
import threading
import traceback
-from app.DataBase.hard_link import parseBytes
from app.log import logger
from app.util.compress_content import parser_reply
from app.util.protocbuf.msg_pb2 import MessageBytesExtra
@@ -651,12 +650,12 @@ if __name__ == '__main__':
else:
show_display_name = appinfo.find('appname').text
print(title, des, url, show_display_name)
- bytesDict = parseBytes(msg[10])
- for msginfo in bytesDict[3]:
- print(msginfo)
- if msginfo[1][1][1] == 3:
- thumb = msginfo[1][2][1]
+ msg_bytes = MessageBytesExtra()
+ msg_bytes.ParseFromString(msg[10])
+ for tmp in msg_bytes.message2:
+ if tmp.field1 == 3:
+ thumb = tmp.field2
print(thumb)
- if msginfo[1][1][1] == 4:
- app_logo = msginfo[1][2][1]
+ if tmp.field2 == 4:
+ app_logo = tmp.field2
print('logo',app_logo)
\ No newline at end of file
diff --git a/app/resources/data/template.html b/app/resources/data/template.html
index 939bb7a..92b55fd 100644
--- a/app/resources/data/template.html
+++ b/app/resources/data/template.html
@@ -465,6 +465,12 @@ input {
margin-left: 5px;
}
}
+
+.system-msg>.emoji_img {
+ width: 18px;
+ height: 18px;
+}
+
.emoji_img {
width: 22px;
height: 22px;
@@ -802,7 +808,7 @@ input {
// 从数据列表中取出对应范围的元素并添加到容器中
for (let i = startIndex; i < endIndex && i < chatMessages.length; i++) {
const message = chatMessages[i];
- if (i == startIndex) { // 判断一下在页面顶部多加一个时间
+ if (i == startIndex && (reachedBottom ? !/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(chatMessages[i - 1].text) : 1)) { // 判断一下在页面顶部多加一个时间
if (!/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(message.text)) {
// 时间戳转成时间
function timestampToTime(timestamp) {
@@ -843,7 +849,7 @@ input {
}
else if (message.type == 0) {
messageElement.className = "item item-center";
- messageElement.innerHTML = `${message.text}`;
+ messageElement.innerHTML = `${replaceEmoji(message.text)}`;
}
else if (message.type == 3) {
// displayname 和 img
diff --git a/app/util/compress_content.py b/app/util/compress_content.py
index f8397e4..bb6ab14 100644
--- a/app/util/compress_content.py
+++ b/app/util/compress_content.py
@@ -8,7 +8,7 @@ import re
from urllib.parse import urlparse
from bs4 import BeautifulSoup
-from app.DataBase.hard_link import parseBytes
+from app.util.protocbuf.msg_pb2 import MessageBytesExtra
from ..util.file import get_file
@@ -149,15 +149,16 @@ def share_card(bytesExtra, compress_content_):
else:
if appinfo is not None:
show_display_name = appinfo.find('appname').text
- bytesDict = parseBytes(bytesExtra)
+ msg_bytes = MessageBytesExtra()
+ msg_bytes.ParseFromString(bytesExtra)
app_logo = ''
thumbnail = ''
- for msginfo in bytesDict[3]:
- if msginfo[1][1][1] == 3:
- thumbnail = msginfo[1][2][1]
+ for tmp in msg_bytes.message2:
+ if tmp.field1 == 3:
+ thumbnail = tmp.field2
thumbnail = "\\".join(thumbnail.split('\\')[1:])
- if msginfo[1][1][1] == 4:
- app_logo = msginfo[1][2][1]
+ if tmp.field2 == 4:
+ app_logo = tmp.field2
app_logo = "\\".join(app_logo.split('\\')[1:])
if sourceusername is not None:
from app.DataBase import micro_msg_db # 放上面会导致循环依赖
diff --git a/app/util/protocbuf/extrabuf.proto b/app/util/protocbuf/extrabuf.proto
new file mode 100644
index 0000000..d912068
--- /dev/null
+++ b/app/util/protocbuf/extrabuf.proto
@@ -0,0 +1,11 @@
+syntax = "proto3";
+package app.protobuf;
+option go_package=".;proto";
+
+message ExtraBuf {
+ message Struct {
+ string field1 = 1;
+ string field2 = 2;
+ }
+ repeated Struct content = 1;
+}
\ No newline at end of file
diff --git a/app/util/protocbuf/extrabuf_pb2.py b/app/util/protocbuf/extrabuf_pb2.py
new file mode 100644
index 0000000..76a5afb
--- /dev/null
+++ b/app/util/protocbuf/extrabuf_pb2.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: extrabuf.proto
+# Protobuf Python Version: 4.25.0
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x65xtrabuf.proto\x12\x0c\x61pp.protobuf\"d\n\x08\x45xtraBuf\x12.\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x1d.app.protobuf.ExtraBuf.Struct\x1a(\n\x06Struct\x12\x0e\n\x06\x66ield1\x18\x01 \x01(\t\x12\x0e\n\x06\x66ield2\x18\x02 \x01(\tB\tZ\x07.;protob\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'extrabuf_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+ _globals['DESCRIPTOR']._options = None
+ _globals['DESCRIPTOR']._serialized_options = b'Z\007.;proto'
+ _globals['_EXTRABUF']._serialized_start=32
+ _globals['_EXTRABUF']._serialized_end=132
+ _globals['_EXTRABUF_STRUCT']._serialized_start=92
+ _globals['_EXTRABUF_STRUCT']._serialized_end=132
+# @@protoc_insertion_point(module_scope)