From 0aa1ac964fe14c8432a0dfe59e17eafeb4a4a8b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E4=BC=9F?= Date: Wed, 6 Dec 2023 15:34:56 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AF=BC=E5=87=BAcsv=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=97=B6=EF=BC=8C=E5=A2=9E=E5=8A=A0=E5=8F=91=E9=80=81=E4=BA=BA?= =?UTF-8?q?=E5=90=8D=E7=A7=B0=E3=80=81=E6=8E=A5=E5=8F=97=E8=80=85=E5=90=8D?= =?UTF-8?q?=E7=A7=B0=E3=80=81=E5=A4=87=E6=B3=A8=E5=90=8D=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/DataBase/micro_msg.py | 31 +++++++++ app/DataBase/msg.py | 2 +- app/DataBase/output_pc.py | 13 ++-- app/DataBase/package_msg.py | 103 +++++++++++++++++++++++++++++ app/util/protocbuf/__init__.py | 0 app/util/protocbuf/msg.proto | 18 +++++ app/util/protocbuf/msg_pb2.py | 54 +++++++++++++++ app/util/protocbuf/readme.md | 34 ++++++++++ app/util/protocbuf/roomdata.proto | 19 ++++++ app/util/protocbuf/roomdata_pb2.py | 45 +++++++++++++ requirements.txt | Bin 838 -> 936 bytes 11 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 app/DataBase/package_msg.py create mode 100644 app/util/protocbuf/__init__.py create mode 100644 app/util/protocbuf/msg.proto create mode 100644 app/util/protocbuf/msg_pb2.py create mode 100644 app/util/protocbuf/readme.md create mode 100644 app/util/protocbuf/roomdata.proto create mode 100644 app/util/protocbuf/roomdata_pb2.py diff --git a/app/DataBase/micro_msg.py b/app/DataBase/micro_msg.py index f36cec8..23e7d7e 100644 --- a/app/DataBase/micro_msg.py +++ b/app/DataBase/micro_msg.py @@ -63,6 +63,37 @@ class MicroMsg: lock.release() return result + def get_contact_by_username(self, username): + if not self.open_flag: + return None + try: + lock.acquire(True) + sql = '''SELECT UserName, Alias, Type, Remark, NickName, PYInitial, RemarkPYInitial, ContactHeadImgUrl.smallHeadImgUrl, ContactHeadImgUrl.bigHeadImgUrl + FROM Contact + INNER JOIN ContactHeadImgUrl ON Contact.UserName = ContactHeadImgUrl.usrName + WHERE UserName = ? + ''' + self.cursor.execute(sql, [username]) + result = self.cursor.fetchone() + finally: + lock.release() + return result + + def get_chatroom_info(self, chatroomname): + ''' + 获取群聊信息 + ''' + if not self.open_flag: + return None + try: + lock.acquire(True) + sql = '''SELECT ChatRoomName, RoomData FROM ChatRoom WHERE ChatRoomName = ?''' + self.cursor.execute(sql, [chatroomname]) + result = self.cursor.fetchone() + finally: + lock.release() + return result + def close(self): if self.open_flag: try: diff --git a/app/DataBase/msg.py b/app/DataBase/msg.py index 9399c82..76a2919 100644 --- a/app/DataBase/msg.py +++ b/app/DataBase/msg.py @@ -72,7 +72,7 @@ class Msg: def get_messages_all(self): sql = ''' - select localId,TalkerId,Type,SubType,IsSender,CreateTime,Status,StrContent,strftime('%Y-%m-%d %H:%M:%S',CreateTime,'unixepoch','localtime') as StrTime,MsgSvrID + select localId,TalkerId,Type,SubType,IsSender,CreateTime,Status,StrContent,strftime('%Y-%m-%d %H:%M:%S',CreateTime,'unixepoch','localtime') as StrTime,MsgSvrID,BytesExtra,StrTalker,Reserved1 from MSG order by CreateTime ''' diff --git a/app/DataBase/output_pc.py b/app/DataBase/output_pc.py index a9e8e15..6f78d93 100644 --- a/app/DataBase/output_pc.py +++ b/app/DataBase/output_pc.py @@ -5,6 +5,7 @@ import os from PyQt5.QtCore import pyqtSignal, QThread from . import msg_db +from .package_msg import PackageMsg from ..DataBase import hard_link_db from ..person_pc import MePC from ..util import get_abs_path @@ -65,8 +66,10 @@ class Output(QThread): # columns = ["用户名", "消息内容", "发送时间", "发送状态", "消息类型", "isSend", "msgId"] columns = ['localId', 'TalkerId', 'Type', 'SubType', 'IsSender', 'CreateTime', 'Status', 'StrContent', - 'StrTime'] - messages = msg_db.get_messages_all() + 'StrTime', 'Remark', 'NickName', 'Sender'] + # messages = msg_db.get_messages_all() + packagemsg = PackageMsg() + messages = packagemsg.get_package_message_all() # 写入CSV文件 with open(filename, mode='w', newline='', encoding='utf-8') as file: writer = csv.writer(file) @@ -165,8 +168,10 @@ class ChildThread(QThread): # columns = ["用户名", "消息内容", "发送时间", "发送状态", "消息类型", "isSend", "msgId"] columns = ['localId', 'TalkerId', 'Type', 'SubType', 'IsSender', 'CreateTime', 'Status', 'StrContent', - 'StrTime'] - messages = msg_db.get_messages_all() + 'StrTime', 'Remark', 'NickName', 'Sender'] + # messages = msg_db.get_messages_all() + packagemsg = PackageMsg() + messages = packagemsg.get_package_message_all() # 写入CSV文件 with open(filename, mode='w', newline='', encoding='utf-8') as file: writer = csv.writer(file) diff --git a/app/DataBase/package_msg.py b/app/DataBase/package_msg.py new file mode 100644 index 0000000..59bfd9a --- /dev/null +++ b/app/DataBase/package_msg.py @@ -0,0 +1,103 @@ +import threading + +from app.DataBase import msg_db, micro_msg_db +from app.util.protocbuf.msg_pb2 import MessageBytesExtra +from app.util.protocbuf.roomdata_pb2 import ChatRoomData + +lock = threading.Lock() + + +def singleton(cls): + _instance = {} + + def inner(): + if cls not in _instance: + _instance[cls] = cls() + return _instance[cls] + + return inner + + +@singleton +class PackageMsg: + def __init__(self): + self.ChatRoomMap = {} + + def get_package_message_all(self): + ''' + 获取完整的聊天记录 + ''' + updated_messages = [] # 用于存储修改后的消息列表 + + messages = msg_db.get_messages_all() + for row in messages: + row_list = list(row) + # 删除不使用的几个字段 + del row_list[12] + del row_list[11] + del row_list[10] + + strtalker = row[11] + info = micro_msg_db.get_contact_by_username(strtalker) + if info is not None: + row_list.append(info[3]) + row_list.append(info[4]) + # 判断是否是群聊 + if strtalker.__contains__('@chatroom'): + # 自己发送 + if row[12] == 1: + row_list.append('我') + else: + # 存在BytesExtra为空的情况，此时消息类型应该为提示性消息。跳过不处理 + if row[10] is None: + continue + # 解析BytesExtra + msgbytes = MessageBytesExtra() + msgbytes.ParseFromString(row[10]) + wxid = '' + for tmp in msgbytes.message2: + if tmp.field1 != 1: + continue + wxid = tmp.field2 + sender = '' + # 获取群聊成员列表 + membersMap = self.get_chatroom_member_list(strtalker) + if membersMap is not None: + if wxid in membersMap: + sender = membersMap.get(wxid) + else: + senderinfo = micro_msg_db.get_contact_by_username(wxid) + if senderinfo is not None: + sender = senderinfo[4] + membersMap[wxid] = senderinfo[4] + if len(senderinfo[3]) > 0: + sender = senderinfo[3] + membersMap[wxid] = senderinfo[3] + row_list.append(sender) + updated_messages.append(tuple(row_list)) + return updated_messages + + def get_chatroom_member_list(self, strtalker): + membermap = {} + ''' + 获取群聊成员 + ''' + try: + lock.acquire(True) + if strtalker in self.ChatRoomMap: + membermap = self.ChatRoomMap.get(strtalker) + else: + chatroom = micro_msg_db.get_chatroom_info(strtalker) + if chatroom is None: + return None + # 解析RoomData数据 + parsechatroom = ChatRoomData() + parsechatroom.ParseFromString(chatroom[1]) + # 群成员数据放入字典存储 + for mem in parsechatroom.members: + if mem.displayName is not None and len(mem.displayName) > 0: + membermap[mem.wxID] = mem.displayName + self.ChatRoomMap[strtalker] = membermap + finally: + lock.release() + return membermap diff --git a/app/util/protocbuf/__init__.py b/app/util/protocbuf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/util/protocbuf/msg.proto b/app/util/protocbuf/msg.proto new file mode 100644 index 0000000..1d88cec --- /dev/null +++ b/app/util/protocbuf/msg.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; +package app.protobuf; +option go_package=".;proto"; + +message SubMessage1 { + int32 field1 = 1; + int32 field2 = 2; +} + +message SubMessage2 { + int32 field1 = 1; + string field2 = 2; +} + +message MessageBytesExtra { + SubMessage1 message1 = 1; + repeated SubMessage2 message2 = 3; +} diff --git a/app/util/protocbuf/msg_pb2.py b/app/util/protocbuf/msg_pb2.py new file mode 100644 index 0000000..f5f31c5 --- /dev/null +++ b/app/util/protocbuf/msg_pb2.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: msg.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tmsg.proto\x12\x0c\x61pp.protobuf\"-\n\x0bSubMessage1\x12\x0e\n\x06\x66ield1\x18\x01 \x01(\x05\x12\x0e\n\x06\x66ield2\x18\x02 \x01(\x05\"-\n\x0bSubMessage2\x12\x0e\n\x06\x66ield1\x18\x01 \x01(\x05\x12\x0e\n\x06\x66ield2\x18\x02 \x01(\t\"m\n\x11MessageBytesExtra\x12+\n\x08message1\x18\x01 \x01(\x0b\x32\x19.app.protobuf.SubMessage1\x12+\n\x08message2\x18\x03 \x03(\x0b\x32\x19.app.protobuf.SubMessage2b\x06proto3') + + + +_SUBMESSAGE1 = DESCRIPTOR.message_types_by_name['SubMessage1'] +_SUBMESSAGE2 = DESCRIPTOR.message_types_by_name['SubMessage2'] +_MESSAGEBYTESEXTRA = DESCRIPTOR.message_types_by_name['MessageBytesExtra'] +SubMessage1 = _reflection.GeneratedProtocolMessageType('SubMessage1', (_message.Message,), { + 'DESCRIPTOR' : _SUBMESSAGE1, + '__module__' : 'msg_pb2' + # @@protoc_insertion_point(class_scope:app.protobuf.SubMessage1) + }) +_sym_db.RegisterMessage(SubMessage1) + +SubMessage2 = _reflection.GeneratedProtocolMessageType('SubMessage2', (_message.Message,), { + 'DESCRIPTOR' : _SUBMESSAGE2, + '__module__' : 'msg_pb2' + # @@protoc_insertion_point(class_scope:app.protobuf.SubMessage2) + }) +_sym_db.RegisterMessage(SubMessage2) + +MessageBytesExtra = _reflection.GeneratedProtocolMessageType('MessageBytesExtra', (_message.Message,), { + 'DESCRIPTOR' : _MESSAGEBYTESEXTRA, + '__module__' : 'msg_pb2' + # @@protoc_insertion_point(class_scope:app.protobuf.MessageBytesExtra) + }) +_sym_db.RegisterMessage(MessageBytesExtra) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _SUBMESSAGE1._serialized_start=27 + _SUBMESSAGE1._serialized_end=72 + _SUBMESSAGE2._serialized_start=74 + _SUBMESSAGE2._serialized_end=119 + _MESSAGEBYTESEXTRA._serialized_start=121 + _MESSAGEBYTESEXTRA._serialized_end=230 +# @@protoc_insertion_point(module_scope) diff --git a/app/util/protocbuf/readme.md b/app/util/protocbuf/readme.md new file mode 100644 index 0000000..15b3c63 --- /dev/null +++ b/app/util/protocbuf/readme.md @@ -0,0 +1,34 @@ +# 说明 + +## 解析 +```shell +protoc --decode_raw < msg_data.txt +``` + +## 根据解析结果，设置.proto文件 +```shell +1 { + 1: 16 + 2: 0 +} +3 { + 1: 1 + 2: "wxid_4b1t09d63spw22" +} +3 { + 1: 7 + 2: "\n\t\n\t\t2\n\t\n\t\n\t\tc6680ab2c57499a1a22e44a7eada76e8_\n\t\n\t1\n\t198\n\tv1_Gj7hfmi5\n\t\n\t\t\n\t\n\n" +} +3 { + 1: 2 + 2: "c13acbc95512d1a59bb686d684fd64d8" +} +3 { + 1: 4 + 2: "yiluoAK_47\\FileStorage\\Cache\\2023-08\\2286b5852db82f6cbd9c2084ccd52358" +} +``` +## 生成python文件 +```shell +protoc --python_out=. msg.proto +``` \ No newline at end of file diff --git a/app/util/protocbuf/roomdata.proto b/app/util/protocbuf/roomdata.proto new file mode 100644 index 0000000..36b1f9c --- /dev/null +++ b/app/util/protocbuf/roomdata.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; +package app.protobuf; +option go_package=".;proto"; + +message ChatRoomData { + message ChatRoomMember { + string wxID = 1; + string displayName = 2; + int32 state = 3; + } + repeated ChatRoomMember members = 1; + int32 field_2 = 2; + int32 field_3 = 3; + int32 field_4 = 4; + int32 room_capacity = 5; + int32 field_6 = 6; + int64 field_7 = 7; + int64 field_8 = 8; +} \ No newline at end of file diff --git a/app/util/protocbuf/roomdata_pb2.py b/app/util/protocbuf/roomdata_pb2.py new file mode 100644 index 0000000..7800214 --- /dev/null +++ b/app/util/protocbuf/roomdata_pb2.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: roomdata.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eroomdata.proto\x12\x0c\x61pp.protobuf\"\x8b\x02\n\x0c\x43hatRoomData\x12:\n\x07members\x18\x01 \x03(\x0b\x32).app.protobuf.ChatRoomData.ChatRoomMember\x12\x0f\n\x07\x66ield_2\x18\x02 \x01(\x05\x12\x0f\n\x07\x66ield_3\x18\x03 \x01(\x05\x12\x0f\n\x07\x66ield_4\x18\x04 \x01(\x05\x12\x15\n\rroom_capacity\x18\x05 \x01(\x05\x12\x0f\n\x07\x66ield_6\x18\x06 \x01(\x05\x12\x0f\n\x07\x66ield_7\x18\x07 \x01(\x03\x12\x0f\n\x07\x66ield_8\x18\x08 \x01(\x03\x1a\x42\n\x0e\x43hatRoomMember\x12\x0c\n\x04wxID\x18\x01 \x01(\t\x12\x13\n\x0b\x64isplayName\x18\x02 \x01(\t\x12\r\n\x05state\x18\x03 \x01(\x05\x62\x06proto3') + + + +_CHATROOMDATA = DESCRIPTOR.message_types_by_name['ChatRoomData'] +_CHATROOMDATA_CHATROOMMEMBER = _CHATROOMDATA.nested_types_by_name['ChatRoomMember'] +ChatRoomData = _reflection.GeneratedProtocolMessageType('ChatRoomData', (_message.Message,), { + + 'ChatRoomMember' : _reflection.GeneratedProtocolMessageType('ChatRoomMember', (_message.Message,), { + 'DESCRIPTOR' : _CHATROOMDATA_CHATROOMMEMBER, + '__module__' : 'roomdata_pb2' + # @@protoc_insertion_point(class_scope:app.protobuf.ChatRoomData.ChatRoomMember) + }) + , + 'DESCRIPTOR' : _CHATROOMDATA, + '__module__' : 'roomdata_pb2' + # @@protoc_insertion_point(class_scope:app.protobuf.ChatRoomData) + }) +_sym_db.RegisterMessage(ChatRoomData) +_sym_db.RegisterMessage(ChatRoomData.ChatRoomMember) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _CHATROOMDATA._serialized_start=33 + _CHATROOMDATA._serialized_end=300 + _CHATROOMDATA_CHATROOMMEMBER._serialized_start=234 + _CHATROOMDATA_CHATROOMMEMBER._serialized_end=300 +# @@protoc_insertion_point(module_scope) diff --git a/requirements.txt b/requirements.txt index afa9ee0da14451a74ae66a3838952e5635aed887..917bb83aebb523af977f0a8f8a89970e7f0c1b27 100644 GIT binary patch delta 106 zcmX@cwt{_w8*>sb0~bR&Lp~6uGvqL&GS~v4F@qk10T_d13mA%kvL!&A#8AqR23BDL hlr>^71(JpgAT`B6*;1g&Vunnh8D&s&jDYf{3;^815eEPO delta 7 OcmZ3%evEB{8#4e4UIKLh