python-drission-page-spider/main.py

from DrissionPage import Chromium, ChromiumOptions, SessionPage
from concurrent.futures import ThreadPoolExecutor
import atexit
import json

options = ChromiumOptions()
# 无头模式
# options.headless()

browser = Chromium(options)
tab = browser.latest_tab


def console_listener(tab):
    #  监听控制台
    tab.console.start()
    steps = tab.console.steps()
    for log in steps:
        # print("[console.log]", log)
        print(f"[console.log] {log.text}")

def network_listener(tab):
    # 监听网络
    tab.listen.start(targets=["/api/"])
    for log in tab.listen.steps():
        try:
            print("=" * 50)
            print(f"[network.log] {log.method} {log.url} {'request'} {''}")
            print("=" * 50 + '\t' + "request")
            print("=" * 50 + '\t' + "request.headers")
            for key in log.request.headers.keys():
                print(key, log.request.headers[key])

            reqType = log.request.headers['Content-Type']
            if reqType == 'application/json':
                print("=" * 50 + '\t' + "request.body")
                print(json.dumps(log.request.body, indent=4))
            print(vars(log.request))

            respType = log.response.headers['Content-Type']
            if respType == 'application/json':
                print("=" * 50 + '\t' + "response.body")
                print(json.dumps(log.response.body, indent=4))
            
            print(vars(log.response))
        except Exception as e:
            print(e)

executor = ThreadPoolExecutor(max_workers=2)
executor.submit(console_listener, tab)
executor.submit(network_listener, tab)

tab.get(url='http://127.0.0.1:24613')
tab.change_mode()


items = tab.eles("t:body")
for item in items:
    print(item)

def exit():
    print('exit')
    browser.quit()
    executor.shutdown()

atexit.register(exit)