python-drission-page-spider/main.py

from loguru import logger
logger.remove()

import sys

logger.add(sys.stdout,
   filter=lambda record: record["extra"].get("name") == "runtime")
logger.add("log/runtime.log",
 rotation="100 MB",
  encoding="utf-8",
   filter=lambda record: record["extra"].get("name") == "runtime")

log = logger.bind(name="runtime")

from DrissionPage import Chromium, ChromiumOptions, SessionPage
from concurrent.futures import ThreadPoolExecutor
import atexit

import listener.console
import listener.network

# 初始化浏览器
options = ChromiumOptions()
# 无头模式
# options.headless()

browser = Chromium(options)
tab = browser.latest_tab

import os
# 初始化线程池
cpu_count = os.cpu_count()
cpu_count = cpu_count if cpu_count is not None else 1
executor = ThreadPoolExecutor(max_workers=cpu_count * 2)
executor.submit(listener.console.console_listener, tab)
executor.submit(listener.network.network_listener, tab)

def exit():
    log.info('退出')
    browser.quit()
    executor.shutdown(False)

atexit.register(exit)

from worker import run_task;
try:
    run_task(browser, executor, log)
except Exception as e:
    log.error(e)