python-drission-page-spider/main.py
shikong 7a19ca02bf
refactor: 重构日志系统并优化线程池配置
重构日志系统,使用loguru替代print语句,提升日志管理能力。优化线程池配置,根据CPU核心数动态设置线程池大小。将任务执行逻辑移至worker.py,提高代码可维护性。
2025-03-21 00:55:10 +08:00

53 lines
1.2 KiB
Python

from loguru import logger
logger.remove()
import sys
logger.add(sys.stdout,
filter=lambda record: record["extra"].get("name") == "runtime")
logger.add("log/runtime.log",
rotation="100 MB",
encoding="utf-8",
filter=lambda record: record["extra"].get("name") == "runtime")
log = logger.bind(name="runtime")
from DrissionPage import Chromium, ChromiumOptions, SessionPage
from concurrent.futures import ThreadPoolExecutor
import atexit
import listener.console
import listener.network
# 初始化浏览器
options = ChromiumOptions()
# 无头模式
# options.headless()
import os
os.environ.setdefault('PYTHONIOENCODING', 'utf-8')
print(os.environ)
browser = Chromium(options)
tab = browser.latest_tab
import os
# 初始化线程池
cpu_count = os.cpu_count()
cpu_count = cpu_count if cpu_count is not None else 1
executor = ThreadPoolExecutor(max_workers=cpu_count * 2)
executor.submit(listener.console.console_listener, tab)
executor.submit(listener.network.network_listener, tab)
def exit():
log.info('退出')
browser.quit()
executor.shutdown(False)
atexit.register(exit)
from worker import run_task;
try:
run_task(browser, executor, log)
except Exception as e:
log.error(e)