爬虫工具链安装

# Python 3.10+ 一键安装爬虫全工具链
pip install \
  requests httpx aiohttp \
  lxml beautifulsoup4 parsel pyquery \
  selenium playwright \
  pymysql pymongo redis \
  scrapy \
  scrapyd gerapy \
  mitmproxy Appium-Python-Client

# 场景一：轻量静态爬虫（最常用）
pip install requests lxml beautifulsoup4 pymysql

# 场景二：异步高并发爬虫
pip install httpx aiohttp lxml parsel redis

# 场景三：Scrapy 全栈项目
pip install scrapy scrapy-redis pymongo scrapyd

# 创建项目专用虚拟环境
python3 -m venv spider-project-env

# 激活（Linux / Mac）
source spider-project-env/bin/activate

# 激活（Windows PowerShell）
.\spider-project-env\Scripts\Activate.ps1

# 在激活的环境中安装依赖
pip install requests lxml beautifulsoup4 pymysql

# 锁定依赖版本
pip freeze > requirements.txt

# 其他开发者复现环境
pip install -r requirements.txt

# 永久配置清华镜像（推荐）
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
pip config set global.trusted-host pypi.tuna.tsinghua.edu.cn

# 其他可用镜像源
# 阿里云: https://mirrors.aliyun.com/pypi/simple
# 中科大: https://pypi.mirrors.ustc.edu.cn/simple
# 豆瓣:   https://pypi.doubanio.com/simple

"""爬虫工具链环境验证脚本 (Python 3.10+)"""
import sys

def check_import(module_name: str, package_name: str = "") -> bool:
    """尝试导入模块，返回是否成功"""
    try:
        __import__(module_name)
        print(f"  [OK] {package_name or module_name}")
        return True
    except ImportError as e:
        print(f"  [FAIL] {package_name or module_name}: {e}")
        return False

def main() -> None:
    print(f"Python {sys.version}")
    print(f"验证爬虫工具链安装状态...\n")

    results: dict[str, bool] = {}

    # 请求库
    print("--- 请求库 ---")
    results["requests"] = check_import("requests")
    results["httpx"] = check_import("httpx")
    results["aiohttp"] = check_import("aiohttp")

    # 解析库
    print("\n--- 解析库 ---")
    results["lxml"] = check_import("lxml")
    results["bs4"] = check_import("bs4", "beautifulsoup4")
    results["parsel"] = check_import("parsel")
    results["pyquery"] = check_import("pyquery")

    # 自动化
    print("\n--- 自动化 ---")
    results["selenium"] = check_import("selenium")
    results["playwright"] = check_import("playwright")

    # 存储库
    print("\n--- 存储库 ---")
    results["pymysql"] = check_import("pymysql")
    results["pymongo"] = check_import("pymongo")
    results["redis"] = check_import("redis")

    # 框架
    print("\n--- 框架 ---")
    results["scrapy"] = check_import("scrapy")

    # 部署
    print("\n--- 部署 ---")
    results["scrapyd"] = check_import("scrapyd")
    results["gerapy"] = check_import("gerapy")

    # App 抓包
    print("\n--- App 抓包 ---")
    results["mitmproxy"] = check_import("mitmproxy")
    results["appium"] = check_import("appium", "Appium-Python-Client")

    # 汇总
    passed = sum(results.values())
    total = len(results)
    print(f"\n{'='*40}")
    print(f"结果: {passed}/{total} 通过")
    if passed == total:
        print("爬虫工具链安装完整，可以开始开发！")
    else:
        failed = [k for k, v in results.items() if not v]
        print(f"未通过: {', '.join(failed)}")

if __name__ == "__main__":
    main()