Selenium 代理：完整指南

Selenium 代理：完整指南

Selenium WebDriver 是一款流行的浏览器自动化工具。在 Selenium 中配置代理需要特殊的方法。

Chrome + Selenium

基础配置

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

chrome_options = Options()
chrome_options.add_argument('--proxy-server=http://proxy.example.com:8080')

driver = webdriver.Chrome(options=chrome_options)
driver.get('https://httpbin.org/ip')

带身份验证（通过扩展程序）

import zipfile

def create_proxy_extension(proxy_host, proxy_port, proxy_user, proxy_pass):
    manifest_json = '''
    {
        "version": "1.0.0",
        "manifest_version": 2,
        "name": "Proxy Auth",
        "permissions": ["proxy", "webRequest", "webRequestBlocking", "<all_urls>"],
        "background": {"scripts": ["background.js"]}
    }
    '''

    background_js = f'''
    var config = {{
        mode: "fixed_servers",
        rules: {{
            singleProxy: {{
                scheme: "http",
                host: "{proxy_host}",
                port: parseInt({proxy_port})
            }}
        }}
    }};

    chrome.proxy.settings.set({{value: config, scope: "regular"}}, function() {{}});

    chrome.webRequest.onAuthRequired.addListener(
        function(details) {{
            return {{
                authCredentials: {{
                    username: "{proxy_user}",
                    password: "{proxy_pass}"
                }}
            }};
        }},
        {{urls: ["<all_urls>"]}},
        ['blocking']
    );
    '''

    with zipfile.ZipFile('proxy_auth.zip', 'w') as zp:
        zp.writestr("manifest.json", manifest_json)
        zp.writestr("background.js", background_js)

    return 'proxy_auth.zip'

# 用法
ext_path = create_proxy_extension('proxy.example.com', 8080, 'user', 'pass')

chrome_options = Options()
chrome_options.add_extension(ext_path)

driver = webdriver.Chrome(options=chrome_options)

Firefox + Selenium

from selenium import webdriver
from selenium.webdriver.firefox.options import Options

profile = webdriver.FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", "proxy.example.com")
profile.set_preference("network.proxy.http_port", 8080)
profile.set_preference("network.proxy.ssl", "proxy.example.com")
profile.set_preference("network.proxy.ssl_port", 8080)

driver = webdriver.Firefox(firefox_profile=profile)

SOCKS5 代理

chrome_options = Options()
chrome_options.add_argument('--proxy-server=socks5://proxy.example.com:1080')

代理轮换

import random

proxies = [
    'http://proxy1.example.com:8080',
    'http://proxy2.example.com:8080',
    'http://proxy3.example.com:8080',
]

def get_driver_with_proxy():
    proxy = random.choice(proxies)
    chrome_options = Options()
    chrome_options.add_argument(f'--proxy-server={proxy}')
    return webdriver.Chrome(options=chrome_options)

绕过检测

chrome_options = Options()
chrome_options.add_argument('--proxy-server=http://proxy:8080')
chrome_options.add_argument('--disable-blink-features=AutomationControlled')
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
chrome_options.add_experimental_option('useAutomationExtension', False)

driver = webdriver.Chrome(options=chrome_options)
driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
    'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'
})

验证

driver.get('https://httpbin.org/ip')
print(driver.page_source)

提示

1. 谨慎使用 headless 模式 —— 许多网站能检测到它
2. 在各个操作之间加入随机延迟
3. 在轮换代理的同时轮换 User-Agent
4. 对于难以处理的网站，使用住宅代理