Running selenium behind a proxy server

Question

I have been using selenium for automatic browser simulations and web scraping in python and it has worked well for me. But now, I have to run it behind a proxy server. So now selenium open up the window but could not open the requested page because of proxy settings not set on the opened browser. Current code is as follows (sample):

from selenium import webdriver sel = webdriver.Firefox() sel.get('http://www.google.com') sel.title sel.quit() 

How do I change the above code to work with proxy server now as well?

Answer 1

You need to set desired capabilities or browser profile, like this:

profile = webdriver.FirefoxProfile() profile.set_preference("network.proxy.type", 1) profile.set_preference("network.proxy.http", "proxy.server.address") profile.set_preference("network.proxy.http_port", "port_number") profile.update_preferences() driver = webdriver.Firefox(firefox_profile=profile) 

Also see related threads:

• how do i set proxy for chrome in python webdriver
• Selenium using Python: enter/provide http proxy password for firefox
• Running Selenium Webdriver with a proxy in Python
• http://krosinski.blogspot.ru/2012/11/selenium-firefox-webdriver-and-proxies.html

Answer 2

The official Selenium documentation (http://docs.seleniumhq.org/docs/04_webdriver_advanced.jsp#using-a-proxy) provides clear and helpful guidelines about using a proxy. For Firefox (which is the browser of choice in your sample code) you should do the following:

from selenium import webdriver from selenium.webdriver.common.proxy import * myProxy = "host:8080" proxy = Proxy({ 'proxyType': ProxyType.MANUAL, 'httpProxy': myProxy, 'ftpProxy': myProxy, 'sslProxy': myProxy, 'noProxy': '' # set this value as desired }) driver = webdriver.Firefox(proxy=proxy) 

Answer 3

This will do the job:

import selenium from selenium.webdriver.common.proxy import * proxyHost = "my.proxy.host or IP" proxyPort = "55555" fp = webdriver.FirefoxProfile() fp.set_preference("network.proxy.type", 1) #fp.set_preference("network.proxy.http", proxyHost) #HTTP PROXY #fp.set_preference("network.proxy.http_port", int(proxyPort)) #fp.set_preference("network.proxy.ssl", proxyHost) #SSL PROXY #fp.set_preference("network.proxy.ssl_port", int(proxyPort)) fp.set_preference('network.proxy.socks', proxyHost) #SOCKS PROXY fp.set_preference('network.proxy.socks_port', int(proxyPort)) fp.update_preferences() driver = webdriver.Firefox(firefox_profile=fp) driver.get("http://www.whatismyip.com/") 

Answer 4

from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.firefox.options import Options proxy = "xxx.xx.xx.xxxx" port = 8080 options = Options() options.set_preference("network.proxy.type", 1) options.set_preference("network.proxy.http", proxy) options.set_preference("network.proxy.http_port", port) options.set_preference("network.proxy.https", proxy) options.set_preference("network.proxy.https_port", port) ser = Service(r"C:/Users/geckodriver.exe") driver = webdriver.Firefox(service=ser, options=options) 

Answer 5

def install_proxy(PROXY_HOST,PROXY_PORT): fp = webdriver.FirefoxProfile() print PROXY_PORT print PROXY_HOST fp.set_preference("network.proxy.type", 1) fp.set_preference("network.proxy.http",PROXY_HOST) fp.set_preference("network.proxy.http_port",int(PROXY_PORT)) fp.set_preference("network.proxy.https",PROXY_HOST) fp.set_preference("network.proxy.https_port",int(PROXY_PORT)) fp.set_preference("network.proxy.ssl",PROXY_HOST) fp.set_preference("network.proxy.ssl_port",int(PROXY_PORT)) fp.set_preference("network.proxy.ftp",PROXY_HOST) fp.set_preference("network.proxy.ftp_port",int(PROXY_PORT)) fp.set_preference("network.proxy.socks",PROXY_HOST) fp.set_preference("network.proxy.socks_port",int(PROXY_PORT)) fp.set_preference("general.useragent.override","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A") fp.update_preferences() return webdriver.Firefox(firefox_profile=fp)