youproxy(如何用Python爬蟲代理增加網站流量)

獲得了免費的代理列表,那么就有很多事情可以干,比如 , 爬取某個網站并且沒有被封IP的風險, 比如, 增加某網站的流量。

完整代碼:

#coding:utf-8importurllib2importurllibimportcookielibimporthashlibimportreimporttimeimportjsonimportunittestfromseleniumimportwebdriverfrombs4importBeautifulSoupfrompip._vendor.distlib._backport.tarfileimportTUREADfromtimeimportsleepfromseleniumimportwebdriverfromselenium.webdriver.common.keysimportKeysfromselenium.webdriver.common.desired_capabilitiesimportDesiredCapabilitiesimportrandomclassSpide:def__init__(self,proxy_ip,proxy_type,proxy_port,use_proxy=False):print'usingtheproxyinfo:',proxy_ipself.proxy_ip=proxy_ipself.proxy_type=proxy_typeself.proxy_port=proxy_portself.proxy=urllib2.ProxyHandler({proxy_type:proxy_ip+":"+proxy_port})self.usercode=""self.userid=""self.cj=cookielib.LWPCookieJar();self.opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj));ifuse_proxy:self.opener=urllib2.build_opener(self.proxy)urllib2.install_opener(self.opener);defadd_view(self):print'--->startaddingview'print'--->proxyinfo',self.proxy_ipservice_args=['--proxy='+self.proxy_ip+':'+self.proxy_port,'--proxy-type='+self.proxy_type,]dcap=dict(DesiredCapabilities.PHANTOMJS)dcap["phantomjs.page.settings.userAgent"]=("Mozilla/5.0(X11;Linuxx86_64)AppleWebKit/53""(KHTML,likeGecko)Chrome/15.0.87")driver=webdriver.PhantomJS(executable_path='/home/bin/phantomjs',service_args=service_args,desired_capabilities=dcap)driver.set_page_load_timeout(90)driver.get("http://www.503error.com/")soup=BeautifulSoup(driver.page_source,'xml')titles=soup.find_all('h2',{'class':'entry-title'})ranCount=random.randint(0,len(titles))print'randomfindalinkofthewebsitetoaccess,randomis:',ranCountrandomlink=titles[ranCount].a.attrs['href']driver.get(randomlink)driver.close()print'finishonce'defget_proxy(self):proxy_info_json=""#firstgettheproxyinfofromprint'-->usingtheip'+self.proxy_ip+'togettheproxyinfo'try:reqRequest_proxy=urllib2.Request('url2');reqRequest_proxy.add_header('Accept','*/*');reqRequest_proxy.add_header('Accept-Language','zh-CN,zh;q=0.8');reqRequest_proxy.add_header('User-Agent','Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/28.0.1500.72Safari/537.36');reqRequest_proxy.add_header('Content-Type','application/x-www-form-urlencoded');proxy_info=urllib2.urlopen(reqRequest_proxy).read();printproxy_infoproxy_info_json=json.loads(proxy_info)return_str=proxy_info_json['protocol']+":"+proxy_info_json['ip']+proxy_info_json['port']exceptException,e:print'proxyhaveproblem'#printproxy_info_json['protocol']#printproxy_info_json['ip']#printproxy_info_json['port']returnproxy_info_json#printproxy_infodefget_proxys100(self):proxy_info_json=""#firstgettheproxyinfofromprint'-->usingtheip'+self.proxy_ip+'togettheproxyinfo100'try:reqRequest_proxy=urllib2.Request('url1');reqRequest_proxy.add_header('Accept','*/*');reqRequest_proxy.add_header('Accept-Language','zh-CN,zh;q=0.8');reqRequest_proxy.add_header('User-Agent','Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/28.0.1500.72Safari/537.36');reqRequest_proxy.add_header('Content-Type','application/x-www-form-urlencoded');proxy_info=urllib2.urlopen(reqRequest_proxy).read();#printproxy_infoproxy_info_json=json.loads(proxy_info)#forporxy_iinproxy_info_json:#printporxy_i#return_str=proxy_info_json['protocol']+":"+proxy_info_json['ip']+proxy_info_json['port']returnproxy_info_jsonexceptException,e:print'proxyhaveproblem'if__name__=="__main__":#firstimegettheproxyprint'STARTADDINGVIEW:'print'GetingthenewproxyinfoFirsttime'print'---------------------------------------------------------------------------------------------------------'forcountinrange(1):test=Spide(proxy_ip='youproxyip',proxy_type='http',proxy_port='3128',use_proxy=False)proxy_list=test.get_proxy()print'->thisisthe:',countprint'->Getingthenewproxyinfo:'print'->usingtheproxytogetproxylistincaseforbiden'print'->proxyinfo',proxy_listproxy100=test.get_proxys100()forproxy1inproxy100:try:print'proxy1:',proxy1Spide1=Spide(proxy_ip=proxy1['ip'],proxy_type=proxy1['type'],proxy_port=proxy1['port'],use_proxy=True)print'beforeaddview'Spide1.add_view()print'->sleep15s'time.sleep(15)#sleeprandomtimetoranTime=random.randint(10,50)print'->sleeprandomtime:',ranTimetime.sleep(ranTime)print'->gettingnewproxy'#proxy_list=Spide1.get_proxy()exceptException,e:print'->somethingwrong,hahah,next'

一點小的注釋:

整個流程為: 1 獲取代理 ->2　訪問首頁　—>3　獲取首頁博客列表,隨機訪問->4隨機等待N秒　->返回第1步

1:你需要更改youproxyip為你一個你已經擁有的代理ip,或者,不用填寫,因為后邊的use_proxy=False, 這個時候你確保你能夠不適用代理訪問到代碼中的兩個自動抓取代理ip地址的網站

2:/home/bin/phantomjs 這個路徑是你安裝的phantomjs路徑

3:代碼中有兩個獲取代理的方法,例子中選擇了一個(不要噴我下邊的循環為什么是一次還要循環,因為這個版本是原來是有外層循環的)

4: 獲取免費代理地址就不寫了,url1 ,url2 為隱藏的獲取免費代理的網站