python爬取網頁內容(怎么在Python中使用Scrapy爬取網頁內容)

爬蟲主程序:

#-*-coding:utf-8-*-importscrapyfromscrapy.httpimportRequestfromzjf.FsmzItemsimportFsmzItemfromscrapy.selectorimportSelector#圈圈:情感生活classMySpider(scrapy.Spider):#爬蟲名name="MySpider"#設定域名allowed_domains=["nvsheng.com"]#爬取地址start_urls=[]#flagx=0#爬取方法defparse(self,response):item=FsmzItem()sel=Selector(response)item['title']=sel.xpath('//h2/text()').extract()item['text']=sel.xpath('//*[@class="content"]/p/text()').extract()item['imags']=sel.xpath('//p[@id="content"]/p/a/img/@src|//p[@id="content"]/p/img/@src').extract()ifMySpider.x==0:page_list=MySpider.getUrl(self,response)forpage_singleinpage_list:yieldRequest(page_single)MySpider.x+=1yielditem#init:動態傳入參數#命令行傳參寫法:scrapycrawlMySpider-astart_url="http://some_url"def__init__(self,*args,**kwargs):super(MySpider,self).__init__(*args,**kwargs)self.start_urls=[kwargs.get('start_url')]defgetUrl(self,response):url_list=[]select=Selector(response)page_list_tmp=select.xpath('//p[@class="viewnewpages"]/a[not(@class="next")]/@href').extract()forpage_tmpinpage_list_tmp:ifpage_tmpnotinurl_list:url_list.append("http://www.nvsheng.com/emotion/px/"+page_tmp)returnurl_list

PipeLines類

#-*-coding:utf-8-*-#Defineyouritempipelineshere##Don'tforgettoaddyourpipelinetotheITEM_PIPELINESsetting#See:http://doc.scrapy.org/en/latest/topics/item-pipeline.htmlfromzjfimportsettingsimportjson,os,re,randomimporturllib.requestimportrequests,jsonfromrequests_toolbelt.multipart.encoderimportMultipartEncoderclassMyPipeline(object):flag=1post_title=''post_text=[]post_text_imageUrl_list=[]cs=[]user_id=''def__init__(self):MyPipeline.user_id=MyPipeline.getRandomUser('37619,18441390,18441391')#processthedatadefprocess_item(self,item,spider):#獲取隨機user_id,模擬發帖user_id=MyPipeline.user_id#獲取正文text_str_tmptext=item['text']text_str_tmp=""forstrintext:text_str_tmp=text_str_tmp+str#print(text_str_tmp)#獲取標題ifMyPipeline.flag==1:title=item['title']MyPipeline.post_title=MyPipeline.post_title+title[0]#保存并上傳圖片text_insert_pic=''text_insert_pic_w=''text_insert_pic_h=''forimag_urlinitem['imags']:img_name=imag_url.replace('/','').replace('.','').replace('|','').replace(':','')pic_dir=settings.IMAGES_STORE+'%s.jpg'%(img_name)urllib.request.urlretrieve(imag_url,pic_dir)#圖片上傳,返回jsonupload_img_result=MyPipeline.uploadImage(pic_dir,'image/jpeg')#獲取json中保存圖片路徑text_insert_pic=upload_img_result['result']['image_url']text_insert_pic_w=upload_img_result['result']['w']text_insert_pic_h=upload_img_result['result']['h']#拼接jsonifMyPipeline.flag==1:cs_json={"c":text_str_tmp,"i":"","w":text_insert_pic_w,"h":text_insert_pic_h}else:cs_json={"c":text_str_tmp,"i":text_insert_pic,"w":text_insert_pic_w,"h":text_insert_pic_h}MyPipeline.cs.append(cs_json)MyPipeline.flag+=1returnitem#spider開啟時被調用defopen_spider(self,spider):pass#sipder關閉時被調用defclose_spider(self,spider):strcs=json.dumps(MyPipeline.cs)jsonData={"apisign":"99ea3eda4b45549162c4a741d58baa60","user_id":MyPipeline.user_id,"gid":30,"t":MyPipeline.post_title,"cs":strcs}MyPipeline.uploadPost(jsonData)#上傳圖片defuploadImage(img_path,content_type):"uploadImagefunctions"#UPLOAD_IMG_URL="http://api.qa.douguo.net/robot/uploadpostimage"UPLOAD_IMG_URL="http://api.douguo.net/robot/uploadpostimage"#傳圖片#imgPath='D:\pics\http___img_nvsheng_com_uploads_allimg_170119_18-1f1191g440_jpg.jpg'm=MultipartEncoder(#fields={'user_id':'192323',#'images':('filename',open(imgPath,'rb'),'image/JPEG')}fields={'user_id':MyPipeline.user_id,'apisign':'99ea3eda4b45549162c4a741d58baa60','image':('filename',open(img_path,'rb'),'image/jpeg')})r=requests.post(UPLOAD_IMG_URL,data=m,headers={'Content-Type':m.content_type})returnr.json()defuploadPost(jsonData):CREATE_POST_URL=http://api.douguo.net/robot/uploadimagespost

reqPost=requests.post(CREATE_POST_URL,data=jsonData)

defgetRandomUser(userStr):user_list=[]user_chooesd=''foruser_idinstr(userStr).split(','):user_list.append(user_id)userId_idx=random.randint(1,len(user_list))user_chooesd=user_list[userId_idx-1]returnuser_chooesd

字段保存Items類

#-*-coding:utf-8-*-#Defineherethemodelsforyourscrapeditems##Seedocumentationin:#http://doc.scrapy.org/en/latest/topics/items.htmlimportscrapyclassFsmzItem(scrapy.Item):#definethefieldsforyouritemherelike:#name=scrapy.Field()title=scrapy.Field()#tutor=scrapy.Field()#strongText=scrapy.Field()text=scrapy.Field()imags=scrapy.Field()

在命令行里鍵入

scrapycrawlMySpider-astart_url=www.aaa.com

以上就是怎么在Python中使用Scrapy爬取網頁內容,小編相信有部分知識點可能是我們日常工作會見到或用到的。希望你能通過這篇文章學到更多知識。更多詳情敬請關注本站行業資訊頻道。