""" Created on Wed Nov 23 00:03:30 2022 @author: fch """ import time ''' 网站网址：https://m.7160.top/rentiyishu/ ''' import random from pyquery import PyQuery as pq import requests import os from joblib import Parallel, delayed import socket socket.setdefaulttimeout(10) # 忽略requests证书警告 from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) requests.DEFAULT_RETRIES = 5 # 增加重试连接次数 IP_list=[] with open('./IP代理池.txt','r') as f: for line in f: line = line.strip('\n') # 删除换行符 IP_list.append(line) f.close() ''' random随机读取到后为字符串类型，用eval转回字典 ''' # proxie = eval(random.choice(IP_list)) user_agent_list = [ 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0', 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', 'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', 'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11', 'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5', 'Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5', 'Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5', 'Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', 'MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', 'Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10', 'Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13', 'Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+', 'Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0', 'Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)', 'UCWEB7.0.2.37/28/999', 'NOKIA5700/ UCWEB7.0.2.37/28/999', 'Openwave/ UCWEB7.0.2.37/28/999', 'Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25', ] def Res_url(url): requests.packages.urllib3.disable_warnings(InsecureRequestWarning) headers = {'User-Agent': random.choice(user_agent_list), 'Referer': 'https://m.7160.top/rentiyishu/', "Connection": "close"} try: s=requests.session() s.keep_alive = False r = s.get(url, headers=headers, proxies=eval(random.choice(IP_list)), timeout=(5, 10),verify=False,allow_redirects=False) r.raise_for_status() r.encoding = r.apparent_encoding return r.text except: print(url) return 'over' ''' 爬取首页中所有伪图片的链接地址和名称，返回成列表 ''' def Parse_page(html): page_url_list,fil_name_list=[],[] doc = pq(html) items = doc('.listUll2 li').items() for item in items: page_url_list.append(item('div a').attr('href')) fil_name_list.append(item('p').text()) return page_url_list,fil_name_list def Get_img(page_url, file_name): num = 1 base_url = page_url.split('.') while True: if num == 1: url = page_url else: url = base_url[0] + '.' + base_url[1] + '.' + base_url[2] + '_' + str(num) + '.' + base_url[3] time.sleep(1) html = Res_url(url) if html != 'over': doc = pq(html) img_url = doc('.ArticleBox img').attr('src') img_basename = doc('.ArticleBox img').attr('alt') if img_basename==None: img_name=img_basename else: img_name=img_basename.replace('/','') if img_url == None: num += 1 else: Save_img(img_url, file_name, img_name) num += 1 else: break #跳出while循环 def Save_img(img_url, file_name, img_name): time.sleep(1) requests.packages.urllib3.disable_warnings(InsecureRequestWarning) headers = {'User-Agent': random.choice(user_agent_list), 'Referer': 'https://m.7160.top/rentiyishu/', "Connection": "close"} houzui = img_url.split('.')[-1] addr = './images_3/' + file_name if not os.path.isdir(addr): os.makedirs(addr) r=requests.get(img_url, headers=headers, proxies=eval(random.choice(IP_list)), verify=False).content if r.status_code==200: with open('./images_3/{}/{}.{}'.format(file_name, img_name, houzui), 'wb') as f: f.write(r.content) f.close() else: print('图片地址不存在') def main(): page_num = 1 base_url = 'https://m.7160.top/rentiyishu/index' while True: if page_num == 1: url = base_url + '.html' else: url = base_url + '_' + str(page_num) + '.html' html = Res_url(url) if html != 'over': print('保存第', page_num, '页') page_url_list, file_name_list=Parse_page(html) ''' 多线程：n_jobs=30即同时30个线程 ''' Parallel(n_jobs=5)(delayed(Get_img)(page_url,file_name) for page_url,file_name in zip(page_url_list,file_name_list)) else: print('程序结束') return '程序结束' page_num += 1 main()
''' 网站网址：http://www.meinuzi.com/ ''' import random from pyquery import PyQuery as pq import requests proxies=[ {'HTTP':'http://122.140.5.115:9999'}, {'HTTP':'http://113.101.96.66:8080'}, {'HTTP':'http://113.124.86.24:9999'}, {'HTTP':'http://121.13.252.58:41564'}, {'HTTP':'http://61.216.185.88:60808'}, {'HTTP':'http://58.20.184.187:9091'}, {'HTTP':'http://183.236.123.242:8060'}, {'HTTP':'http://116.9.163.205:58080'}, {'HTTP':'http://222.74.73.202:42055'}, {'HTTP':'http://183.247.202.208:30001'}, {'HTTP':'http://39.108.101.55:1080'}, {'HTTP':'http://47.105.91.226:8118'}, ] proxie=random.choice(proxies) user_agent_list = ['Chrome/86.0.4240.198', 'Chrome/101.0.4951.64', 'Chrome/96.0.4664.45', 'Chrome/94.0.4606.41' ] headers = {'User-Agent': random.choice(user_agent_list), 'Referer':'http://www.meinuzi.com'} ''' 请求url，返回HTML ''' def Req_url(url): try: r=requests.get(url,headers=headers,proxies=proxie) r.raise_for_status() r.encoding=r.apparent_encoding return r.text except: # print('ERROR!') return 'over' ''' 解析初始页面，找到图片页面的网址 ''' def Parse_page(html): doc=pq(html) items=doc('.m-list-main li').items() i=1 for item in items: page_url=item('.u-img a').attr('href') name=item('.u-img img').attr('alt') print("保存第",i,'张图') Get_img(page_url,name) i+=1 ''' 解析图片页面的网址，找到每张图片的url ''' def Get_img(page_url,name): html=Req_url(page_url) doc=pq(html) img_baseurl=doc('.g-mn .m-list-content img').attr('src') Save_img(img_baseurl,name) ''' 根据得到的图片url，保存到文件夹 ''' def Save_img(img_baseurl,name): img_url=headers['Referer']+img_baseurl data=requests.get(img_url,headers=headers,proxies=proxie,timeout=15).content houzui=img_url.split('.')[-1] with open('./images_1/{}.{}'.format(name,houzui),'wb') as f: f.write(data) def main(): num = 7 # 从第num页开始 ''' 当返回值为‘over’的时候，说明页面不存在，即已经爬取全部页面 ''' while True: page_url='http://www.meinuzi.com/index_'+str(num)+'.html' html = Req_url(page_url) if html != 'over': ''' 页面存在但无图片数据，标签中只有换行符时 ''' doc=pq(html) item = doc('.m-list-main ul').text() if item!='' \ '': print('**********第',num,'页**********') Parse_page(html) num+=1 else: print('程序结束') return '程序结束' else: print('程序结束') return '程序结束' main()
# -*- coding: utf-8 -*- """ Created on Sat Nov 19 22:44:37 2022 @author: fch """ import random from pyquery import PyQuery as pq import requests import os import time import json proxies=[ {'HTTP':'http://122.140.5.115:9999'}, {'HTTP':'http://113.101.96.66:8080'}, {'HTTP':'http://113.124.86.24:9999'}, {'HTTP':'http://121.13.252.58:41564'}, {'HTTP':'http://61.216.185.88:60808'}, {'HTTP':'http://58.20.184.187:9091'}, {'HTTP':'http://183.236.123.242:8060'}, {'HTTP':'http://116.9.163.205:58080'}, {'HTTP':'http://222.74.73.202:42055'}, {'HTTP':'http://183.247.202.208:30001'}, {'HTTP':'http://39.108.101.55:1080'}, {'HTTP':'http://47.105.91.226:8118'}, ] proxie=random.choice(proxies) user_agent_list = ['Chrome/86.0.4240.198', 'Chrome/101.0.4951.64', 'Chrome/96.0.4664.45', 'Chrome/94.0.4606.41' ] headers = {'User-Agent': random.choice(user_agent_list)} list1 = [ "Chrome/68.0.3440.106", "Chrome/67.0.3396.99", "Chrome/64.0.3282.186", "Chrome/62.0.3202.62", "Chrome/45.0.2454.101" ] header={'User-Agent': random.choice(list1), 'Referer':'http://81.68.202.74/', 'sec-ch-ua': 'Google Chrome', 'sec-ch-ua-platform':'Windows' } base_url = 'http://81.68.202.74/datu/page/' # verify=False忽略SSL警告 def Base_page(Page_num): url = base_url + str(Page_num) try: r = requests.get(url, headers=headers,proxies=proxie) r.raise_for_status() r.encoding = r.apparent_encoding return r.text except: print('ERROR!!!') def Page_url(fir_html): doc = pq(fir_html) items = doc('.posts-wrapper .entry-media').items() num = 1 for item in items: page_url=item('a').attr('href') fil_name=item('.lazyload').attr('alt').replace(":","") # print(fil_name,page_url) print('------ 个数：',num,'------') Parse_img(page_url, fil_name) num+=1 def Parse_img(page_url,fil_name): time.sleep(0.25) i=1 r = requests.get(page_url, headers=headers,proxies=proxie) r.raise_for_status() r.encoding = r.apparent_encoding sec_html=r.text doc=pq(sec_html) items=doc('.entry-content p img').items() for item in items: dict={ 'url':item.attr('src'), 'name':item.attr('title') } print("保存第",i,'张图') i+=1 time.sleep(1.25) Save_img(dict,fil_name) def Save_img(dic,fil_name): url=dic['url'] # time.sleep(0.25) data = requests.get(url,headers=header,proxies=proxie).content name=dic['name'] houzui=url.split('.')[-1] addr='./images_2/'+fil_name if not os.path.isdir(addr): os.makedirs(addr) with open('./images_2/{}/{}.{}'.format(fil_name,name,houzui),'wb') as f: f.write(data) def main(): #左闭右开，到21页 for Page_num in range(1,22): print("*************第",Page_num,'页*************') fir_html = Base_page(Page_num) Page_url(fir_html) print("结束！") main()
from urllib.parse import urlencode import requests import re def Get_page(page_num): headers={ 'Host':'so.toutiao.com', 'user-agent':'Mozilla/5.0', 'cookie':'tt=acd600b5ac865033f0ee83a63ed44675; ' } base_url='https://so.toutiao.com/search?' params = { 'keyword': '街拍', 'pd': 'atlas', 'dvpf': 'pc', 'aid': '4916', 'page_num': page_num, 'search_json': '%7B%22from_search_id%22%3A%2220220104115420010212192151532E8188%22%2C%22origin_keyword%22%3A%22%E8%A1%97%E6%8B%8D%22%2C%22image_keyword%22%3A%22%E8%A1%97%E6%8B%8D%22%7D', 'rawJSON': 1, 'search_id': '2022062517173701021219402539E36546' } url=base_url+urlencode(params) try: r=requests.get(url,headers=headers) if r.status_code==200: return r.json() except: print('Failed!') def get_images(json): images = json['rawData']['data'] for image in images: yield { 'name': image['text'], 'url':image['img_url'] } def save_image(dict): data=requests.get(dict['url']).content #把“和 ：都替换成‘，’，否则python路径会识别错误 name=dict['name'].replace(':',',') name=name.replace('"',',') name=name.replace('?','') name=name.split('|')[-1] #碰到‘|’就取最后一个 houzui=dict['url'].split('.')[-1] #截取后缀，比如JPEG，png等图片的格式名称 with open("./images/{}.{}".format(name,houzui),'wb') as f: f.write(data) def main(): for page_num in range(1,20): json=Get_page(page_num) dict=get_images(json) for d in dict: save_image(d) main()
import pymysql from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait from pyquery import PyQuery as pq import pymongo '''链接mongodb''' client=pymongo.MongoClient("mongodb://*********:27017") db=client.test col=db.taobao '''连接mysql''' db=pymysql.Connect(host='*********',user='root',password='*********',port=3306,db='xiaomishop') cursor=db.cursor() table='taobao' '''初始url''' kw='iPad' #关键词 url='https://s.taobao.com/search?q='+kw driver=webdriver.Chrome() #指定浏览器对象 wait=WebDriverWait(driver,50) #最大等待时长 def Get_page(page_num): ''' 获取page_num页的页面信息(html) :param page_num: :return: Null ''' print('='*25,'正在爬取第',page_num,'页','='*25) try: driver.get(url) input = wait.until(EC.presence_of_element_located((By.CLASS_NAME,'J_Input'))) # 直到定位到搜索框 submit=wait.until(EC.element_to_be_clickable((By.CLASS_NAME,'J_Submit'))) #直到定位到能点击‘确定’ input.clear() input.send_keys(page_num) submit.click() Get_product() except TimeoutError: Get_page(page_num) def Get_product(): ''' 获取每页上所有商品的名称、价格、图片、购买量、店铺名称、店铺位置 :return:Null ''' html=driver.page_source doc=pq(html) items=doc('.J_MouserOnverReq').items() for item in items: product={ 'name': item.find('.ctx-box .row-2 a').text(), 'pic_img':item.find('.pic-box-inner .J_Itemlist_Pic_657878599145').attr('src'), 'price':item.find('.ctx-box .row-1 strong').text(), 'nums':item.find('.ctx-box .row-1 .deal-cnt').text(), 'shop':item.find('.ctx-box .row-3 .shop span').text(), 'location':item.find('.ctx-box .row-3 .location').text() } # write_to_mongodb(product) # write_to_mysql(product) print(product) def write_to_mongodb(dic): ''' 字典结果保存到MongoDB :param dic: :return: Null ''' col.insert_one(dic) def write_to_mysql(dic): ''' 字典结果写入mysql :param dic: :return: ''' keys=','.join(dic.keys()) values=','.join(['%s']*len(dic)) sql='insert into {table}({keys}) values ({values})'.format(table=table,keys=keys,values=values) try: if cursor.execute(sql,tuple(dic.values())): print('successful') db.commit() except: print('failed') db.rollback() if __name__ == '__main__': for page_num in range(1,101): Get_page(page_num)
import time from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver import ActionChains '''基本使用''' #声明浏览器对象 browser=webdriver.Chrome() try: '''访问页面''' browser.get('https://www.baidu.com') input=browser.find_element(By.ID,'kw') input.send_keys('python') input.send_keys(Keys.ENTER) wait=WebDriverWait(browser,10) wait.until(EC.presence_of_element_located((By.ID,'content_left'))) print('='*50) print(browser.current_url) print('=' * 50) print(browser.get_cookies()) print('=' * 50) # print(browser.page_source) browser.quit() except: browser.close() '''节点交互''' driver=webdriver.Chrome() try: driver.get('https://www.taobao.com') input=driver.find_element(By.ID,'q') input.send_keys('手机') # input.clear() #清除 button=driver.find_element(By.CLASS_NAME,'btn-search') button.click() driver.close() except: driver.close() '''动作链''' driver=webdriver.Chrome() try: driver.get('https://www.runoob.com/try/try.php?filename=jqueryui-api-droppable') driver.switch_to.frame('iframeResult') #切换frame source=driver.find_element(By.ID,'draggable') target=driver.find_element(By.ID,'droppable') actions=ActionChains(driver) actions.drag_and_drop(source,target) actions.perform() except: driver.close() '''执行JavaScript,比如下拉进度条''' driver=webdriver.Chrome() try: driver.get('https://www.zhihu.com/explore') driver.execute_script('window.scrollTo(0,document.body.scrollHeight)') driver.execute_script('alert("To Bottom")') except: print('error') driver.close() '''获取属性''' driver=webdriver.Chrome() try: driver.get('https://www.zhihu.com/explore') logo=driver.find_element(By.ID,'special') print(logo) print(logo.get_attribute('class')) driver.close() except: print('error') driver.close() '''获取文本''' driver=webdriver.Chrome() try: driver.get('https://www.zhihu.com/explore') res=driver.find_element(By.CLASS_NAME,'ExploreSpecialCard-contentTitle') print(res.text) driver.close() except: print('error') driver.close() '''获取id、位置、标签吗和大小''' driver=webdriver.Chrome() try: driver.get('https://www.zhihu.com/explore') res=driver.find_element(By.CLASS_NAME,'ExploreSpecialCard-contentTitle') print('id',res.id) print('位置',res.location) print('标签',res.tag_name) print('大小',res.size) driver.close() except: print('error') driver.close() '''延时显式等待''' driver=webdriver.Chrome() try: driver.get('https://www.taobao.com') #引入WebDriverWait对象，设置最长等待时间 wait=WebDriverWait(driver,10) #调用until方法，传入等待条件;EC.presence_of_element_located代表节点出现 input=wait.until(EC.presence_of_element_located((By.ID,'q'))) button=wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'.btn-search'))) print(input,button) driver.close() except: print('error') driver.close() '''前进和后退''' driver=webdriver.Chrome() try: driver.get('https://www.taobao.com/') driver.get('https://www.baidu.com/') driver.get('https://www.python.org/') driver.back() #后退 time.sleep(1) driver.forward() #前进 driver.close() except: print('error') driver.close()
import pymysql from pyquery import PyQuery as pq import requests def Get_HTML(url): headers = {'User-Agent': 'Mozilla/5.0'} try: r = requests.get(url, headers=headers) r.raise_for_status() r.encoding = r.apparent_encoding return r.text except: print("Error!") def parse_page(html): doc=pq(html) title=doc('.li03 a').text().split(' ') #名称 actor=doc('.li04 span').text().split(' ') #演员表 index=doc('.li01 b').text().split(' ') #序号 href=[] #链接 for i in doc('.li03 a').items(): #遍历 href.append(i.attr('href')) score=doc('.li05 span').text().split(' ') #评分 #保存为字典 Result={} for i in range(100): result={ '序号':index[i], '名称':title[i], '链接':href[i], '演员':actor[i], '评分':score[i] } Result[index[i]]=result return Result def write_to_mysql(dict): db=pymysql.connect(host='*******',user='root',password='******',passwd='3306',db='*******') cursor=db.cursor() table='movie' #表名 keys=','.join(dict.keys()) values=','.join(['%s']*len(dict)) sql='insert into {table}({keys}) values ({values})'.format(table=table,keys=keys,values=values) try: if cursor.execute(sql,tuple(dict.values())): print("successful") db.commit() except: print("failed") db.rollback() db.close() #若主键存在，进行更新操作；否则进行插入数据 def update_data(dict): db = pymysql.connect(host='*******',user='root',password='******',passwd='3306',db='*******') cursor=db.cursor() table='movie' #表名 keys=','.join(dict.keys()) values=','.join(['%s']*len(dict)) sql='insert into {table}({keys}) values ({values}) ON DUPLICATE KEY UPDATE'.format(table=table,keys=keys,values=values) update=','.join([' {key}=%s'.format(key=key) for key in dict]) sql+=update try: if cursor.execute(sql,tuple(dict.values())*2): print("successful") db.commit() except: print("failed") db.rollback() db.close() #查询 def search(): db = pymysql.connect(host='*******',user='root',password='******',passwd='3306',db='*******') cursor = db.cursor() sql = 'select * from movie' try: cursor.execute(sql) print(cursor.rowcount) row = cursor.fetchone() while row: print('Row:', row) row = cursor.fetchone() except: print("Error") db.close() def main(): url = 'https://www.1905.com/vod/top/lst/' html = Get_HTML(url) item=parse_page(html) for i in range(len(item)): write_to_mysql(item[str(i+1)]) main()
import requests import json import re import pandas as pd import time from pyecharts.components import Table from pyecharts import options as opts from pyecharts.charts import Bar, Page,Pie,Line,Timeline,Map from pyecharts.commons.utils import JsCode from pyecharts.options import DataZoomOpts,ComponentTitleOpts def Get_HTML(): headers = {'User-Agent': 'Mozilla/5.0'} url = 'https://m.look.360.cn/events/feiyan?sv=&version=&market=&device=2&net=4&stype=&scene=&sub_scene=&refer_scene=&refer_subscene=&f=jsonp&location=true&sort=2&_=1649252949072&callback=jsonp2' try: r = requests.get(url, headers=headers) r.raise_for_status() r.encoding = r.apparent_encoding re = r.text[7:-2] response = json.loads(re) return response except: print("error!") def China_data_Total(): url='http://m.look.360.cn/subject/400?sign=360_6aa05217' headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0'} html = requests.get(url, headers=headers).text pattern=re.compile('"qcmsint1":"0"},"feiyanTrend":(.*),"local":',re.S) item=re.findall(pattern,html) response=json.loads(item[0]) data = { "截止日期":response['total']['modifyTime'], "现存确诊":response['total']['currentConfirmed'], "境外输入":response['total']['overseasInput'], "现存无症状":response['total']['asymptom'], "累计确诊":response['total']['diagnosed'], "累计治愈":response['total']['cured'], "累计死亡":response['total']['died'], "现存_较昨日":response['newDiff']['currentConfirmed'], "境外输入_较昨日":response['newDiff']['overseasInput'], "无症状_较昨日":response['newDiff']['asymptom'], "累计_较昨日":response['newDiff']['diagnosed'], "治愈_较昨日":response['newDiff']['cured'], "死亡_较昨日":response['newDiff']['died'] } return data # 国内数据 def China_data(response): P_name, P_Ljqz, P_Cured, P_Died, P_Xcqz, P_Xzqz = [], [], [], [], [], [] #省数据 C_name, C_Ljqz, C_Cured, C_Died, C_Xcqz, C_Xzqz = [], [], [], [], [], [] #市数据 for i in range(34): P_name.append(response['data'][i]['cityShortName']) # 城市名称 P_Ljqz.append(response['data'][i]['diagnosed']) # 累计确诊 P_Cured.append(response['data'][i]['cured']) # 治愈 P_Died.append(response['data'][i]['died']) # 死亡 P_Xcqz.append(response['data'][i]['currentConfirmed']) # 现存确诊 P_Xzqz.append(response['data'][i]['diffDiagnosed']) # 新增确诊 data = { "地区": P_name, "新增确诊": P_Xzqz, "现存确诊": P_Xcqz, "累计确诊": P_Ljqz, "治愈人数": P_Cured, "死亡人数": P_Died, } return data #河南省数据 def Henan_data(): url = 'https://m.look.360.cn/events/feiyanCityInfo?sv=&version=&market=&device=2&net=4&stype=&scene=&sub_scene=&refer_scene=&refer_subscene=&f=jsonp&ename=henan&_=1650426134574&callback=jsonp4' headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0'} html = requests.get(url, headers=headers).text[7:-2] response=json.loads(html) # print(response['data']['cityInfo']) C_name, C_Ljqz, C_Cured, C_Died, C_Xcqz, C_Xzqz = [], [], [], [], [], [] # 市数据 for i in range(len(response['data']['cityInfo'])): C_name.append(response['data']['cityInfo'][i]['cityName']) #名称 C_Ljqz.append(response['data']['cityInfo'][i]['diagnosed']) # 累计确诊 C_Cured.append(response['data']['cityInfo'][i]['cured']) # 治愈 C_Died.append(response['data']['cityInfo'][i]['died']) # 死亡 C_Xzqz.append(response['data']['cityInfo'][i]['diffDiagnosed']) # 新增确诊 data = { "地区": C_name, "新增确诊": C_Xzqz, "累计确诊": C_Ljqz, "治愈人数": C_Cured, "死亡人数": C_Died } return data def Abroad_data(response): # 国外数据 C_name, Ljqz, Cured, Died, Xcqz, Xzqz = [], [], [], [], [], [] for i in range(196): c_name = response['country'][i]['provinceName'] # 城市名称 ljqz = response['country'][i]['diagnosed'] # 累计确诊 cured = response['country'][i]['cured'] # 治愈 died = response['country'][i]['died'] # 死亡 xzqz = response['country'][i]['diffDiagnosed'] # 新增确诊 C_name.append(c_name) Ljqz.append(ljqz) Cured.append(cured) Died.append(died) Xzqz.append(xzqz) data = { "地区": C_name, "新增确诊": Xzqz, "累计确诊": Ljqz, "治愈人数": Cured, "死亡人数": Died } return data def TimeLine(): headers = {'User-Agent': 'Mozilla/5.0'} url = 'https://api.look.360.cn/events/feiyanHomeMulTrend?sv=&version=&market=&device=2&net=4&stype=&scene=&sub_scene=&refer_scene=&refer_subscene=&f=jsonp&_=1650032782170&callback=jsonp3' try: r = requests.get(url, headers=headers) r.raise_for_status() r.encoding = r.apparent_encoding re = r.text[7:-2] response = json.loads(re) except: print("error!") time,Ljqz, Cured, Died, Xcqz, Xzqz, Xz_Died, Xz_Cured= [], [], [], [], [], [], [], [] for i in range(len(response['data']['timeline_n1'])): time.append(response['data']['timeline_n1'][i]['time']) #时间 Xzqz.append(response['data']['timeline_n1'][i]['diagnosed']) #新增确诊 Xcqz.append(response['data']['timeline_n2'][i]['total_currentConfirmed']) #现存确诊 Ljqz.append(response['data']['timeline_n2'][i]['total_diagnosed']) #累计确诊 Cured.append(response['data']['timeline_n3'][i]['total_cured']) #累计治愈 Died.append(response['data']['timeline_n3'][i]['total_died']) #累计死亡 Xz_Died.append(response['data']['timeline_n4'][i]['died']) #新增死亡 Xz_Cured.append(response['data']['timeline_n4'][i]['cured']) #新增治愈 data={ "时间":time, "新增确诊":Xzqz, "现存确诊":Xcqz, "累计确诊":Ljqz, "累计治愈":Cured, "累计死亡":Died, "新增死亡":Xz_Died, "新增治愈":Xz_Cured } return data def View(data_1,data_2,data_3,data_4): #柱状图 c_1 = Timeline(init_opts=opts.InitOpts(chart_id='1')) c_1.add_schema(play_interval=1000, label_opts=opts.series_options.LabelOpts(color='#cfe2f3'),is_auto_play = True) for i in range(3): #全国累计详情柱状图 C_1 = ( Bar() .add_xaxis(data_1['地区'][i*11:(i+1)*11]) .add_yaxis("累计确诊", data_1['累计确诊'][i*11:(i+1)*11],stack='stack1',label_opts=opts.LabelOpts(is_show=False)) .add_yaxis("新增确诊", data_1['新增确诊'][i*11:(i+1)*11],stack='stack1',label_opts=opts.LabelOpts(is_show=False)) .add_yaxis("死亡人数", data_1['死亡人数'][i * 11:(i + 1) * 11],stack='stack1',label_opts=opts.LabelOpts(is_show=False)) .add_yaxis("治愈人数", data_1['治愈人数'][i * 11:(i + 1) * 11],stack='stack1',label_opts=opts.LabelOpts(is_show=False)) .add_yaxis("现存确诊", data_1['现存确诊'][i * 11:(i + 1) * 11],stack='stack1',label_opts=opts.LabelOpts(is_show=False)) .reversal_axis() # .set_global_opts(legend_opts=opts.LegendOpts(selected_mode='single')) # 单选模式 .set_global_opts(xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(color='white')),#调整x轴字体颜色 yaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(color='white')), legend_opts=opts.LegendOpts(selected_mode='single'), title_opts=opts.TitleOpts("国内疫情详情",pos_right=0,pos_top=20,title_textstyle_opts=opts.TextStyleOpts(font_size=13,color="#cfe2f3"))) ) c_1.add(C_1, "第{}页".format(i+1)) # 全国地图 c_2 = ( Map(init_opts=opts.InitOpts(chart_id='2')) # .add("死亡人数", [list(z) for z in zip(data_1['地区'], data_1['死亡人数'])], "china",is_selected=False) #默认关闭该标签 .add("累计确诊",[list(z) for z in zip(data_1['地区'], data_1['累计确诊'])], "china") .add("死亡人数", [list(z) for z in zip(data_1['地区'], data_1['死亡人数'])], "china") .add("治愈人数", [list(z) for z in zip(data_1['地区'], data_1['治愈人数'])], "china") .add("新增确诊", [list(z) for z in zip(data_1['地区'], data_1['新增确诊'])], "china") .add("现存确诊", [list(z) for z in zip(data_1['地区'], data_1['现存确诊'])], "china") .set_global_opts( title_opts=opts.TitleOpts(""), visualmap_opts=opts.VisualMapOpts(max_=10000), legend_opts=opts.LegendOpts(selected_mode='single') #单选模式 ) ) #时间线折线图 c_3 = ( Line(init_opts=opts.InitOpts(chart_id='3')) .add_xaxis(data_3['时间']) .add_yaxis("新增确诊",data_3['新增确诊'],label_opts=opts.LabelOpts(is_show=False),linestyle_opts=opts.LabelOpts(color='#DC143C')) .add_yaxis("现存确诊",data_3['现存确诊'],label_opts=opts.LabelOpts(is_show=False),linestyle_opts=opts.LabelOpts(color='#a98175')) .add_yaxis("累计确诊",data_3['累计确诊'], label_opts=opts.LabelOpts(is_show=False),linestyle_opts=opts.LabelOpts(color='#60281e')) .add_yaxis("累计治愈",data_3['累计治愈'], label_opts=opts.LabelOpts(is_show=False),linestyle_opts=opts.LabelOpts(color='#2add9c')) .add_yaxis("累计死亡",data_3['累计死亡'], label_opts=opts.LabelOpts(is_show=False),linestyle_opts=opts.LabelOpts(color='black')) .add_yaxis("新增死亡",data_3['新增死亡'], label_opts=opts.LabelOpts(is_show=False),linestyle_opts=opts.LabelOpts(color='#758a99')) .add_yaxis("新增治愈",data_3['新增治愈'], label_opts=opts.LabelOpts(is_show=False),linestyle_opts=opts.LabelOpts(color='#1bd1a5')) .set_global_opts( title_opts=opts.TitleOpts(title="疫情汇总时间线",pos_top=20,title_textstyle_opts=opts.TextStyleOpts(font_size=13,color="#cfe2f3")), tooltip_opts=opts.TooltipOpts(trigger="axis"), xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(color='white')), # 调整x轴字体颜色 yaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(color='white')), legend_opts=opts.LegendOpts(selected_mode='single') ) ) #河南省地图 c_4 = ( Map(init_opts=opts.InitOpts(chart_id='4')) # .add("死亡人数", [list(z) for z in zip(data_1['地区'], data_1['死亡人数'])], "china",is_selected=False) #默认关闭该标签 .add("累计确诊", [list(z) for z in zip(data_4['地区'], data_4['累计确诊'])], "河南") .add("死亡人数", [list(z) for z in zip(data_4['地区'], data_4['死亡人数'])], "河南") .add("治愈人数", [list(z) for z in zip(data_4['地区'], data_4['治愈人数'])], "河南") .add("新增确诊", [list(z) for z in zip(data_4['地区'], data_4['新增确诊'])], "河南") .set_global_opts( title_opts=opts.TitleOpts(title='省内详细信息',pos_top=30,title_textstyle_opts=opts.TextStyleOpts(font_size=13, color="#cfe2f3")), visualmap_opts=opts.VisualMapOpts(max_=500), legend_opts=opts.LegendOpts(selected_mode='single') # 单选模式 ) ) #省内死亡详情饼图 c_5 = ( Pie(init_opts=opts.InitOpts(chart_id='5')) .add("死亡人数", [list(z) for z in zip(data_4['地区'], data_4['死亡人数'])]) .set_global_opts(title_opts=opts.TitleOpts('市死亡人数',title_textstyle_opts=opts.TextStyleOpts(font_size=13, color="gray"),), legend_opts=opts.LegendOpts(is_show=False)) # .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"),) ) # 汇总数据与昨日相比 data = China_data_Total() c_7 = ( Pie(init_opts=opts.InitOpts(chart_id='7')) .set_global_opts( title_opts=opts.TitleOpts( title="现存确诊" + " " * 17 + "境外输入" + " " * 17 + "现存无症状", title_textstyle_opts=opts.TextStyleOpts(font_size=10, color="white"), )) ) c_8 = ( Pie(init_opts=opts.InitOpts(chart_id='8')) .set_global_opts( title_opts=opts.TitleOpts( title="累计确诊"+ " " * 17 + "累计治愈"+ " " * 17 + "累计死亡", title_textstyle_opts=opts.TextStyleOpts(font_size=10, color="white"), )) ) c_9 = ( Pie(init_opts=opts.InitOpts(chart_id='9')) .set_global_opts( title_opts=opts.TitleOpts( title=str(data['现存_较昨日']) + " " * 25 + str(data['境外输入_较昨日']) + " " * 25 + str(data['无症状_较昨日']), title_textstyle_opts=opts.TextStyleOpts(font_size=10, color="#e9e7ef"), )) ) c_10 = ( Pie(init_opts=opts.InitOpts(chart_id='10')) .set_global_opts( title_opts=opts.TitleOpts( title=str(data['累计_较昨日']) + " " * 25 + str(data['治愈_较昨日']) + " " * 25+ str(data['死亡_较昨日']), title_textstyle_opts=opts.TextStyleOpts(font_size=10, color="#e9e7ef"))) ) #数字 c_11 = ( Pie(init_opts=opts.InitOpts(chart_id='16')) .set_global_opts( title_opts=opts.TitleOpts( title=str(data['现存确诊']) + " " * 3 + str(data['境外输入']) + " " * 3 + str(data['现存无症状']), title_textstyle_opts=opts.TextStyleOpts(font_size=23, color="#CD3700"))) ) c_12 = ( Pie(init_opts=opts.InitOpts(chart_id='17')) .set_global_opts( title_opts=opts.TitleOpts( title=str(data['累计确诊']) + " " * 3 + str(data['累计治愈']) + " " * 3 + str(data['累计死亡']), title_textstyle_opts=opts.TextStyleOpts(font_size=23, color="#7F7F7F"))) ) tu_1 = ( Line(init_opts=opts.InitOpts(width="1500px", height="850px", bg_color={"type": "pattern", "image": JsCode("img"), "repeat": "no-repeat"}, chart_id='11')) .add_xaxis([None]) .add_yaxis("", [None]) .set_global_opts( title_opts=opts.TitleOpts(), yaxis_opts=opts.AxisOpts(is_show=False), xaxis_opts=opts.AxisOpts(is_show=False)) ) tu_1.add_js_funcs( """ var img = new Image(); img.src = './images/bg_body.jpg'; """ ) tu_2 = ( Line(init_opts=opts.InitOpts( bg_color={"type": "pattern", "image": JsCode("img"), "repeat": "no-repeat"}, chart_id='12')) .add_xaxis([None]) .add_yaxis("", [None]) .set_global_opts( yaxis_opts=opts.AxisOpts(is_show=False), xaxis_opts=opts.AxisOpts(is_show=False)) ) tu_2.add_js_funcs( """ var img = new Image(); img.src = './images/bg_box5.png'; """ ) tu_3 = ( Line(init_opts=opts.InitOpts(chart_id='13', bg_color={"type": "pattern", "image": JsCode("img"), "repeat": "no-repeat"}, )) .add_xaxis([None]) .add_yaxis("", [None]) .set_global_opts( yaxis_opts=opts.AxisOpts(is_show=False), xaxis_opts=opts.AxisOpts(is_show=False)) ) tu_3.add_js_funcs( """ var img = new Image(); img.src = './images/bg_box5.png'; """ ) tu_4 = ( Line(init_opts=opts.InitOpts(chart_id='14', bg_color={"type": "pattern", "image": JsCode("img"), "repeat": "no-repeat"}, )) .add_xaxis([None]) .add_yaxis("", [None]) .set_global_opts( yaxis_opts=opts.AxisOpts(is_show=False), xaxis_opts=opts.AxisOpts(is_show=False)) ) tu_4.add_js_funcs( """ var img = new Image(); img.src = './images/bg_box4.png'; """ ) tu_5 = ( Line(init_opts=opts.InitOpts(width="1500px", height="200px", bg_color={"type": "pattern", "image": JsCode("img"), "repeat": "no-repeat"}, chart_id='15')) .add_xaxis([None]) .add_yaxis("", [None]) .set_global_opts( title_opts=opts.TitleOpts(title="全球疫情信息 截至" + data['截止日期'], pos_left='center', title_textstyle_opts=opts.TextStyleOpts(font_size=21, color='#51c2d5'), pos_top='23%'), yaxis_opts=opts.AxisOpts(is_show=False), xaxis_opts=opts.AxisOpts(is_show=False)) ) tu_5.add_js_funcs( """ var img = new Image(); img.src = './images/bg_title.png'; """ ) page = Page(layout=Page.DraggablePageLayout, page_title="可视化大屏") page.add(tu_1,tu_2,tu_3,tu_4,c_1,c_2,c_3,c_4,c_5,c_7,c_8,c_9,c_10,tu_5,c_11,c_12) page.render('国内疫情信息.html') def Style(): # 固定样式 Page.save_resize_html(source="./国内疫情信息.html", # 上面的HTML文件名称 cfg_file="chart_config_3.json", # 保存的json配置文件 dest="new_国内疫情信息可视化大屏.html" # 新HTML文件名称，可以空缺，默认resize_render.html ) def main(): a=0 while True: data_1 = China_data(Get_HTML()) data_2 = Abroad_data(Get_HTML()) data_3 = TimeLine() data_4=Henan_data() date=China_data_Total()['截止日期'] if date!=a: View(data_1, data_2, data_3,data_4) Style() a=date main()
# -*- coding:utf-8 -*- import requests import json import re import pandas as pd import time from pyecharts import options as opts from pyecharts.charts import Bar, Page,Pie,Line,Timeline,Grid,Map from pyecharts.commons.utils import JsCode def sleeptime(hour, min, sec): # 时间转换 return hour * 3600 + min * 60 + sec def Get_HTML(): headers = {'User-Agent': 'Mozilla/5.0'} url = 'https://m.look.360.cn/events/feiyan?sv=&version=&market=&device=2&net=4&stype=&scene=&sub_scene=&refer_scene=&refer_subscene=&f=jsonp&location=true&sort=2&_=1649252949072&callback=jsonp2' try: r = requests.get(url, headers=headers) r.raise_for_status() r.encoding = r.apparent_encoding re = r.text[7:-2] response = json.loads(re) return response except: print("error!") def China_data_Total(): url='http://m.look.360.cn/subject/400?sign=360_6aa05217' headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0'} html = requests.get(url, headers=headers).text pattern=re.compile('"qcmsint1":"0"},"feiyanTrend":(.*),"local":',re.S) item=re.findall(pattern,html) response=json.loads(item[0]) data = { "截止日期":response['total']['modifyTime'], "现存确诊":response['total']['currentConfirmed'], "境外输入":response['total']['overseasInput'], "现存无症状":response['total']['asymptom'], "累计确诊":response['total']['diagnosed'], "累计治愈":response['total']['cured'], "累计死亡":response['total']['died'], "现存_较昨日":response['newDiff']['currentConfirmed'], "境外输入_较昨日":response['newDiff']['overseasInput'], "无症状_较昨日":response['newDiff']['asymptom'], "累计_较昨日":response['newDiff']['diagnosed'], "治愈_较昨日":response['newDiff']['cured'], "死亡_较昨日":response['newDiff']['died'] } return data def China_data(response): # 国内数据 C_name, Ljqz, Cured, Died, Xcqz, Xzqz = [], [], [], [], [], [] for i in range(34): c_name = response['data'][i]['city'] # 城市名称 ljqz = response['data'][i]['diagnosed'] # 累计确诊 cured = response['data'][i]['cured'] # 治愈 died = response['data'][i]['died'] # 死亡 xcqz = response['data'][i]['currentConfirmed'] # 现存确诊 xzqz = response['data'][i]['diffDiagnosed'] # 新增确诊 C_name.append(c_name) Ljqz.append(ljqz) Cured.append(cured) Died.append(died) Xcqz.append(xcqz) Xzqz.append(xzqz) data = { "地区": C_name, "新增确诊": Xzqz, "现存确诊": Xcqz, "累计确诊": Ljqz, "治愈人数": Cured, "死亡人数": Died } return data def Abroad_data(response): # 国外数据 C_name, Ljqz, Cured, Died, Xcqz, Xzqz = [], [], [], [], [], [] for i in range(196): c_name = response['country'][i]['provinceName'] # 城市名称 ljqz = response['country'][i]['diagnosed'] # 累计确诊 cured = response['country'][i]['cured'] # 治愈 died = response['country'][i]['died'] # 死亡 xzqz = response['country'][i]['diffDiagnosed'] # 新增确诊 C_name.append(c_name) Ljqz.append(ljqz) Cured.append(cured) Died.append(died) Xzqz.append(xzqz) data = { "地区": C_name, "新增确诊": Xzqz, "累计确诊": Ljqz, "治愈人数": Cured, "死亡人数": Died } return data def View(data_1, data_2): # 可视化 # 柱状图 area_color_js = ( "new echarts.graphic.LinearGradient(0, 0, 0, 1, " "[{offset: 0, color: '#7bbfea'}, {offset: 1, color: '#3fbbff0d'}], false)" ) t_1 = Timeline() t_1.add_schema(play_interval=1000, label_opts=opts.series_options.LabelOpts( color='#cfe2f3')) for i in range(len(data_1['地区'])): c_1 = ( Bar(init_opts=opts.InitOpts(width="300px",height='300px')) .add_xaxis(['新增确诊','现存确诊','累计确诊','治愈人数','死亡人数'],) .add_yaxis("",[data_1['新增确诊'][i],data_1['现存确诊'][i],data_1['累计确诊'][i],data_1['治愈人数'][i],data_1['死亡人数'][i]], color='#44cef6',) .reversal_axis() .set_global_opts( title_opts=opts.TitleOpts(" (地区: {} )".format(data_1['地区'][i]),title_textstyle_opts=opts.TextStyleOpts(color="#cfe2f3")), xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(color='white')),#调整x轴字体颜色 yaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(color='white')) ) ) t_1.add(c_1, "{}".format(data_1['地区'][i])) # 柱状图 t_2 = Timeline() t_2.add_schema(play_interval=1000, label_opts=opts.series_options.LabelOpts(color='#cfe2f3')) for i in range(len(data_2['地区'])): c_2 = ( Bar() .add_xaxis(['新增确诊', '累计确诊', '治愈人数', '死亡人数'], ) .add_yaxis("", [data_2['新增确诊'][i], data_2['累计确诊'][i], data_2['治愈人数'][i],data_2['死亡人数'][i]], label_opts=opts.LabelOpts(position="right"), color='#44cef6', ) .reversal_axis() .set_global_opts( title_opts=opts.TitleOpts(" (国家: {} )".format(data_2['地区'][i]),pos_right='0',title_textstyle_opts=opts.TextStyleOpts(color="#cfe2f3")), xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(color='white',)), # 调整x轴字体颜色 yaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(color='white'),position='right') ) ) t_2.add(c_2, "{}".format(data_2['地区'][i])) # 饼图 # 转换数据为类型，#转换后数据为['美国', 1010774] # data_pair = [list(z) for z in zip(x_data, y_data)] # 转换后数据为['美国', 1010774] c_3 = ( Line() .add_xaxis(data_2['地区']) .add_yaxis( series_name="新增确诊", y_axis=data_2['新增确诊'], label_opts=opts.LabelOpts(is_show=False,), ) .add_yaxis( series_name="累计确诊", y_axis=data_2['累计确诊'], label_opts=opts.LabelOpts(is_show=False,), ) .add_yaxis( series_name="治愈人数", y_axis=data_2['治愈人数'], label_opts=opts.LabelOpts(is_show=False,), ) .add_yaxis( series_name="死亡人数", y_axis=data_2['死亡人数'], label_opts=opts.LabelOpts(is_show=False,), ) .set_global_opts( title_opts=opts.TitleOpts(title="国外疫情数据",pos_right='0',title_textstyle_opts=opts.TextStyleOpts(color="#cfe2f3")), datazoom_opts=opts.DataZoomOpts(), tooltip_opts=opts.TooltipOpts(trigger="axis"), xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(color='white', )), # 调整x轴字体颜色 yaxis_opts=opts.AxisOpts( position='right', axistick_opts=opts.AxisTickOpts(is_show=True,), splitline_opts=opts.SplitLineOpts(is_show=True), axislabel_opts=opts.LabelOpts(color='white') ), ) ) c_5 = ( #地图 Map() .add("累计确诊", [['河南', 2840], ['香港', 308175], ['吉林', 37712] , ['台湾', 27410], ['上海', 11366], ['浙江', 2819], ['福建', 2905],['山东', 2646], ['黑龙江', 2505], ['广东', 6681], ['四川', 1948], ['辽宁', 1604], ['河北', 1981], ['江苏', 2083], ['广西', 1518], ['北京', 1821], ['海南', 243], ['山西', 350], ['陕西', 3247], ['天津', 1798], ['云南', 2079], ['安徽', 1054], ['江西', 1063], ['湖南', 1335], ['青海', 43], ['内蒙古', 1687],['重庆',2423], ['贵州', 178], ['湖北', 68392], ['甘肃', 681], ['新疆', 999], ['宁夏', 122], ['澳门', 82], ['西藏', 1]], "china") .set_global_opts( title_opts=opts.TitleOpts(title="国内现存确诊",title_textstyle_opts=opts.TextStyleOpts(color="#cfe2f3")),visualmap_opts=opts.VisualMapOpts(max_=10000)) ) data = China_data_Total() c_6 = ( Pie(init_opts=opts.InitOpts()) .set_global_opts( title_opts=opts.TitleOpts( title="现存确诊:" + str(data['现存确诊']) + " " * 5 + "境外输入:" + str(data['境外输入']) + " " * 5 + "现存无症状:" + str( data['现存无症状']), title_textstyle_opts=opts.TextStyleOpts(font_size=15, color="#ffb3a7"), )) ) c_7 = ( Pie(init_opts=opts.InitOpts()) .set_global_opts( title_opts=opts.TitleOpts( title="累计确诊:" + str(data['累计确诊']) + " " * 5 + "累计治愈:" + str(data['累计治愈']) + " " * 5 + "累计死亡:" + str( data['累计死亡']), title_textstyle_opts=opts.TextStyleOpts(font_size=15, color="#ffb3a7"), )) ) c_8 = ( Pie(init_opts=opts.InitOpts()) .set_global_opts( title_opts=opts.TitleOpts( title="较昨日:" + str(data['现存_较昨日']) + " " * 25 + "较昨日:" + str(data['境外输入_较昨日']) + " " * 25 + "较昨日:" + str( data['无症状_较昨日']), title_textstyle_opts=opts.TextStyleOpts(font_size=10, color="#e9e7ef"), )) ) c_9 = ( Pie(init_opts=opts.InitOpts()) .set_global_opts( title_opts=opts.TitleOpts( title="较昨日:" + str(data['累计_较昨日']) + " " * 25 + "较昨日:" + str(data['治愈_较昨日']) + " " * 25 + "较昨日:" + str( data['死亡_较昨日']), title_textstyle_opts=opts.TextStyleOpts(font_size=10, color="#e9e7ef"), )) ) c_10 = ( # 地图 Map() .add("死亡人数",[['河南', 22], ['香港', 8827], ['吉林', 5], ['台湾', 854], ['上海', 7], ['浙江', 1], ['福建', 1], ['山东', 7], ['广东', 8], ['黑龙江', 13], ['四川', 3], ['江苏', 0], ['河北', 7], ['辽宁', 2], ['广西', 2], ['北京', 9], ['海南', 6], ['山西', 0], ['陕西', 3], ['云南', 2], ['安徽', 6], ['湖南', 4], ['江西', 1], ['天津', 3], ['青海', 0], ['内蒙古', 1], ['重庆', 6], ['贵州', 2], ['湖北', 4512], ['甘肃', 2], ['新疆', 3], ['宁夏', 0], ['澳门', 0], ['西藏', 0]], "china") .set_global_opts( title_opts=opts.TitleOpts(title="国内各地死亡人数", title_textstyle_opts=opts.TextStyleOpts(color="#cfe2f3")), visualmap_opts=opts.VisualMapOpts(max_=10000)) ) tu_1 = ( Line(init_opts=opts.InitOpts(width="1500px", height="850px", bg_color={"type": "pattern", "image": JsCode("img"), "repeat": "no-repeat"})) .add_xaxis([None]) .add_yaxis("", [None]) .set_global_opts( title_opts=opts.TitleOpts(), yaxis_opts=opts.AxisOpts(is_show=False), xaxis_opts=opts.AxisOpts(is_show=False)) ) tu_1.add_js_funcs( """ var img = new Image(); img.src = './images/bg_body.jpg'; """ ) tu_2 = ( Line(init_opts=opts.InitOpts(width="600px", height="700px", bg_color={"type": "pattern", "image": JsCode("img"), "repeat": "no-repeat"})) .add_xaxis([None]) .add_yaxis("", [None]) .set_global_opts( title_opts=opts.TitleOpts( pos_left='center', title_textstyle_opts=opts.TextStyleOpts(font_size=25, color='#51c2d5'), pos_top='5%'), yaxis_opts=opts.AxisOpts(is_show=False), xaxis_opts=opts.AxisOpts(is_show=False)) ) tu_2.add_js_funcs( """ var img = new Image(); img.src = './images/data_icon.png'; """ ) tu_3 = ( Line(init_opts=opts.InitOpts(width="1500px", height="200px", bg_color={"type": "pattern", "image": JsCode("img"), "repeat": "no-repeat"})) .add_xaxis([None]) .add_yaxis("", [None]) .set_global_opts( title_opts=opts.TitleOpts(title="全球疫情信息 截至"+data['截止日期'], pos_left='center', title_textstyle_opts=opts.TextStyleOpts(font_size=21, color='#51c2d5'), pos_top='5%'), yaxis_opts=opts.AxisOpts(is_show=False), xaxis_opts=opts.AxisOpts(is_show=False)) ) tu_3.add_js_funcs( """ var img = new Image(); img.src = './images/bg_title.png'; """ ) # page = Page(layout=Page.DraggablePageLayout, page_title="模拟") # page.add(tu_1,t_1, t_2, c_3,c_5,c_6,c_7,c_8,c_9,tu_2,tu_3,c_10) # page.render('全球疫情信息.html') # 固定样式 Page.save_resize_html(source="全球疫情信息.html", # 上面的HTML文件名称 cfg_file="chart_config.json", # 保存的json配置文件 dest="new_全球疫情信息.html" # 新HTML文件名称，可以空缺，默认resize_render.html ) def main(): second = sleeptime(5, 0, 0) # 设置时间 data_1 = China_data(Get_HTML()) # 国内数据 data_2 = Abroad_data(Get_HTML()) # 国外数据 data_11 = pd.DataFrame(data_1) data_22 = pd.DataFrame(data_2) while True: data_11.to_csv('./国内疫情数据.csv') data_22.to_csv('./国外疫情数据.csv') View(data_1, data_2) time.sleep(second) # main() data_1 = China_data(Get_HTML()) # 国内数据 data_2 = Abroad_data(Get_HTML()) # 国外数据 data_11 = pd.DataFrame(data_1) data_22 = pd.DataFrame(data_2) # data_11 # data_22 View(data_1,data_2) # print(data_1)
import requests import json from lxml import etree import pandas as pd import time from pyecharts import options as opts from pyecharts.charts import Bar def sleeptime(hour, min, sec): #时间转换 return hour * 3600 + min * 60 + sec def Get_HTML(url): headers={'User-Agent':'Mozilla/5.0'} try: r=requests.get(url,headers=headers) r.raise_for_status() r.encoding=r.apparent_encoding return r.text except: print("error!") def Parse_page(): C_name,Ljqz,Cured,Died,Xcqz,Xzqz=[],[],[],[],[],[] url='https://m.look.360.cn/events/feiyan?sv=&version=&market=&device=2&net=4&stype=&scene=&sub_scene=&refer_scene=&refer_subscene=&f=jsonp&location=true&sort=2&_=1649252949072&callback=jsonp2' re=Get_HTML(url) re=re[7:-2] response=json.loads(re) for i in range(34): c_name=response['data'][i]['data']['provinceName'] #城市名称 ljqz=response['data'][i]['diagnosed'] #累计确诊 cured=response['data'][i]['cured'] #治愈 died=response['data'][i]['died'] #死亡 xcqz=response['data'][i]['currentConfirmed'] #现存确诊 xzqz=response['data'][i]['diffDiagnosed'] #新增确诊 C_name.append(c_name) Ljqz.append(ljqz) Cured.append(cured) Died.append(died) Xcqz.append(xcqz) Xzqz.append(xzqz) # columns=['新增确诊','现存确诊','累计确诊','治愈人数','死亡人数'] # index=C_name data={ "地区":C_name, "新增确诊":Xzqz, "现存确诊":Xcqz, "累计确诊":Ljqz, "治愈人数":Cured, "死亡人数":Died } # print(data) return data def View(): data=Parse_page() c = ( Bar() .add_xaxis(data['地区']) .add_yaxis("新增确诊",data['新增确诊'], stack="stack1") .add_yaxis("现存确诊",data['现存确诊'] , stack="stack1") .add_yaxis("累计确诊",data['累计确诊'] , stack="stack1") .add_yaxis("治愈人数",data['治愈人数'] , stack="stack1") .add_yaxis("死亡人数",data['死亡人数'] , stack="stack1") .set_global_opts( title_opts=opts.TitleOpts(title="国内疫情数据数据（全部）"), datazoom_opts=opts.DataZoomOpts(type_="inside"), ) .render("国内疫情数据数据.html") ) def main(): data=Parse_page() data=pd.DataFrame(data) second = sleeptime(24,0,0) while True: # print(data) data.to_csv('./国内疫情数据.csv') View() time.sleep(second) # main() data=Parse_page() data=pd.DataFrame(data) data