2022年 11月 8日

【Python】从新浪抓取美国天然气NG历史交易数据并进行可视化

 
  1. import urllib.request
  2. import re
  3. import datetime
  4. from bs4 import BeautifulSoup
  5. import matplotlib.pyplot as plt
  6. from matplotlib.ticker import MultipleLocator, FormatStrFormatter
  7. import time
  8. import matplotlib.ticker as ticker
  9. import re
  10. import json
  11.  
  12. from urllib import request
  13. from urllib import parse
  14.  
  15. # 设置全局变量，用于保存提取的数据
  16. HISTORY_ALL = []#存放所有历史数据
  17. div_list = []#存放网页中的div表
  18. infos = ['日期','收盘价', '开盘价','最高价', '最低价','成交量']
  19. PER_PAGE_DAYS = 25    #单个网页爬取的天数
  20.  
  21. cyc_val = 17.502  #设置成本价
  22.  
  23. def getHtml(url):
  24.     """
  25.     打开网页
  26.     :param url:
  27.     :return:
  28.     """
  29.     while True:
  30.         try:
  31.             html = urllib.request.urlopen(url, timeout=5).read()
  32.             break
  33.         except:
  34.             print("超时重试")
  35.     html = html.decode('gbk')
  36.     return html
  37. def time_long(time1, time2, type="day"):
  38.     """
  39.     计算日期之间的时间差
  40.     :param time1: 较小的时间（datetime类型）
  42.     :param time2: 较大的时间（datetime类型）
  44.     :param type: 返回结果的时间类型（暂时就是返回相差天数）
  46.     :return: 相差的天数
  48.     """
  49.  
  50.     day1 = time.strptime(str(time1), '%Y-%m-%d')
  51.  
  52.     day2 = time.strptime(str(time2), '%Y-%m-%d')
  53.  
  54.     if type == 'day':
  55.  
  56.         day_num = (int(time.mktime(day2)) - int(time.mktime(day1))) / (
  57.  
  58.             24 * 60 * 60)
  59.  
  60.     return abs(int(day_num))
  61. def get_end_point(start_time,days):
  62.     (year, month, day) = start_time.split("-")
  63.     day1 = datetime.date(int(year), int(month), int(day))
  64.  
  65.     ended_point = day1 + datetime.timedelta(days)
  66.     return ended_point
  67. def get_current_USDCNY():
  68.     """
  69.     获取美元兑人民币实时汇率
  70.     :return:
  71.     """
  72.     # 爬取网页
  73.     url = "http://srh.bankofchina.com/search/whpj/search.jsp"
  74.     Form_Data = {}
  75.     Form_Data['erectDate'] = ''
  76.     Form_Data['nothing'] = ''
  77.     Form_Data['pjname'] = '1316'
  78.     data = parse.urlencode(Form_Data).encode('utf-8')
  79.     html = request.urlopen(url, data).read()
  80.     soup = BeautifulSoup(html, 'html.parser')
  81.     # 解析数据
  82.     div = soup.find('div', attrs={'class': 'BOC_main publish'})
  83.     table = div.find('table')
  84.  
  85.     tr = table.find_all('tr')
  86.  
  87.     td = tr[1].find_all('td')
  88.     print(td[0].get_text(), td[1].get_text(), td[2].get_text(),td[3].get_text(), td[4].get_text(), td[5].get_text(), td[6].get_text())
  89.     return float(td[3].get_text())/100,td[6].get_text()
  90.  
  91. USDCNY ,current_time= get_current_USDCNY()
  92. print("实时汇率   "+current_time+"       USDCNY:"+str(USDCNY))
  93.  
  94. #输入起止时间
  95. while 1:
  96.     print("----------------Please input the start time and the end time----------------")
  97.     start_time = input()
  98.     end_time = input()
  99.     print("----------------The start time ----------------"+start_time)
  100.     print("----------------The end time   ----------------"+end_time)
  101.  
  102.     if start_time>=end_time:
  103.         print("----------------Input date invalided.Please try again ----------------")
  104.     else:
  105.         break
  106.  
  107. #DEBUG
  108. # start_time = "2018-08-09"
  109. # end_time = "2018-10-06"
  110.  
  111. daysLen=time_long(end_time,start_time)
  112. print(daysLen)
  113.  
  114. if daysLen < 20:
  115.     ylabel_dis = 1
  116. else:
  117.     ylabel_dis = int(daysLen/20)   #用于控制绘图时横轴的标注
  118.  
  119. per_loop_len = 0               #用于保存每次爬取的长度，为实际爬取的长度
  120.  
  121. if(daysLen <= PER_PAGE_DAYS):
  122.     per_loop_len = daysLen
  123.     loop = 1
  124.     left = 0
  125. else:
  126.     per_loop_len = PER_PAGE_DAYS
  127.     loop = int(daysLen/PER_PAGE_DAYS)
  128.     left = daysLen%PER_PAGE_DAYS
  129.  
  130. print("爬取循环次数"+str(loop))
  131. print("爬取循环次数"+str(left))
  132.  
  133. ended_point = start_time
  134. cnt = 0
  135. end_flag = 0
  136. total_ok = 0
  137.  
  138. xlabel = []
  139. date = []
  140. maxVal = []
  141. minVal = []
  142. PERP_PAGE_INFO = []
  143.  
  144. while 1:
  145.     #===========================日期处理部分，按每页25天的大小爬取网页
  146.     print("=============================PAGE"+str(cnt)+"=============================")
  147.     start_point = ended_point
  148.     ended_point = get_end_point(str(start_point), per_loop_len);
  149.     # print(str(cnt) + ":" + str(left) + ":" + str(per_loop_len))
  150.     print("start_point:" + str(start_point))
  151.     print("ended_point:" + str(ended_point))
  152.  
  153.     #===========================生成URL
  154.     Url = 'https://vip.stock.finance.sina.com.cn/q/view/vFutures_History.php?jys=NYME&pz=NG&hy=&breed=NG&type=global&start=' + str(start_point) + '&end=' + str(ended_point)
  155.     html = getHtml(Url)
  156.     soup = BeautifulSoup(html, 'lxml')
  157.     #============================得到本页表格
  158.     table = soup.find('div', attrs={'class': 'historyList'})
  159.     trs = table.find_all('tr')
  160.     for tr in trs:
  161.         tds = tr.find_all('td')
  162.         div_list.clear()
  163.         for td in tds:
  164.             divs = td.find_all('div', attrs={'align': 'center'})
  165.             for div in divs:
  166.                 val = div.get_text()
  167.                 div_list.append(val)
  168.         data = {}
  169.         for i in range(0, len(div_list)):
  170.             data.update({infos[i]: div_list[i]})
  171.             # print(div_list[i])
  172.         PERP_PAGE_INFO.append(data)
  173.  
  174.     for i in range(1, len(PERP_PAGE_INFO)-2):
  175.         total_ok = total_ok+1
  176.         xlabel.append(total_ok)
  177.         date.append(PERP_PAGE_INFO[len(PERP_PAGE_INFO)-1-i].get("日期"))
  178.         USD_maxVal = float(PERP_PAGE_INFO[len(PERP_PAGE_INFO)-1-i].get("最高价"))
  179.         maxVal.append(USD_maxVal*USDCNY)
  180.         USD_minVal = float(PERP_PAGE_INFO[len(PERP_PAGE_INFO)-1-i].get("最低价"))
  181.         if USD_minVal == 0.020:#仅仅针对2016-02-24的异常最小值进行修正
  182.             USD_minVal = 1.7+0.020
  183.         minVal.append(USD_minVal*USDCNY)
  184.         HISTORY_ALL.append(PERP_PAGE_INFO[len(PERP_PAGE_INFO)-1-i])
  185.     print("the length of per page : --->"+str(len(xlabel)))
  186.     # print(xlabel)
  187.     # print(date)
  188.     # print(maxVal)
  189.     # print(minVal)
  190.  
  191.     if end_flag == 1:
  192.         print("-----------------END-------------------")
  193.         break
  194.     cnt = cnt+1
  195.     if cnt == loop:
  196.         if left == 0:
  197.             print("-----------------END-------------------")
  198.             break
  199.         else :
  200.             per_loop_len = left
  201.             left = 0
  202.             end_flag = 1
  203.     PERP_PAGE_INFO.clear()
  204. print(HISTORY_ALL)
  205.  
  206. # 成本均价,绘制成本线
  207. length = len(xlabel)
  208. CYC = []
  209. for i in range(0,length):
  210.     CYC.append(cyc_val)
  211.  
  212. # ---------------------------------------画图---------------------------------------
  213. fig,ax = plt.subplots()
  214. plt.rcParams['figure.dpi'] = 300 #分辨率
  215.  
  216. # plot data
  217. ax.plot(date, maxVal, 'b', label='maxVal', linewidth=0.4)
  218. ax.plot(date, minVal, 'r', label='minVal', linewidth=0.4)
  219. ax.plot(date, CYC, 'c', label='CYC', linewidth=0.3)
  220.  
  221. #解决横轴过密的问题1
  222. for label in ax.get_xticklabels():
  223.     label.set_visible(False)
  224. for label in ax.get_xticklabels()[::ylabel_dis]:
  225.     label.set_visible(True)
  226.  
  227. #设置x轴标签文字的大小(size),倾斜角度(rotation),字体大小(fontsize)
  228. plt.xticks(size='small',rotation=90,fontsize=5)
  229.  
  230. plt.title('NG Historical Data')  # 添加图表标题
  231. plt.ylabel('Price/RMB')  # 添加 y 轴标题
  232. plt.xlabel('Date')  # 添加 x 轴标题
  233. plt.legend ()#设置图例
  234. #以分辨率 300 来保存图片
  235. plt.savefig('NG-'+start_time+"-"+end_time+'.png', dpi=300) #指定分辨率保存
  236. # show the figure
  237. plt.show()

使用方式：运行程序。运行环境;pycharm  python3.65 Anconda3

输入起始时间：如 2016-08-01

输入结束时间：如2019-09-06

格式要正确，等待爬取数据。

运行过程如下图