#!/usr/bin/python # -*- coding: utf-8 -*- #--------------------------------------------- # 抓上櫃交易明細 # Version : 1.1 # Author : Amin white # Release Date : 2012-06-27 # Python version : 2.7.2 #--------------------------------------------- import csv, os, datetime, urllib, codecs, time from sgmllib import SGMLParser #每日上櫃交易資訊 class brokerBS_otc(SGMLParser): def __init__(self): SGMLParser.__init__(self) #初始化變數 def reset(self): SGMLParser.reset(self) self.webexist = False self.broker = False self.brokerid = [] self.pricevol = False self.price_vol = 0 self.count = 0 self.price = [] self.sellvol = [] self.buyvol = [] #解析網頁元素 def parse(self,data): self.feed(data) self.close() #對網頁表格控制 def start_td(self, attrs): if len(attrs) == 2: if attrs[0][1] == 'table-mainbody-left' and attrs[1][1] == '2': self.webexist = True elif len(attrs) == 1: if attrs[0][1] == 'table-body-left' or attrs[0][1] == 'page_table-body-LEFT': self.broker = True elif attrs[0][1] == 'table-body-right' or attrs[0][1] == 'page_table-body-RIGHT': self.pricevol = True self.price_vol += 1 self.price_vol %= 3 #處理網頁資料 def handle_data(self, text): #取得劵商代碼 if self.broker: data = text.split(" ") self.brokerid.append(data[0]) self.broker = False self.count += 1 #取得買買交易明細 if self.pricevol: #取得買賣價格 if self.price_vol == 1: self.price.append(text) self.pricevol = False #取得買的數量 elif self.price_vol == 2: self.buyvol.append(text.replace(',', '')) self.pricevol = False #取得賣的數量 elif self.price_vol == 0: self.sellvol.append(text.replace(',', '')) self.pricevol = False self.brokerpric = 0 #上市櫃股票代碼與名稱 class Parser_strMode(SGMLParser): def __init__(self): SGMLParser.__init__(self) def reset(self): SGMLParser.reset(self) self.stockinfo = False self.no = 0 self.stock = [] self.col1 = "" self.col2 = "" self.col3 = "" def parse(self,data): self.feed(data) self.close() def start_td(self, attrs): for name, value in attrs: if len(attrs) == 1: if name == 'bgcolor' and value == '#FAFAD2': self.stockinfo = True self.no+=1 self.no%=7 def handle_data(self, text): if self.stockinfo: if self.no == 1: data = text.split(" ") if data[0].isalnum(): self.col1 = data[0] self.col2 = data[1] elif self.no == 6: self.col3 = text elif self.no == 7: self.no = 0 if self.no == 6 and self.col3 == 'ESVUFR' or self.col3 == 'EUOMSR' or self.col3 == 'EMXXXA' or self.col3 == 'ESVUFA': self.stock.append(self.col1) self.stockinfo = False def Getstockid(webindex): url = "http://brk.twse.com.tw:8000/isin/C_public.jsp?strMode=%s" %webindex webcode = urllib.urlopen(url) if webcode.code == 200: stock = Parser_strMode() stock.parse(webcode.read()) webcode.close() stock.stock.sort() if len(stock.stock) >0: return stock else: return 0 #將Stock id寫入CSV file def GetStockidtoCSVfile(filepath, stock): #開啟檔案 writefile = file(filepath, 'wb') #將檔案以UTF8的格式儲存 writefile.write(codecs.BOM_UTF8) #將資料儲存至CSV檔中 writer = csv.writer(writefile) #寫入抬頭名稱 writer.writerow([u'股票代號'.encode('utf8')]) #開始寫檔 for i in range(0, len(stock)): writer.writerow(['%s' %stock[i].encode('utf8')]) #關閉檔案 writefile.close() def GetTodayDate(mode): #取得今天的日期 todaydate = datetime.datetime.today() #取得年 year = todaydate.strftime('%Y') #取得月 month = todaydate.strftime('%m') #取得日 day = todaydate.strftime('%d') #組合年月日 if mode == 1: #西元 todaydate = year + month + day elif mode == 2: #民國 todaydate = str(int(year) - 1911) + month + day #回傳 return todaydate def main(startpos): #取得當天交易的日期,格式如20121001 todaydate = GetTodayDate(1) todaydate = '20121005' #取得股票代碼 stockid = Getstockid(4) if len(stockid.stock) == 0: return 0 #當天交易明細存放的路徑,可自行更改 workdir = 'D:\\stock_database\\test\\otc\\' + todaydate #建立交易明細存放的資料夾 if not os.path.isdir(workdir): os.makedirs(workdir) #將股票代碼儲存至檔案裡 StockidFile = workdir + '\\' + todaydate +'.csv' GetStockidtoCSVfile(StockidFile, stockid.stock) #開始對上櫃每個股票代碼抓取交易明細 for i in range(startpos, len(stockid.stock)): stock_id = stockid.stock[i] #建立交易明細檔案的名稱 CSVFile = workdir + '\\' + stock_id + '_' + todaydate +'.csv' #開起檔案準備寫檔 fileobj = file(CSVFile, 'wb') #將檔案以UTF8的格式儲存 fileobj.write(codecs.BOM_UTF8) #將交易明細寫入檔案前的格式設定 writer = csv.writer(fileobj, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) #寫入明細的抬頭名稱 writer.writerow([u'序號'.encode('utf8'), u'交易日期'.encode('utf8'), u'股票種類'.encode('utf8'), u'股票代號'.encode('utf8'), \ u'券商'.encode('utf8'), u'價格'.encode('utf8'), u'買進股數'.encode('utf8'), u'賣出股數'.encode('utf8')]) #對交易筆數與讀取網頁次數的初始化 serialnumber = 1 Retrytime = 0 #開始抓取網頁的交易明細 for j in range(1, 5001): while True: url = "http://www.gretai.org.tw/ch/stock/aftertrading/broker_trading/brokerBS.php?stk_code=%s&topage=%s" %(stock_id, str(j)) webcode = urllib.urlopen(url) #網頁順利開啟的回傳值200 if webcode.code == 200: stock = brokerBS_otc() stock.parse(webcode.read()) webcode.close() stock.close() #如果交易明細存在或是重複抓取網頁次數超過30次,則離開回圈 if stock.webexist or Retrytime > 30 : break; time.sleep(1) print "Retry time : " + str(Retrytime) + " " + stock_id Retrytime+=1 print 'No : ' + str(j) + ' stock kind : otc stockid : ' + stock_id + ' date : ' + todaydate + u' 券商筆數 : ' + str(len(stock.brokerid)) #將交易明細寫入檔案中 for k in range(0, len(stock.brokerid)): writer.writerow([ '%d' %serialnumber, '%s' %todaydate, 'otc'.encode('utf8'), '%s' %stock_id, \ '%s' %stock.brokerid[k], '%s' %stock.price[k], '%s' %stock.buyvol[k], '%s' %stock.sellvol[k]]) serialnumber += 1 #找出網頁交易筆數當小於100筆的就是交易明細的最後一頁 if len(stock.brokerid) < 100: stocklog = 'stockid : ' + stock_id + ' Date : ' + todaydate + ' total web page : %s' %(str(j)) + ' progress rate : ' + str(i) + '/' + str(len(stockid.stock)) + '\n' print stocklog break; print 'No : ' + str(j) + ' stockid : ' + stock_id + ' date : ' + todaydate + u" write to csv OK......" #關閉寫檔動作 fileobj.close() if __name__ == "__main__": main(0)
程式執行畫面
執行結果