python爬虫爬取github项目里的评论.zip

#https://github.com/bitcoin/bitcoin/issues?page=3&q=is%3Aissue+is%3Aclosed import urllib.request import re from bs4 import BeautifulSoup import io import sys import openpyxl import xlwt import xlrd from xlutils.copy import copy P=sys.argv[1] record=[] ''' def excelwrite(filename,value): workbook = xlrd.open_workbook(filename) sheet = workbook.sheet_by_index(0) rowNum = sheet.nrows colNum = sheet.ncols newbook = copy(workbook) newsheet = newbook.get_sheet(0) # 在末尾增加新行 for i in range(0,len(value)): newsheet.write(rowNum, i, value[i]) # 覆盖保存 newbook.save(filename) ''' def write07Excel(path,value): wb = openpyxl.load_workbook(path) ws = wb['Sheet1'] #for x in data: ws.append(value) wb.save(path) #print("写入成功") def gettitle(page=1): try: url="https://github.com/bitcoin/bitcoin/issues?page="+str(page)+"&q=is%3Aissue+is%3Aclosed" data = urllib.request.urlopen(url).read() z_data = data.decode('UTF-8') soup = BeautifulSoup(z_data, 'lxml') a = soup.select('li > div > div > a') b=soup.select('span.opened-by') c=soup.select('relative-time') test=soup.select('div.float-left.col-9.lh-condensed.p-2') #hostsfile = open('record.txt', 'w', newline='',encoding='UTF-8') for i in range(0,len(b)): temp=[] temp.append(a[i].get_text()) #标题 temp.append("closed") #状态 temp.append(c[i].attrs['datetime']) #问题提出时间 z="" for j in test[i].select('a.d-inline-block.IssueLabel.v-align-text-top'): z+=j.get_text()+'/' temp.append(z) #标签 #sn=b[i].get_text().replace(" ","").split('\n')[1].replace("#","").replace("\n","") m = re.search('\d+',b[i].get_text()) n=m.group(0) temp.append(m.group(0)) #任务ID s,t=getdata(m.group(0)) #s=getdata(m.group(0)) temp.append(t) #任务关闭时间 for i in s: temp.append(i) #temp.append(s) write07Excel("closed.xlsx",temp) #record.append(temp) #hostsfile.close() print('hosts刷新成功:',len(a)) except Exception as err: print(str(err)) def getdata(sn): temp=[] try: url="https://github.com/bitcoin/bitcoin/issues/"+str(sn) data = urllib.request.urlopen(url).read() z_data = data.decode('UTF-8') soup = BeautifulSoup(z_data, 'lxml') a = soup.select('task-lists table > tbody > tr > td.d-block.comment-body.markdown-body.js-comment-body') b = soup.select('div.discussion-item.discussion-item-closed') author=soup.find_all('h3',attrs={'class':'timeline-comment-header-text f5 text-normal'}) #print(sn,len(a),len(author)) temp.append(len(a)-1) #评论数 temp.append(author[0].select('a.author')[0].get_text()) #问题提出者 if len(b[0].select('a.author'))>0: temp.append(b[0].select('a.author')[0].get_text()) #问题关闭者 else: temp.append("") temp.append(a[0].get_text()) for i in range(1,len(a)): tempd=[] tempd.append(sn) #问题id tempd.append(author[i].select('a.author')[0].get_text()) #评论人id tempd.append(author[i].select('relative-time')[0].attrs['datetime']) #评论时间 tempd.append(a[i].get_text()) #评论内容 #评论内容 write07Excel("closed_comment.xlsx",tempd) except Exception as err: print(str(err)) #pass if len(b[0].select('relative-time'))>0: return temp,b[0].select('relative-time')[0].attrs['datetime'] else: return temp,"" ''' def write07Excel(path,value,row): wb = openpyxl.Workbook() sheet = wb.active sheet.title = 'Sheet1' for i in range(0, len(value)): for j in range(0, len(value[i])): sheet.cell(row=i+1, column=j+1, value=str(value[i][j])) wb.save(path) ''' if __name__=="__main__": #for i in range(37,146): try: gettitle(float(P)) print("第"+str(P)+"页完成") except Exception as err: hostsfile = open('record.txt', 'w', newline='') hostsfile.write(str(P)+":"+str(err) + "\n") hostsfile.close() #print("第"+str(i)+"页抓取失败") #write07Excel("closed.xlsx",record)