python根据邮箱地址对邮箱进行分类（百万以上数据）

#-*- coding:utf-8 –*- import time #多线程处理，因为多线程处理的结果并没有多少提高，所以去掉了多线程 #email地址比较函数 def cmp_emailprix(x,y): split_x = x.split('@') split_y = y.split('@') prix_x = split_x.pop() prix_y = split_y.pop() u1 = prix_x.upper() u2 = prix_y.upper() if(u1 < u2): return -1 if(u1 > u2): return 1 return 0 #获取文件行数 用来分配线程的 def getfilelines(filename): count = 0 thefile = open(filename,'rb') while True: buffer = thefile.read(8192*1024) if not buffer: break count += buffer.count('\n') return count #主函数 def dealemail(filename,thread_num): #lines_count = getfilelines(filename) thefile = open(filename,'rb') alle = list() for line in thefile: alle.append(line) sort_alle = sorted(alle,cmp_emailprix) #排序所有email，遍历所有排序过的email，如果当前遍历的email和上一个遍历的email不同， #则新建一个list 然后将这个list添加到字典中 dicte = {} lastemail = 'start@!!!!!!!!!!!!!!!!!!!!#####$$$$$$$$$$TTTT.com.cn.net'; for email in sort_alle: if cmp_emailprix(lastemail,email)!=0: newlist = list() splitat = email.split('@') key = splitat.pop() px = key.strip('\n') px = px.strip('\r') #此处可以用正则表达式 px = px.replace(' ','').replace('>','').replace('<','').replace('/','').replace('\\','').replace('|','').replace(':','').replace('"','').replace('*','').replace('?','') key = px key = key.lower() #print 'key=',key newlist.append(email) dicte[key] = newlist else: splitat = email.split('@') key = splitat.pop() px = key.strip('\n') px = px.strip('\r') #此处可以用正则表达式 px = px.replace(' ','').replace('>','').replace('<','').replace('/','').replace('\\','').replace('|','').replace(':','').replace('"','').replace('*','').replace('?','') key = px key = key.lower() #print 'key=',key curlist = dicte[key] curlist.append(email) lastemail = email #遍历字典，将不同的后缀名写入到同一个文件中区 <>,/,\,|,:,"",*,? listxxx = list() for (k,v) in dicte.items(): print 'open file',k,'.txt and save' filen = 'email/'+k +'.txt' #如果后缀名邮箱个数不大100，则将这个邮箱写入到指定文件中 if(len(dicte[k])>100): file_object = open(filen, 'w+') file_object.writelines(dicte[k]) else: listxxx.extend(dicte[k]) print 'open file','xxx','.txt and save' file_object1 = open('email/'+ 'xxx' +'.txt','w+') file_object1.writelines(listxxx) return 0 start = time.time() dealemail('111.txt',2) end = time.time() print 'deal the emials used ',end-start,'s!'