python 单线程多线程和多进程的比较

# -*- coding: UTF-8 -*- import threading import redis from com.functions import * from com.dbhelper import * from com.dbconfig import * import multiprocessing import threading class MyThread(threading.Thread): def __init__(self,func,args,name=''): threading.Thread.__init__(self) self.name=name self.func=func self.args=args def run(self): apply(self.func,self.args) def get_avg_value(keylist): avg_value=[] for i in range(len(keylist)): tmp_list = [] for j in range(len(keylist[i])): tmp_list.append(keylist[i][j][1]) avg_value.append(sum(tmp_list)/len(tmp_list)) # print 'len of tmplist',len(tmp_list) del tmp_list return avg_value def get_data_from_redis(para_conn_pool_redis,para_conn_mysql,para_keylist): # keylist=[] tmp_result_avg=[] re = redis.Redis(connection_pool=para_conn_pool_redis) redis_pipe=re.pipeline() insert_mysql="INSERT INTO MSSql_Data_TableSpace_total(machine_name, host_name, db_name, table_name, schema_name, row_count, reserved_kb, data_kb, index_size_kb, unused_kb, collection_time) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" count=0 keys=[] for item in para_keylist: redis_pipe.zrange(item,0,-1,withscores=True) if count>500: count=0 tmp_dataset=redis_pipe.execute() for keyvalue in tmp_dataset: keys.append(keyvalue[0][0]) tmp_result_avg+=(get_avg_value(tmp_dataset)) count+=1 tmp_dataset=redis_pipe.execute() for keyvalue in tmp_dataset: keys.append(keyvalue[0][0]) tmp_result_avg+=(get_avg_value(tmp_dataset)) result=[] # for i in range(0,len(tmp_result_avg),5): # tmp=para_keylist[i].split(':') # tmp=tmp[:-1] # tmp.append(tmp_result_avg[i]) # tmp.append(tmp_result_avg[i+1]) # tmp.append(tmp_result_avg[i + 2]) # tmp.append(tmp_result_avg[i + 3]) # tmp.append(tmp_result_avg[i + 4]) # tmp.append(keys[i]) # result.append(tuple(tmp)) # del tmp # para_conn_mysql.executemany(insert_mysql,result) #queue.put(tmp_result_avg) #def func_get_avg_value(redis_keys,thread_num): def main(): pool = redis.ConnectionPool(host='', port=, password='', db=) # re=redis.Redis(connection_pool=pool) # redis_pipe=re.pipeline() re = redis.Redis(connection_pool=pool) mysqlconn = DBHelper(connectshort("xxxxx")) keyslist=re.keys() # keyslist=keyslist[0:500000] keyslist.sort() keyslist=keyslist[0:50000] #thread_num=4 threads=[] queues=[] results=[] final_results=[] thread_num=4 # sublist_count=len(keyslist)/thread_num # for i in range(thread_num): # if i ==(thread_num-1): # t = MyThread(get_data_from_redis,(pool,mysqlconn,keyslist[i*sublist_count:]),get_data_from_redis.__name__) # else: # t = MyThread(get_data_from_redis,(pool, mysqlconn, keyslist[i * sublist_count:(i+1)*sublist_count]), get_data_from_redis.__name__) # t.start() # threads.append(t) # for process in threads: # process.join() get_data_from_redis(pool,mysqlconn,keyslist) if __name__ == '__main__': main()