A股数据库,使用flask后端，采集数据并存入数据库，以及API.zip

#encoding:utf-8 import sys import numpy as np #verion1: get all companies data from tushare and store them in Mongodb import pymongo import datetime import tushare as ts import time import json import pandas as pd from collections import OrderedDict import pytz import types import requests from io import BytesIO, StringIO import os import click import re from os import listdir from os.path import isfile, join from os import walk from pandas import DataFrame import constants import math import time import numpy ''' 手动插入每天股票信息，每天进行操作，把当天的数据存入todaydata文件夹中，然后执行完本脚本以后，把 todaydata数据手动移动到newdata中 ''' class DayOperation: todaydata=constants.todaydata def __init__(self,Ip,port): self.ip=Ip self.port=port #股票按照时间进行索引存储，便于计算指数 def storagebyTime(self): onlyfiles = [ f for f in listdir(self.todaydata) if isfile(join(self.todaydata,f)) ] for f in onlyfiles: fdate=f[0:10] kind=f[11] if kind=='i': continue self.pricetime[fdate].drop() #read from using pandas totaltables=self.pricetime.collection_names() for f in onlyfiles: fdate=f[0:10] kind=f[11] if kind=='i': continue #self.pricetime[fdate].drop() df = pd.read_csv(self.todaydata+"/"+f) if len(f) < 12: continue if fdate in totaltables: print fdate+u"的股票数据已经存在" continue #df.index = df.code print u"插入"+fdate+u"的股票数据" records = json.loads(df.T.to_json()).values() self.pricetime[fdate].insert_many(records) def storagedayprice(self): # get stock data in newdata dir onlyfiles = [ f for f in listdir(self.todaydata) if isfile(join(self.todaydata,f)) ] #read from using pandas #print self.connection.collection_names() for f in onlyfiles: fdate=f[0:10] kind=f[11] if kind=='i': continue df = pd.read_csv(self.todaydata+"/"+f) if len(f) < 12: continue codes=df.code series={} for i in range(0,len(codes)): code=df.iloc[i].code series["open"]=df.iloc[i].open series["high"]=df.iloc[i].high series["low"]=df.iloc[i].low series["close"]=df.iloc[i].close series["change"]=df.iloc[i].change series["volume"]=df.iloc[i].volume series["money"]=df.iloc[i].money series["traded_market_value"]=df.iloc[i].traded_market_value series["market_value"]=df.iloc[i].market_value series["adjust_price"]=df.iloc[i].adjust_price series["report_type"]=df.iloc[i].report_type series["PE_TTM"]=df.iloc[i].PE_TTM series["PS_TTM"]=df.iloc[i].PS_TTM series["PC_TTM"]=df.iloc[i].PC_TTM series["PB"]=df.iloc[i].PB series["adjust_price_f"]=df.iloc[i].adjust_price_f series['date'] = datetime.datetime.strptime(df.iloc[i].date, "%Y-%m-%d") name=code #检查一下今天数据是否存在 countstock=self.oriprice[name].find({"date":series['date']}).count() if countstock ==0: print (u"插入股票"+name+" "+str(series["date"])+u"数据") self.oriprice[name].insert_one(series) else: print(u"股票"+name+" "+str(series['date'])+u"已经存在") # get stock data in newdata dir onlyfiles = [ f for f in listdir(self.todaydata) if isfile(join(self.todaydata,f)) ] #read from using pandas #print self.connection.collection_names() for f in onlyfiles: fdate=f[0:10] kind=f[11] if kind=='d': continue df = pd.read_csv(self.todaydata+"/"+f) if len(f) < 12: continue codes=df.index_code series={} for i in range(0,len(codes)): code=df.iloc[i].index_code series["open"]=df.iloc[i].open series["high"]=df.iloc[i].high series["low"]=df.iloc[i].low series["close"]=df.iloc[i].close series["change"]=df.iloc[i].change series["volume"]=df.iloc[i].volume series["money"]=df.iloc[i].money series['date'] = datetime.datetime.strptime(df.iloc[i].date, "%Y-%m-%d") name=code #check exits countstock=self.index[name].find({"date":series['date']}).count() if countstock ==0: print (u"插入指数"+name+str(series["date"])+u"数据") self.index[name].insert_one(series) else: print(u"指数"+name+" "+str(series['date'])+u"已经存在") #每天计算一次，计算行业的指数 def ComputeIndustryIndexDay(self): alldatabase=self.pricetime.collection_names() timelist = [] for t in alldatabase: timelist.append(datetime.datetime.strptime(t, "%Y-%m-%d")) #按照时间先后顺序排列 timelist.sort() #每个指数初始化为1000 allindustry=self.industry.collection_names() for i in allindustry: indexname = i+"detail" self.industryindex[indexname].drop() for i in allindustry: indexstart = 1000 codes = self.industry[i].find() listcodes=[] for code in codes: listcodes.append(code) for d in timelist: day=d.strftime('%Y-%m-%d') detail=self.pricetime[day].find() series={"code":[],"change":[],"market":[],"pe":[]} for de in detail: series["code"].append(de["code"]) series["change"].append(de["change"]) series["market"].append(de["market_value"]) series["pe"].append(de["PE_TTM"]) #print len(series["code"]) totaldata=zip(series['change'],series["market"],series["pe"]) stockdf = pd.DataFrame(data=list(totaldata),index=series["code"],columns = ['change','market',"pe"]) #原始数据存在重复的行，这里去掉重复数据 stockdf=stockdf.drop_duplicates() todaydiff = 0 summarket = 0 totalpe = 0 n = 0 for code in listcodes: if code.keys()[0][0]=='3' or code.keys()[0][0]=='0' or code.keys()[0][0]=='6': c = code.keys()[0] else: c =code.keys()[1] if c[0]=='3' or c[0]=='0': tmp = 'sz'+c else: tmp='sh'+c if tmp in stockdf.index: if isinstance(stockdf.ix[tmp].change, np.float) and isinstance(stockdf.ix[tmp].market, np.float): todaydiff =todaydiff+stockdf.ix[tmp].change*stockdf.ix[tmp].market summarket =summarket+stockdf.ix[tmp].market if isinstance(stockdf.ix[tmp].pe, np.float) and stockdf.ix[tmp].pe > 0 and stockdf.ix[tmp].pe < 500: