save stock_day

    Arguments:
        engine {[type]} -- [description]

    Keyword Arguments:
        client {[type]} -- [description] (default: {DATABASE})
178340
def QA_SU_save_stock_day(engine, client=DATABASE, paralleled=False):
    

    engine = select_save_engine(engine, paralleled=paralleled)
    engine.QA_SU_save_stock_day(client=client)
201701
def save(self, session):
        

        if not HAS_SQL:  # pragma: nocover
            return
        try:
            conn, c = self.connect()
            c.execute('DELETE FROM {0}'.format(self.table_name))
            values = {
                'value': sqlite3.Binary(pickle.dumps(session, protocol=2)),
            }
            c.execute('INSERT INTO {0} VALUES (:value)'.format(self.table_name), values)
            conn.commit()
            conn.close()
        except:  # pragma: nocover
            log.traceback(logging.DEBUG)
20342
def QA_SU_save_etf_list(engine, client=DATABASE):
    

    engine = select_save_engine(engine)
    engine.QA_SU_save_etf_list(client=client)
845159
def store(self, key, value):
        
        self.client.set(key, value, time=self.timeout)
711285
def get_db_prep_save(self, value, connection, prepared=False):
        
        log.debug("get_db_prep_save: {}, {}", value, type(value))
        if not value:
            return ''
            # For underlying (database) string types, e.g. VARCHAR, this
            # function must always return a string type.
            # https://docs.djangoproject.com/en/1.8/howto/custom-model-fields/
        return python_localized_datetime_to_human_iso(value)
122863
def QA_SU_save_etf_list(client=DATABASE, ui_log=None, ui_progress=None):
    
    try:
        QA_util_log_info(
            '##JOB16 Now Saving ETF_LIST ====',
            ui_log=ui_log,
            ui_progress=ui_progress,
            ui_progress_int_value=5000
        )
        etf_list_from_tdx = QA_fetch_get_stock_list(type_="etf")
        pandas_data = QA_util_to_json_from_pandas(etf_list_from_tdx)

        if len(pandas_data) > 0:
            # 获取到数据后才进行drop collection 操作
            client.drop_collection('etf_list')
            coll = client.etf_list
            coll.create_index('code')
            coll.insert_many(pandas_data)
        QA_util_log_info(
            "完成ETF列表获取",
            ui_log=ui_log,
            ui_progress=ui_progress,
            ui_progress_int_value=10000
        )
    except Exception as e:
        QA_util_log_info(e, ui_log=ui_log)
        print(" Error save_tdx.QA_SU_save_etf_list exception!")
        pass
697726
def QA_SU_save_index_min(client=DATABASE, ui_log=None, ui_progress=None):
    

    __index_list = QA_fetch_get_stock_list('index')
    coll = client.index_min
    coll.create_index(
        [
            ('code',
             pymongo.ASCENDING),
            ('time_stamp',
             pymongo.ASCENDING),
            ('date_stamp',
             pymongo.ASCENDING)
        ]
    )
    err = []

    def __saving_work(code, coll):

        QA_util_log_info(
            '##JOB05 Now Saving Index_MIN ==== {}'.format(str(code)),
            ui_log=ui_log
        )
        try:

            for type in ['1min', '5min', '15min', '30min', '60min']:
                ref_ = coll.find({'code': str(code)[0:6], 'type': type})
                end_time = str(now_time())[0:19]
                if ref_.count() > 0:
                    start_time = ref_[ref_.count() - 1]['datetime']

                    QA_util_log_info(
                        '##JOB05.{} Now Saving {} from {} to {} =={} '.format(
                            ['1min',
                             '5min',
                             '15min',
                             '30min',
                             '60min'].index(type),
                            str(code),
                            start_time,
                            end_time,
                            type
                        ),
                        ui_log=ui_log
                    )

                    if start_time != end_time:
                        __data = QA_fetch_get_index_min(
                            str(code),
                            start_time,
                            end_time,
                            type
                        )
                        if len(__data) > 1:
                            coll.insert_many(
                                QA_util_to_json_from_pandas(__data[1::])
                            )
                else:
                    start_time = '2015-01-01'

                    QA_util_log_info(
                        '##JOB05.{} Now Saving {} from {} to {} =={} '.format(
                            ['1min',
                             '5min',
                             '15min',
                             '30min',
                             '60min'].index(type),
                            str(code),
                            start_time,
                            end_time,
                            type
                        ),
                        ui_log=ui_log
                    )

                    if start_time != end_time:
                        __data = QA_fetch_get_index_min(
                            str(code),
                            start_time,
                            end_time,
                            type
                        )
                        if len(__data) > 1:
                            coll.insert_many(
                                QA_util_to_json_from_pandas(__data)
                            )
        except:
            err.append(code)

    executor = ThreadPoolExecutor(max_workers=4)

    res = {
        executor.submit(__saving_work,
                        __index_list.index[i_][0],
                        coll)
        for i_ in range(len(__index_list))
    }  # multi index ./.
    count = 0
    for i_ in concurrent.futures.as_completed(res):
        strLogProgress = 'DOWNLOAD PROGRESS {} '.format(
            str(float(count / len(__index_list) * 100))[0:4] + '%'
        )
        intLogProgress = int(float(count / len(__index_list) * 10000.0))
        QA_util_log_info(
            'The {} of Total {}'.format(count,
                                        len(__index_list)),
            ui_log=ui_log
        )
        QA_util_log_info(
            strLogProgress,
            ui_log=ui_log,
            ui_progress=ui_progress,
            ui_progress_int_value=intLogProgress
        )
        count = count + 1
    if len(err) < 1:
        QA_util_log_info('SUCCESS', ui_log=ui_log)
    else:
        QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)
        QA_util_log_info(err, ui_log=ui_log)
576142
def save(self):
        '''
        
        '''
        client = self._new_api_client()
        params = {'id': self.id} if hasattr(self, 'id') else {}
        action =  'patch' if hasattr(self, 'id') else 'post'
        saved_model = client.make_request(self, action, url_params=params, post_data=self._to_json)
        self.__init__(**saved_model._to_dict)
1133162
def save_to_store(self):
        
        if not self._store:
            raise AttributeError('No datastore defined!')
        saved_data = self.save_to_data(in_place=True)
        data = Serializer.serialize(saved_data)
        self._store.store_blob(data, 'all_keys_with_undefined')
457990
def engine(self):
        """"""

        if not self._engine:

            if 'postgres' in self.driver:

                if 'connect_args' not in self.engine_kwargs:
                    self.engine_kwargs['connect_args'] = {
                        'application_name': '{}:{}'.format(self._application_prefix, os.getpid())
                    }

                # For most use, a small pool is good to prevent connection exhaustion, but these settings may
                # be too low for the main public web application.

                self._engine = create_engine(self.dsn, echo=self._echo,
                                             pool_size=5, max_overflow=5, **self.engine_kwargs)

            else:
                self._engine = create_engine(
                    self.dsn, echo=self._echo, **self.engine_kwargs)

            #
            # Disconnect connections that have a different PID from the one they were created in.
            # This protects against re-use in multi-processing.
            #
            @event.listens_for(self._engine, 'connect')
            def connect(dbapi_connection, connection_record):
                connection_record.info['pid'] = os.getpid()

            @event.listens_for(self._engine, 'checkout')
            def checkout(dbapi_connection, connection_record, connection_proxy):

                from sqlalchemy.exc import DisconnectionError
                pid = os.getpid()
                if connection_record.info['pid'] != pid:

                    connection_record.connection = connection_proxy.connection = None
                    raise DisconnectionError(
                        "Connection record belongs to pid %s, attempting to check out in pid %s" %
                        (connection_record.info['pid'], pid))

            if self.driver == 'sqlite':
                @event.listens_for(self._engine, 'connect')
                def pragma_on_connect(dbapi_con, con_record):
                    """ISSUE some Sqlite pragmas when the connection is created."""

                    # dbapi_con.execute('PRAGMA foreign_keys = ON;')
                    # Not clear that there is a performance improvement.

                    # dbapi_con.execute('PRAGMA journal_mode = WAL')
                    dbapi_con.execute('PRAGMA synchronous = OFF')
                    dbapi_con.execute('PRAGMA temp_store = MEMORY')
                    dbapi_con.execute('PRAGMA cache_size = 500000')
                    if self._foreign_keys:
                        dbapi_con.execute('PRAGMA foreign_keys=ON')

            with self._engine.connect() as conn:
                _validate_version(conn, self.dsn)

        return self._engine
377365
def write_to_datastore(self):
    """"""
    client = self._datastore_client
    with client.no_transact_batch() as client_batch:
      for batch_id, batch_data in iteritems(self._data):
        batch_key = client.key(self._entity_kind_batches, batch_id)
        batch_entity = client.entity(batch_key)
        for k, v in iteritems(batch_data):
          if k != 'images':
            batch_entity[k] = v
        client_batch.put(batch_entity)
        self._write_single_batch_images_internal(batch_id, client_batch)
1084871
def from_tushare(dataframe, dtype='day'):
    

    if dtype in ['day']:
        return QA_DataStruct_Stock_day(
            dataframe.assign(date=pd.to_datetime(dataframe.date)
                            ).set_index(['date',
                                         'code'],
                                        drop=False),
            dtype='stock_day'
        )
    elif dtype in ['min']:
        return QA_DataStruct_Stock_min(
            dataframe.assign(datetime=pd.to_datetime(dataframe.datetime)
                            ).set_index(['datetime',
                                         'code'],
                                        drop=False),
            dtype='stock_min'
        )
796642
function(opt) {
        console.log("\n\n############ENGINE CONFIGURATION OPTIONS############\n");
        console.dir(opt);
        console.log("\n####################################################\n");
        if (!opt.hasOwnProperty('engine')) throw ("Please specify a scheduler engine");
        if (opt.engine === "sge")
            engine = sgeLib;
        else if (opt.engine === "slurm")
            engine = slurmLib;
        else if (opt.engine === "emulator") {
            engine = emulatorLib;
            emulator = true;
        }
        if (opt.engine != "emulator") {
            if (!opt.hasOwnProperty('binaries')) throw "You must specify scheduler engine binaries";
        }
        engine.configure(opt.binaries, jobsArray);
        console.log("Engine configured is " + engine.type());
    }
985364
def QA_SU_save_stock_info(engine, client=DATABASE):
    

    engine = select_save_engine(engine)
    engine.QA_SU_save_stock_info(client=client)
284783
def special_case_mysql(self, u, kwargs):
        """"""

        kwargs['pool_recycle'] = int(u.query.pop('max_idle', 3600))

        # default to the MyISAM storage engine
        storage_engine = u.query.pop('storage_engine', 'MyISAM')
        kwargs['connect_args'] = {
            'init_command': 'SET default_storage_engine=%s' % storage_engine,
        }

        if 'use_unicode' in u.query:
            if u.query['use_unicode'] != "True":
                raise TypeError("Buildbot requires use_unicode=True " +
                                "(and adds it automatically)")
        else:
            u.query['use_unicode'] = True

        if 'charset' in u.query:
            if u.query['charset'] != "utf8":
                raise TypeError("Buildbot requires charset=utf8 " +
                                "(and adds it automatically)")
        else:
            u.query['charset'] = 'utf8'

        return u, kwargs, None
29224
def QA_SU_save_stock_min(client=DATABASE, ui_log=None, ui_progress=None):
    
    # 导入聚宽模块且进行登录
    try:
        import jqdatasdk
        # 请自行将 JQUSERNAME 和 JQUSERPASSWD 修改为自己的账号密码
        jqdatasdk.auth("JQUSERNAME", "JQUSERPASSWD")
    except:
        raise ModuleNotFoundError

    # 股票代码格式化
    code_list = list(
        map(
            lambda x: x + ".XSHG" if x[0] == "6" else x + ".XSHE",
            QA_fetch_get_stock_list().code.unique().tolist(),
        ))
    coll = client.stock_min
    coll.create_index([
        ("code", pymongo.ASCENDING),
        ("time_stamp", pymongo.ASCENDING),
        ("date_stamp", pymongo.ASCENDING),
    ])
    err = []

    def __transform_jq_to_qa(df, code, type_):
        """
        处理 jqdata 分钟数据为 qa 格式，并存入数据库
        1. jdatasdk 数据格式:
                          open  close   high    low     volume       money
        2018-12-03 09:31:00  10.59  10.61  10.61  10.59  8339100.0  88377836.0
        2. 与 QUANTAXIS.QAFetch.QATdx.QA_fetch_get_stock_min 获取数据进行匹配，具体处理详见相应源码

                          open  close   high    low           vol        amount    ...
        datetime
        2018-12-03 09:31:00  10.99  10.90  10.99  10.90  2.211700e+06  2.425626e+07 ...
        """

        if df is None or len(df) == 0:
            raise ValueError("没有聚宽数据")

        df = df.reset_index().rename(columns={
            "index": "datetime",
            "volume": "vol",
            "money": "amount"
        })

        df["code"] = code
        df["date"] = df.datetime.map(str).str.slice(0, 10)
        df = df.set_index("datetime", drop=False)
        df["date_stamp"] = df["date"].apply(lambda x: QA_util_date_stamp(x))
        df["time_stamp"] = (
            df["datetime"].map(str).apply(lambda x: QA_util_time_stamp(x)))
        df["type"] = type_

        return df[[
            "open",
            "close",
            "high",
            "low",
            "vol",
            "amount",
            "datetime",
            "code",
            "date",
            "date_stamp",
            "time_stamp",
            "type",
        ]]

    def __saving_work(code, coll):
        QA_util_log_info(
            "##JOB03 Now Saving STOCK_MIN ==== {}".format(code), ui_log=ui_log)
        try:
            for type_ in ["1min", "5min", "15min", "30min", "60min"]:
                col_filter = {"code": str(code)[0:6], "type": type_}
                ref_ = coll.find(col_filter)
                end_time = str(now_time())[0:19]
                if coll.count_documents(col_filter) > 0:
                    start_time = ref_[coll.count_documents(
                        col_filter) - 1]["datetime"]
                    QA_util_log_info(
                        "##JOB03.{} Now Saving {} from {} to {} == {}".format(
                            ["1min",
                             "5min",
                             "15min",
                             "30min",
                             "60min"].index(type_),
                            str(code)[0:6],
                            start_time,
                            end_time,
                            type_,
                        ),
                        ui_log=ui_log,
                    )
                    if start_time != end_time:
                        df = jqdatasdk.get_price(
                            security=code,
                            start_date=start_time,
                            end_date=end_time,
                            frequency=type_.split("min")[0]+"m",
                        )
                        __data = __transform_jq_to_qa(
                            df, code=code[:6], type_=type_)
                        if len(__data) > 1:
                            coll.insert_many(
                                QA_util_to_json_from_pandas(__data)[1::])
                else:
                    start_time = "2015-01-01 09:30:00"
                    QA_util_log_info(
                        "##JOB03.{} Now Saving {} from {} to {} == {}".format(
                            ["1min",
                             "5min",
                             "15min",
                             "30min",
                             "60min"].index(type_),
                            str(code)[0:6],
                            start_time,
                            end_time,
                            type_,
                        ),
                        ui_log=ui_log,
                    )
                    if start_time != end_time:
                        __data == __transform_jq_to_qa(
                            jqdatasdk.get_price(
                                security=code,
                                start_date=start_time,
                                end_date=end_time,
                                frequency=type_.split("min")[0]+"m",
                            ),
                            code=code[:6],
                            type_=type_
                        )
                        if len(__data) > 1:
                            coll.insert_many(
                                QA_util_to_json_from_pandas(__data)[1::])
        except Exception as e:
            QA_util_log_info(e, ui_log=ui_log)
            err.append(code)
            QA_util_log_info(err, ui_log=ui_log)

    # 聚宽之多允许三个线程连接
    executor = ThreadPoolExecutor(max_workers=2)
    res = {
        executor.submit(__saving_work, code_list[i_], coll)
        for i_ in range(len(code_list))
    }
    count = 0
    for i_ in concurrent.futures.as_completed(res):
        QA_util_log_info(
            'The {} of Total {}'.format(count,
                                        len(code_list)),
            ui_log=ui_log
        )

        strProgress = "DOWNLOAD PROGRESS {} ".format(
            str(float(count / len(code_list) * 100))[0:4] + "%")
        intProgress = int(count / len(code_list) * 10000.0)

        QA_util_log_info(
            strProgress,
            ui_log,
            ui_progress=ui_progress,
            ui_progress_int_value=intProgress
        )
        count = count + 1
    if len(err) < 1:
        QA_util_log_info("SUCCESS", ui_log=ui_log)
    else:
        QA_util_log_info(" ERROR CODE \n ", ui_log=ui_log)
        QA_util_log_info(err, ui_log=ui_log)
85689
function Engine(sqlfilepath,callback) {                                                      
	this._resourceFields={};
	this._tags={};
	this._db=new TransactionDatabase( new sqlite3.Database(sqlfilepath) );

	var self=this;
	self._db.run("CREATE TABLE tag (name TEXT PRIMARY KEY, resource_count INTEGER, last_used_at INTEGER)",[], (err) => {
		self._db.serialize( () => {
			if (!err) {
				/* 
				if no error, the table did not exist, so let's create the remaining tables since this db is not yet initialized
				*/
				
				/*
				following line is commented since i get foreign key errors although i shouldn't. 
				there must be some subtility i miss in parallel operations, although they appear to run
				all in the correct order. i finally don't care too much, since the code ensures consistency 
				pretty much by its own.
				*/
				//self._db.run("PRAGMA foreign_keys = '1'");
				self._db.run("CREATE TABLE resource (_url TEXT, _created_at INTEGER, _modified_at INTEGER, _path TEXT)");
				self._db.run("CREATE UNIQUE INDEX resource_path on resource (_path)");
				self._db.run("CREATE TABLE tree_idx (desc_id INTEGER, anc_id INTEGER, FOREIGN KEY (anc_id) REFERENCES resource (rowid), FOREIGN KEY (desc_id) REFERENCES resource (rowid))");
				self._db.run("CREATE UNIQUE INDEX tree_descendant on tree_idx (desc_id,anc_id)");
				self._db.run("CREATE INDEX tree_ancestor on tree_idx (anc_id)");
				self._db.run("CREATE TABLE resource_field (name TEXT, type INTEGER)");
				self._db.run("CREATE TABLE resource_tag (resource_id INTEGER, tag_id INTEGER, tag_path TEXT, comment TEXT, FOREIGN KEY (resource_id) REFERENCES resource (rowid), FOREIGN KEY (tag_id) REFERENCES tag (rowid))");
				self._db.run("CREATE UNIQUE INDEX resource_tag_relation on resource_tag (resource_id,tag_id)");
				self._db.run("CREATE INDEX resource_tag_by_tag on resource_tag (tag_id)");
				self._db.run("CREATE INDEX resource_tag_by_tagpath on resource_tag (tag_path)");
				self._db.run("CREATE TABLE meta (key TEXT, value TEXT)");
				self._db.run("INSERT INTO meta (key,value) VALUES ('format_version','1')");
			}
			/*
			load all resource fields, which are cached.
			*/
			self._db.each("SELECT name,type FROM resource_field", (err, row) => {
				self._resourceFields[row.name]=row.type;
			}, (err) => {
				/*
				load all tags, which are cached as well.
				*/
				self._db.each("SELECT rowid,name,resource_count,last_used_at FROM tag", (err, row) => {
					self._tags[row.name]=row;
				}, (err) => {
					if (callback) callback(err,self);
				});
			});
		});
	});
	
	/*
	Qed version of the engine API: all public methods of the engine are replicated on the engine.q object, but
	returning a promise istead of calling a callback.
	*/
	this.q={
		cb: self,
		define: (fieldName,fieldType) => {
			var d=Q.defer();
			self.define(fieldName,fieldType, (err) => {
				if (err) d.reject(err);
				else d.resolve();
			});
			return d.promise;
		},
		loadBulk: (data) => {
			var d=Q.defer();
			self.loadBulk(data, (err) => {
				if (err) d.reject(err);
				else d.resolve();
			});
			return d.promise;
		},
		update: (url,tags,data) => {
			var d=Q.defer();
			self.update(url,tags,data, (err,id) => {
				if (err) d.reject(err);
				else d.resolve(id);
			});
			return d.promise;
		},
		remove: (url) => {
			var d=Q.defer();
			self.remove(url, (err,id) => {
				if (err) d.reject(err);
				else d.resolve(id);
			});
			return d.promise;
		},                                   
		removeBulk: (urlList) => {
			var d=Q.defer();
			self.removeBulk(urlList, (err,id) => {
				if (err) d.reject(err);
				else d.resolve(id);
			});
			return d.promise;
		},                                   
		rename: (url,newUrl,renameDescendants) => {
			var d=Q.defer();
			self.rename(url, newUrl, renameDescendants, (err,id) => {
				if (err) d.reject(err);
				else d.resolve(id);
			});
			return d.promise;
		},
		findResources: (tags,orderBy,limit,offset) => {
			var d=Q.defer();
			self.findResources(tags,orderBy,limit,offset, (err,res) => {
				if (err) d.reject(err);
				else d.resolve(res);
			});
			return d.promise;
		},
		getResource: (url) => {
			var d=Q.defer();
			self.getResource(url, (err,res) => {
				if (err) d.reject(err);
				else d.resolve(res);
			});
			return d.promise;
		},
		getBulk: (urlList) => {
			var d=Q.defer();
			self.getBulk(urlList, (err,res) => {
				if (err) d.reject(err);
				else d.resolve(res);
			});
			return d.promise;
		},
		tagCloud: (tags,limit,useOr) => {
			var d=Q.defer();
			self.tagCloud(tags,limit,useOr, (err,res) => {
				if (err) d.reject(err);
				else d.resolve(res);
			});
			return d.promise;
		},
		getSuggestions: (prefix,exclude,limit,offset,minWeight) => {
			var d=Q.defer();
			self.getSuggestions(prefix,exclude,limit,offset,minWeight, (err,res) => {
				if (err) d.reject(err);
				else d.resolve(res);
			});
			return d.promise;
		},
		clear: () => {
			var d=Q.defer();
			self.clear( (err,res) => {
				if (err) d.reject(err);
				else d.resolve(res);
			});
			return d.promise;
		}
	};
	
	/*
	ensure that whatever API you are using (the callback API or the Qed one), you will
	always be able to do:
		engine.cb.someMethod()
	and:
		engine.q.someMethod()
	getting explicitely the desired version of the method.
	*/
	this.q.q=this.q;
	this.cb=this;
	
}