Records the daily stats (numeric or datatable blob) into the archive tables.
346955
protected function insertDayReports()
    {
        $this->insertDayNumericMetrics();

        // insert DataTable reports
        foreach ($this->getRecordNames() as $recordName) {
            $blob = $this->getDataArray($recordName)->asDataTable()->getSerialized($this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
            $this->getProcessor()->insertBlobRecord($recordName, $blob);
        }
    }
10683
function record(name, num, statistics, aggregations, timestamp, callback) {
        timestamp = timestamp || Date.now();
        function recordStats(key) {
            statistics && statistics.forEach(function(stat) {
                    if(stat == 'sum') {
                        multi.hincrby(key, 'sum', num)
                    } else if(scripts[stat]){
                        evalScript(multi, stat, [key, num],[],function(err, result) {
                                if(err) throw err;
                                if(result && result.indexOf('ERR') == 0) {
                                    // console.log(scripts[stat]);
                                    throw new Error(result);
                                }
                        })
                    }
            });
        }
        if(!statistics) statistics = ['sum'];
        if(num == undefined) num = 1;

        var multi = redis.multi();

        granularities.forEach(function(gran) {
                var key = getGranKey(name, gran, timestamp);
                recordStats(key);
                var unitPeriod = gran[0];
                multi.expire(key, points * unitPeriod / 1000);
        });

        if(aggregations) for(var i = 0; i < aggregations.length; i++) {
            var aggr = aggregations[i];
            var key = getAggrKey(name, aggr, timestamp);
            recordStats(key);
        }

        multi.exec(callback || log);
    }
133294
def to_db(self, table, dtypes=None):
        
        if self._check_db() is False:
            return
        if table not in self.db.tables:
            self.db.create_table(table)
            self.info("Created table ", table)
        self.start("Saving data to database table " + table + " ...")
        recs = self.to_records_()
        self.insert(table, recs, dtypes)
        self.end("Data inserted in table", table)
291173
async def on_date(self, date: datetime.date) -> dict:
        """"""
        return await self._request(
            'get', 'dailystats/{0}'.format(date.strftime('%Y-%m-%d')))
1136161
public function cacheDataByArchiveNameReports()
    {
        $api = API::getInstance();
        $api->getIndividualReportsSummary(true);
        $api->getIndividualMetricsSummary(true);

        $now = Date::now()->getLocalized(Date::DATE_FORMAT_SHORT);
        Option::set(DBStats::TIME_OF_LAST_TASK_RUN_OPTION, $now);
    }
666588
private void addReplicaStatsOf(RecordStore recordStore, LocalMapOnDemandCalculatedStats onDemandStats) {
        if (!hasRecords(recordStore)) {
            return;
        }

        long backupEntryCount = 0;
        long backupEntryMemoryCost = 0;

        int totalBackupCount = recordStore.getMapContainer().getTotalBackupCount();
        for (int replicaNumber = 1; replicaNumber <= totalBackupCount; replicaNumber++) {
            int partitionId = recordStore.getPartitionId();
            Address replicaAddress = getReplicaAddress(partitionId, replicaNumber, totalBackupCount);
            if (!isReplicaAvailable(replicaAddress, totalBackupCount)) {
                printWarning(partitionId, replicaNumber);
                continue;
            }
            if (isReplicaOnThisNode(replicaAddress)) {
                backupEntryMemoryCost += recordStore.getOwnedEntryCost();
                backupEntryCount += recordStore.size();
            }
        }

        if (NATIVE != recordStore.getMapContainer().getMapConfig().getInMemoryFormat()) {
            onDemandStats.incrementHeapCost(backupEntryMemoryCost);
        }
        onDemandStats.incrementBackupEntryMemoryCost(backupEntryMemoryCost);
        onDemandStats.incrementBackupEntryCount(backupEntryCount);
        onDemandStats.setBackupCount(recordStore.getMapContainer().getMapConfig().getTotalBackupCount());
    }
1356829
public void saveStatistics(ClientStats stats, String file) throws IOException {
        this.client.get().writeSummaryCSV(stats, file);
    }
403201
def dump(archive, calc_id=0, user=None):
    
    t0 = time.time()
    assert archive.endswith('.zip'), archive
    getfnames = 'select ds_calc_dir || ".hdf5" from job where ?A'
    param = dict(status='complete')
    if calc_id:
        param['id'] = calc_id
    if user:
        param['user_name'] = user
    fnames = [f for f, in db(getfnames, param) if os.path.exists(f)]
    zipfiles(fnames, archive, 'w', safeprint)
    pending_jobs = db('select id, status, description from job '
                      'where status="executing"')
    if pending_jobs:
        safeprint('WARNING: there were calculations executing during the dump,'
                  ' they have been not copied')
        for job_id, status, descr in pending_jobs:
            safeprint('%d %s %s' % (job_id, status, descr))

    # this also checks that the copied db is not corrupted
    smart_save(db.path, archive, calc_id)

    dt = time.time() - t0
    safeprint('Archived %d calculations into %s in %d seconds'
              % (len(fnames), archive, dt))
712092
void recordStats(@Nullable String value) {
    requireArgument(value == null, "record stats does not take a value");
    requireArgument(!recordStats, "record stats was already set");
    recordStats = true;
  }
875726
public function aggregateMultipleReports()
    {
        $dataTableRecords = array(self::USERID_ARCHIVE_RECORD);
        $columnsAggregationOperation = null;
        $this->getProcessor()->aggregateDataTableRecords(
            $dataTableRecords,
            $this->maximumRowsInDataTableLevelZero,
            $this->maximumRowsInDataTableLevelZero,
            $columnToSort = 'nb_visits',
            $columnsAggregationOperation,
            $columnsToRenameAfterAggregation = null,
            $countRowsRecursive = array());
    }
824223
public void recountRecords()
    {
        try   {
            Object bookmark = null;
            if (m_bRestoreCurrentRecord)
                if (m_recordSub.getEditMode() == DBConstants.EDIT_CURRENT)
                    bookmark = m_recordSub.getHandle(DBConstants.BOOKMARK_HANDLE);
            m_recordSub.close();
            while (m_recordSub.hasNext())
            {   // Recount each sub-record
                m_recordSub.next();
            }
            if (bookmark != null)
                m_recordSub.setHandle(bookmark, DBConstants.BOOKMARK_HANDLE);
        } catch (DBException ex)    {
            ex.printStackTrace();
        }
    }
971396
def __dbfRecords(self):
        """"""
        f = self.__getFileObj(self.dbf)
        for record in self.records:
            if not self.fields[0][0].startswith("Deletion"):
                f.write(b(' ')) # deletion flag
            for (fieldName, fieldType, size, dec), value in zip(self.fields, record):
                fieldType = fieldType.upper()
                size = int(size)
                if fieldType.upper() == "N":
                    value = str(value).rjust(size)
                elif fieldType == 'L':
                    value = str(value)[0].upper()
                else:
                    value = str(value)[:size].ljust(size)
                assert len(value) == size
                value = b(value)
                f.write(value)
190025
function () {
      var rowCounts = new Uint32Array(nrows);
      var columnCounts = new Uint32Array(ncols);
      for (var i = 0; i < nrows; i++) {
        var obj = matrix[i];
        if (obj.values != null) {
          // trim to size
          var values = new Uint16Array(obj.length);
          var indices = new Uint32Array(obj.length);
          values.set(obj.values.slice(0, obj.length));
          indices.set(obj.indices.slice(0, obj.length));
          matrix[i] = {values: values, indices: indices};
          rowCounts[i] = values.length;
          for (var j = 0; j < obj.length; j++) {
            columnCounts[indices[j]] += 1;
          }
        }
      }
      dataset = new morpheus.Dataset({
        rows: nrows, columns: ncols, name: morpheus.Util.getBaseFileName(morpheus.Util
          .getFileName(fileOrUrl)), array: matrix, dataType: dataType, defaultValue: 0
      });

      dataset.getRowMetadata().add('count>0').array = rowCounts;
      dataset.getColumnMetadata().add('count>0').array = columnCounts;
    }
1258443
private TableHistoryStatDO tableHistoryStatModelToDo(TableStat tableStat) {
        TableHistoryStatDO tableHistoryStatDO = new TableHistoryStatDO();
        tableHistoryStatDO.setId(tableStat.getId());
        tableHistoryStatDO.setPipelineId(tableStat.getPipelineId());
        tableHistoryStatDO.setDataMediaPairId(tableStat.getDataMediaPairId());
        tableHistoryStatDO.setStartTime(tableStat.getStartTime());
        tableHistoryStatDO.setEndTime(tableStat.getEndTime());
        tableHistoryStatDO.setFileSize(tableStat.getFileSize());
        tableHistoryStatDO.setFileCount(tableStat.getFileCount());
        tableHistoryStatDO.setDeleteCount(tableStat.getDeleteCount());
        tableHistoryStatDO.setInsertCount(tableStat.getInsertCount());
        tableHistoryStatDO.setUpdateCount(tableStat.getUpdateCount());
        tableHistoryStatDO.setGmtCreate(tableStat.getGmtCreate());
        tableHistoryStatDO.setGmtModified(tableStat.getGmtModified());
        return tableHistoryStatDO;

    }
1190277
def record_to_objects(self):
        """"""
        from ambry.orm.exc import NotFoundError

        fr = self.record

        contents = fr.unpacked_contents

        if not contents:
            return

        # Zip transposes an array when in the form of a list of lists, so this transposes so
        # each row starts with the heading and the rest of the row are the values
        # for that row. The bool and filter return false when none of the values
        # are non-empty. Then zip again to transpose to original form.

        non_empty_rows = drop_empty(contents)

        s = self._dataset._database.session

        for i, row in enumerate(non_empty_rows):

            if i == 0:
                header = row
            else:
                d = dict(six.moves.zip(header, row))

                if 'widths' in d:
                    del d['widths']  # Obsolete column in old spreadsheets.

                if 'table' in d:
                    d['dest_table_name'] = d['table']
                    del d['table']

                if 'order' in d:
                    d['stage'] = d['order']
                    del d['order']

                if 'dest_table' in d:
                    d['dest_table_name'] = d['dest_table']
                    del d['dest_table']

                if 'source_table' in d:
                    d['source_table_name'] = d['source_table']
                    del d['source_table']

                d['d_vid'] = self._dataset.vid

                d['state'] = 'synced'

                try:
                    ds = self._dataset.source_file(str(d['name']))
                    ds.update(**d)
                except NotFoundError:
                    name = d['name']
                    del d['name']

                    try:
                        ds = self._dataset.new_source(name, **d)
                    except:
                        print(name, d)
                        import pprint
                        pprint.pprint(d)
                        raise
                except:  # Odd error with 'none' in keys for d
                    print('!!!', header)
                    print('!!!', row)
                    raise

                s.merge(ds)

        self._dataset._database.commit()
888616
protected function insertDayReports()
    {
        ArchivingHelper::clearActionsCache();

        $this->insertPageUrlsReports();
        $this->insertDownloadsReports();
        $this->insertOutlinksReports();
        $this->insertPageTitlesReports();
        $this->insertSiteSearchReports();
    }
661680
def record(self, i=0):
        """"""
        f = self.__getFileObj(self.dbf)
        if not self.numRecords:
            self.__dbfHeader()
        i = self.__restrictIndex(i)
        recSize = self.__recordFmt()[1]
        f.seek(0)
        f.seek(self.__dbfHeaderLength() + (i * recSize))
        return self.__record()