Main function, runs archiving on all websites with new activity
340234
public function run()
    {
        $timer = new Timer;

        $this->logSection("START");
        $this->logger->info("Starting Matomo reports archiving...");

        $numWebsitesScheduled = $this->websites->getNumSites();
        $numWebsitesArchived = 0;

        $cliMulti = $this->makeCliMulti();
        if ($this->maxConcurrentArchivers && $cliMulti->supportsAsync()) {
            $numRunning = 0;
            $processes = Process::getListOfRunningProcesses();
            $instanceId = SettingsPiwik::getPiwikInstanceId();

            foreach ($processes as $process) {
                if (strpos($process, 'console core:archive') !== false &&
                    (!$instanceId
                      || strpos($process, '--matomo-domain=' . $instanceId) !== false
                      || strpos($process, '--matomo-domain="' . $instanceId . '"') !== false
                      || strpos($process, '--matomo-domain=\'' . $instanceId . "'") !== false
                      || strpos($process, '--piwik-domain=' . $instanceId) !== false
                      || strpos($process, '--piwik-domain="' . $instanceId . '"') !== false
                      || strpos($process, '--piwik-domain=\'' . $instanceId . "'") !== false)) {
                    $numRunning++;
                }
            }
            if ($this->maxConcurrentArchivers < $numRunning) {
                $this->logger->info(sprintf("Archiving will stop now because %s archivers are already running and max %s are supposed to run at once.", $numRunning, $this->maxConcurrentArchivers));
                return;
            } else {
                $this->logger->info(sprintf("%s out of %s archivers running currently", $numRunning, $this->maxConcurrentArchivers));
            }
        }

        do {
            if ($this->isMaintenanceModeEnabled()) {
                $this->logger->info("Archiving will stop now because maintenance mode is enabled");
                return;
            }

            $idSite = $this->websites->getNextSiteId();
            $numWebsitesArchived++;

            if (null === $idSite) {
                break;
            }

            if ($numWebsitesArchived > $numWebsitesScheduled) {
                // this is needed because a cron:archive might run for example for 5 hours. Meanwhile 5 other
                // `cron:archive` have been possibly started... this means meanwhile, within the 5 hours, the
                // `list of SharedSiteIds` have been potentially emptied and filled again from the beginning.
                // This means 5 hours later, even though all websites that were originally in the list have been
                // finished by now, the `cron:archive` will stay active and continue processing because the list of
                // siteIds to archive was resetted by another `cron:archive` command. Potentially some `cron:archive`
                // will basically never end because by the time the `cron:archive` finishes, the sharedSideIds have
                // been resettet. This can eventually lead to some random concurrency issues when there are like
                // 40 `core:archive` active at the same time.
                $this->logger->info("Stopping archiving as the initial list of websites has been processed.");
                return;
            }

            flush();
            $requestsBefore = $this->requests;
            if ($idSite <= 0) {
                continue;
            }

            $skipWebsiteForced = in_array($idSite, $this->shouldSkipSpecifiedSites);
            if ($skipWebsiteForced) {
                $this->logger->info("Skipped website id $idSite, found in --skip-idsites ");
                $this->skipped++;
                continue;
            }

            $shouldCheckIfArchivingIsNeeded    = !$this->shouldArchiveSpecifiedSites && !$this->shouldArchiveAllSites && !$this->dateLastForced;
            $hasWebsiteDayFinishedSinceLastRun = in_array($idSite, $this->websiteDayHasFinishedSinceLastRun);
            $isOldReportInvalidatedForWebsite  = $this->isOldReportInvalidatedForWebsite($idSite);

            if ($shouldCheckIfArchivingIsNeeded) {
                // if not specific sites and not all websites should be archived, we check whether we actually have
                // to process the archives for this website (only if there were visits since midnight)
                if (!$hasWebsiteDayFinishedSinceLastRun && !$isOldReportInvalidatedForWebsite) {

                    try {
                        if ($this->isWebsiteUsingTheTracker($idSite)) {

                            if(!$this->hadWebsiteTrafficSinceMidnightInTimezone($idSite)) {
                                $this->logger->info("Skipped website id $idSite as archiving is not needed");

                                $this->skippedDayNoRecentData++;
                                $this->skipped++;
                                continue;
                            }
                        } else {
                           $this->logger->info("- website id $idSite is not using the tracker");
                        }
                    } catch (UnexpectedWebsiteFoundException $e) {
                        $this->logger->info("Skipped website id $idSite, got: UnexpectedWebsiteFoundException");
                        continue;
                    }

                } elseif ($hasWebsiteDayFinishedSinceLastRun) {
                    $this->logger->info("Day has finished for website id $idSite since last run");
                } elseif ($isOldReportInvalidatedForWebsite) {
                    $this->logger->info("Old report was invalidated for website id $idSite");
                }
            }

            /**
             * This event is triggered before the cron archiving process starts archiving data for a single
             * site.
             *
             * @param int $idSite The ID of the site we're archiving data for.
             */
            Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite));

            $completed = $this->archiveSingleSite($idSite, $requestsBefore);

            /**
             * This event is triggered immediately after the cron archiving process starts archiving data for a single
             * site.
             *
             * @param int $idSite The ID of the site we're archiving data for.
             */
            Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $completed));
        } while (!empty($idSite));

        $this->logger->info("Done archiving!");

        $this->logSection("SUMMARY");
        $this->logger->info("Total visits for today across archived websites: " . $this->visitsToday);

        $totalWebsites = count($this->allWebsites);
        $this->skipped = $totalWebsites - $this->websitesWithVisitsSinceLastRun;
        $this->logger->info("Archived today's reports for {$this->websitesWithVisitsSinceLastRun} websites");
        $this->logger->info("Archived week/month/year for {$this->archivedPeriodsArchivesWebsite} websites");
        $this->logger->info("Skipped {$this->skipped} websites");
        $this->logger->info("- {$this->skippedDayNoRecentData} skipped because no new visit since the last script execution");
        $this->logger->info("- {$this->skippedDayArchivesWebsites} skipped because existing daily reports are less than {$this->todayArchiveTimeToLive} seconds old");
        $this->logger->info("- {$this->skippedPeriodsArchivesWebsite} skipped because existing week/month/year periods reports are less than {$this->processPeriodsMaximumEverySeconds} seconds old");

        if($this->skippedPeriodsNoDataInPeriod) {
            $this->logger->info("- {$this->skippedPeriodsNoDataInPeriod} skipped periods archiving because no visit in recent days");
        }

        if($this->skippedDayOnApiError) {
            $this->logger->info("- {$this->skippedDayOnApiError} skipped because got an error while querying reporting API");
        }
        $this->logger->info("Total API requests: {$this->requests}");

        //DONE: done/total, visits, wtoday, wperiods, reqs, time, errors[count]: first eg.
        $percent = $this->websites->getNumSites() == 0
            ? ""
            : " " . round($this->processed * 100 / $this->websites->getNumSites(), 0) . "%";
        $this->logger->info("done: " .
            $this->processed . "/" . $this->websites->getNumSites() . "" . $percent . ", " .
            $this->visitsToday . " vtoday, $this->websitesWithVisitsSinceLastRun wtoday, {$this->archivedPeriodsArchivesWebsite} wperiods, " .
            $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " .
            (empty($this->errors)
                ? self::NO_ERROR
                : (count($this->errors) . " errors."))
        );

        $this->logger->info($timer->__toString());
    }
87310
def handle_whole_archive(building_block, res, linker, flag)
      if is_whole_archive(building_block)
        res.push(flag) if flag and !flag.empty?
      end
    end
499384
static function executeActions()
    {
        if ( empty( self::$actionList ) )
        {
            return;
        }

        $fileContentCache = array();
        $doneDestList = array();

        $ini = eZINI::instance( 'staticcache.ini');
        $clearByCronjob = ( $ini->variable( 'CacheSettings', 'CronjobCacheClear' ) == 'enabled' );

        if ( $clearByCronjob )
        {
            $db = eZDB::instance();
        }

        foreach ( self::$actionList as $action )
        {
            list( $action, $parameters ) = $action;

            switch( $action ) {
                case 'store':
                    list( $destination, $source ) = $parameters;

                    if ( isset( $doneDestList[$destination] ) )
                        continue 2;

                    if ( $clearByCronjob )
                    {
                        $param = $db->escapeString( $destination . ',' . $source );
                        $db->query( 'INSERT INTO ezpending_actions( action, param ) VALUES ( \'static_store\', \''. $param . '\' )' );
                        $doneDestList[$destination] = 1;
                    }
                    else
                    {
                        if ( !isset( $fileContentCache[$source] ) )
                        {
                            if ( eZHTTPTool::getDataByURL( $source, true, eZStaticCache::USER_AGENT ) )
                                $fileContentCache[$source] = eZHTTPTool::getDataByURL( $source, false, eZStaticCache::USER_AGENT );
                            else
                                $fileContentCache[$source] = false;
                        }
                        if ( $fileContentCache[$source] === false )
                        {
                            eZDebug::writeError( "Could not grab content (from $source), is the hostname correct and Apache running?", 'Static Cache' );
                        }
                        else
                        {
                            eZStaticCache::storeCachedFile( $destination, $fileContentCache[$source] );
                            $doneDestList[$destination] = 1;
                        }
                    }
                    break;
            }
        }
        self::$actionList = array();
    }
560506
public function main()
  {
    $this->logInfo("Preserving last commit time under directory %s", $this->myWorkDirName);

    $this->fetchAllFilesUnderGit();

    $this->fetchLastCommitTimes();

    $this->setFilesMtime();

    $this->logInfo("Found %d files", $this->myCount);
  }
630251
@Override
	public void run()
	{

		try {
			ArchiveManager archives = new ArchiveManager();
			ArticleReaderInterface articleReader = null;
			ArchiveDescription description = null;
			Task<Revision> task = null;
			DiffCalculatorInterface diffCalc;

			if (MODE_STATISTICAL_OUTPUT) {
				diffCalc = new TimedDiffCalculator(new TaskTransmitter());
			}
			else {
				diffCalc = new DiffCalculator(new TaskTransmitter());
			}

			long start, time;

			while (archives.hasArchive()) {


				// Retrieve Archive
				try {
					description = archives.getArchive();

					// initialize filter
					ArticleFilter nameFilter = new ArticleFilter();

					articleReader = InputFactory.getTaskReader(description,
							nameFilter);
					ArticleConsumerLogMessages.logArchiveRetrieved(logger,
							description);

					// Exception while accessing the archive
				}
				catch (ArticleReaderException e) {

					articleReader = null;
					ArticleConsumerLogMessages.logExceptionRetrieveArchive(
							logger, description, e);
				}

				// Process Archive
				while (articleReader != null) {
					try {
						if (articleReader.hasNext()) {

							start = System.currentTimeMillis();
							//read the next article (may be null if filtered)
							task = articleReader.next();
							time = System.currentTimeMillis() - start;

							// task will be null if the name filter removed that
							// article
							if (task == null) {
								continue;
							}

							ArticleConsumerLogMessages
									.logArticleRead(logger, task, time,
											articleReader.getBytePosition());

							start = System.currentTimeMillis();
							//calculate the diff for this article version
							diffCalc.process(task);
							time = System.currentTimeMillis() - start;

							DiffConsumerLogMessages.logArticleProcessed(logger,
									task, time);

						}
						else {
							ArticleConsumerLogMessages.logNoMoreArticles(
									logger, description);
							articleReader = null;
						}

						// Reset current article
					}
					catch (ArticleReaderException e) {

						ArticleConsumerLogMessages.logTaskReaderException(
								logger, e);
						articleReader.resetTaskCompleted();

					}
					catch (DiffException e) {

						DiffConsumerLogMessages.logDiffException(logger, e);
						articleReader.resetTaskCompleted();
						diffCalc.reset();
					}
				}
			}
			diffCalc.closeTransmitter();

			ArticleConsumerLogMessages.logNoMoreArchives(logger);

			// Critical Exceptions
		}
		catch (ConfigurationException e) {
			DiffToolLogMessages.logException(logger, e);
			throw new RuntimeException(e);
		}
		catch (UnsupportedEncodingException e) {
			DiffToolLogMessages.logException(logger, e);
			throw new RuntimeException(e);
		}
		catch (IOException e) {
			DiffToolLogMessages.logException(logger, e);
			throw new RuntimeException(e);
		}
		catch (TimeoutException e) {
			DiffToolLogMessages.logException(logger, e);
			throw new RuntimeException(e);
		}
		catch (Exception e) {
			DiffToolLogMessages.logException(logger, e);
			throw new RuntimeException(e);
		}
	}
156858
function archiveResource(resource, primary, archive) {
        if (!resource.archive) { return true; }

        var name = resource.name;

        primary.bind(name);
        archive.bind(name);

        // get the archive criteria
        var criteria = resource.archive();

        log.info('Getting archive docs for ' + name);

        // find all the documents that match the archive criteria
        return getDocsToArchive(criteria, primary[name])
            .then(function (docs) {

                log.info('Found ' + docs.length + ' ' + name + ' docs to archive');

                // insert all of the documents into the archive database
                var deferred = Q.defer();
                archive[name].insert(docs, function (err, results) {
                    err ? deferred.reject(err) : deferred.resolve(results);
                });
                return deferred.promise;
            })
            .then(function () {

                log.info('Docs inserted into archive DB for ' + name + '. Now deleting docs in main db');

                // delete the data from the primary database
                var deferred = Q.defer();
                primary[name].remove(criteria, function (err, results) {

                    log.info('Deleting docs complete for ' + name);

                    err ? deferred.reject(err) : deferred.resolve(results);
                });
                return deferred.promise;

            });
    }
888616
protected function insertDayReports()
    {
        ArchivingHelper::clearActionsCache();

        $this->insertPageUrlsReports();
        $this->insertDownloadsReports();
        $this->insertOutlinksReports();
        $this->insertPageTitlesReports();
        $this->insertSiteSearchReports();
    }
564755
def publish(self, func):
        """"""
        project = self.session.get_default_project()
        func_name = "projects/{}/locations/{}/functions/{}".format(
            project, self.region, func.name)
        func_info = self.get(func.name)
        source_url = None

        archive = func.get_archive()
        if not func_info or self._delta_source(archive, func_name):
            source_url = self._upload(archive, self.region)

        config = func.get_config()
        config['name'] = func_name
        if source_url:
            config['sourceUploadUrl'] = source_url

        # todo - we'll really need before() and after() for pre-provisioning of
        # resources (ie topic for function stream on create) and post provisioning (schedule
        # invocation of extant function).
        #
        # convergent event source creation
        for e in func.events:
            e.add(func)

        if func_info is None:
            log.info("creating function")
            response = self.client.execute_command(
                'create', {
                    'location': "projects/{}/locations/{}".format(
                        project, self.region),
                    'body': config})
        else:
            delta = delta_resource(func_info, config, ('httpsTrigger',))
            if not delta:
                response = None
            else:
                update_mask = ','.join(delta)
                log.info("updating function config %s", update_mask)
                response = self.client.execute_command(
                    'patch', {
                        'name': func_name,
                        'body': config,
                        'updateMask': update_mask})
        return response
1222075
def main():
    """"""
    org = Organization(name='junk org').create()
    pprint(org.get_values())  # e.g. {'name': 'junk org', …}
    org.delete()
67364
function runProgramLogic() {
  var entries = db.getCollection("entries");
  var entryCount = entries.count();
  var now = new Date();

  console.log("old number of entries in database : " + entryCount);

  entries.insert({ x: now.getTime(), y: 100 - entryCount });
  entryCount = entries.count();

  console.log("new number of entries in database : " + entryCount);
  console.log("");
  
  // manually save
  db.saveDatabase(function(err) {
    if (err) {
      console.log(err);
    }
    else {
      console.log("saved... it can now be loaded or reloaded with up to date data");
    }
  });
}
634674
function archive() {
            vm.error = undefined;
            var promises = [];
            vm.todos.filter(function (todo) {
                return todo.completed;
            }).forEach(function (todo) {
                promises.push(_archive(todo));
            });

            $q.all(promises).then(function () {
                _fetch();
            }).catch(function () {
                vm.error = 'An error occurred when archiving completed todos';
            });
        }
1314377
@Override
    void archive() {
        FTPArchiveClient client = new FTPArchiveClient();
        try {
            client.init(clientArguments);
        } catch (UnknownHostException e) {
            throw new Audit4jRuntimeException("Excepting in running archive client.", e);
        }
    }
611860
def run(connection):
    

    parser = argparse.ArgumentParser(description="""
    Process database dumps.

    Either download of upload a dump file to the objectstore.

    downloads the latest dump and uploads with envronment and date
    into given container destination
    """)

    parser.add_argument(
        'location',
        nargs=1,
        default=f'{DUMPFOLDER}/database.{ENV}.dump',
        help="Dump file location")

    parser.add_argument(
        'objectstore',
        nargs=1,
        default=f'{DUMPFOLDER}/database.{ENV}.dump',
        help="Dump file objectstore location")

    parser.add_argument(
        '--download-db',
        action='store_true',
        dest='download',
        default=False,
        help='Download db')

    parser.add_argument(
        '--upload-db',
        action='store_true',
        dest='upload',
        default=False,
        help='Upload db')

    parser.add_argument(
        '--container',
        action='store_true',
        dest='container',
        default=False,
        help='Upload db')

    parser.add_argument(
        '--days',
        type=int,
        nargs=1,
        dest='days',
        default=0,
        help='Days to keep database dumps')

    args = parser.parse_args()

    if args.days:
        LOG.debug('Cleanup old dumps')
        remove_old_dumps(
            connection, args.objectstore[0], args.days[0])
    elif args.download:
        download_database(
            connection, args.objectstore[0], args.location[0])
    elif args.upload:
        upload_database(
            connection, args.objectstore[0], args.location[0])
    else:
        parser.print_help()
201609
function main() {
    const options = queryOptions();
    switch (options.porting) {
        case 'build':
            runBuild(options)
            .catch((reason) => {
                console.error(reason);
                process.exit(1);
            });
            break;
        case 'arrange':
            runArrange(options);
            break;
        case 'rebase':
            runRebase(options);
            break;
        case 'clean':
            runClean(options);
            break;
    }
}
954139
function archive(cb) {
  db.getClient(function(err, client, done) {
    client.query(qb.select() + ' WHERE published = true', function(err, r) {
      if(err) {
        cb(err);
        done(err);
      } else {
        cb(null, r.rows);
        done();
      }
    });
  });
}
781647
private function runHousekeeping(): void
    {
        $data_filesystem = app(FilesystemInterface::class);
        $root_filesystem = new Filesystem(new Local(Webtrees::ROOT_DIR));

        // Clear files in the (user-specified) data folder - which might not be local files
        $this->housekeeping_service->deleteOldFiles($data_filesystem, 'cache', self::MAX_CACHE_AGE);

        $this->housekeeping_service->deleteOldFiles($data_filesystem, 'thumbnail-cache', self::MAX_THUMBNAIL_AGE);

        // Clear files in /data - which need to be local files
        $this->housekeeping_service->deleteOldFiles($root_filesystem, self::TMP_DIR, self::MAX_TMP_FILE_AGE);

        // Clear entries in database tables
        $this->housekeeping_service->deleteOldLogs(self::MAX_LOG_AGE);

        $this->housekeeping_service->deleteOldSessions(self::MAX_SESSION_AGE);
    }
165564
def main(*argv):
    
    try:
        #   Inputs
        #
        adminUsername = argv[0]
        adminPassword = argv[1]
        siteURL = argv[2]
        groupTitle = argv[3]
        groupTags = argv[4]
        description = argv[5]
        access = argv[6]

        #   Logic
        #
        #   Connect to the site
        #
        sh = arcrest.AGOLTokenSecurityHandler(adminUsername, adminPassword)
        admin = arcrest.manageorg.Administration(url=siteURL,
                                                securityHandler=sh,
                                                initialize=True)
        community = admin.community
        #   Create Group
        #
        res = community.createGroup(title=groupTitle,
                                    tags=groupTags,
                                    description=description,
                                    snippet="",
                                    phone="",
                                    access=access,
                                    sortField="title",
                                    sortOrder="asc",
                                    isViewOnly=False,
                                    isInvitationOnly=False,
                                    thumbnail=None)
        arcpy.SetParameterAsText(7, str(res))
    except arcpy.ExecuteError:
        line, filename, synerror = trace()
        arcpy.AddError("error on line: %s" % line)
        arcpy.AddError("error in file name: %s" % filename)
        arcpy.AddError("with error message: %s" % synerror)
        arcpy.AddError("ArcPy Error Message: %s" % arcpy.GetMessages(2))
    except FunctionError, f_e:
        messages = f_e.args[0]
        arcpy.AddError("error in function: %s" % messages["function"])
        arcpy.AddError("error on line: %s" % messages["line"])
        arcpy.AddError("error in file name: %s" % messages["filename"])
        arcpy.AddError("with error message: %s" % messages["synerror"])
        arcpy.AddError("ArcPy Error Message: %s" % messages["arc"])
    except:
        line, filename, synerror = trace()
        arcpy.AddError("error on line: %s" % line)
        arcpy.AddError("error in file name: %s" % filename)
        arcpy.AddError("with error message: %s" % synerror)