<commit_before>#include <map>
#include <algorithm>
#include "cantera/oneD/Domain1D.h"

#include "cantera/oneD/refine.h"

using namespace std;

namespace Cantera
{

template<class M>
bool has_key(const M& m, size_t j)
{
    if (m.find(j) != m.end()) {
        return true;
    }
    return false;
}

static void r_drawline()
{
    string s(78,'#');
    s += '\n';
    writelog(s.c_str());
}

/**
 * Return the square root of machine precision.
 */
static doublereal eps()
{
    doublereal e = 1.0;
    while (1.0 + e != 1.0) {
        e *= 0.5;
    }
    return sqrt(e);
}

Refiner::Refiner(Domain1D& domain) :
    m_ratio(10.0), m_slope(0.8), m_curve(0.8), m_prune(-0.001),
    m_min_range(0.01), m_domain(&domain), m_npmax(3000)
{
    m_nv = m_domain->nComponents();
    m_active.resize(m_nv, true);
    m_thresh = eps();
}

int Refiner::analyze(size_t n, const doublereal* z,
                     const doublereal* x)
{

    if (n >= m_npmax) {
        writelog("max number of grid points reached ("+int2str(m_npmax)+".\n");
        return -2;
    }

    if (m_domain->nPoints() <= 1) {
        //writelog("can't refine a domain with 1 point: "+m_domain->id()+"\n");
        return 0;
    }

    m_loc.clear();
    m_c.clear();
    m_keep.clear();

    m_keep[0] = 1;
    m_keep[n-1] = 1;

    m_nv = m_domain->nComponents();

    // check consistency
    if (n != m_domain->nPoints()) {
        throw CanteraError("analyze","inconsistent");
    }

    /**
     * find locations where cell size ratio is too large.
     */
    size_t j;
    vector_fp dz(n-1, 0.0);
    string name;
    doublereal vmin, vmax, smin, smax, aa, ss;
    doublereal dmax, r;
    vector_fp v(n), s(n-1);

    for (size_t i = 0; i < m_nv; i++) {
        if (m_active[i]) {
            name = m_domain->componentName(i);
            //writelog("refine: examining "+name+"\n");
            // get component i at all points
            for (j = 0; j < n; j++) {
                v[j] = value(x, i, j);
            }

            // slope of component i
            for (j = 0; j < n-1; j++)
                s[j] = (value(x, i, j+1) - value(x, i, j))/
                       (z[j+1] - z[j]);

            // find the range of values and slopes

            vmin = *min_element(v.begin(), v.end());
            vmax = *max_element(v.begin(), v.end());
            smin = *min_element(s.begin(), s.end());
            smax = *max_element(s.begin(), s.end());

            // max absolute values of v and s
            aa = std::max(fabs(vmax), fabs(vmin));
            ss = std::max(fabs(smax), fabs(smin));

            // refine based on component i only if the range of v is
            // greater than a fraction 'min_range' of max |v|. This
            // eliminates components that consist of small fluctuations
            // on a constant background.

            if ((vmax - vmin) > m_min_range*aa) {

                // maximum allowable difference in value between
                // adjacent points.

                dmax = m_slope*(vmax - vmin) + m_thresh;
                for (j = 0; j < n-1; j++) {
                    r = fabs(v[j+1] - v[j])/dmax;
                    if (r > 1.0) {
                        m_loc[j] = 1;
                        m_c[name] = 1;
                        //if (int(m_loc.size()) + n > m_npmax) goto done;
                    }
                    if (r >= m_prune) {
                        m_keep[j] = 1;
                        m_keep[j+1] = 1;
                    } else {
                        //writelog(string("r = ")+fp2str(r)+"\n");
                        if (m_keep[j] == 0) {
                            //if (m_keep[j-1] > -1 && m_keep[j+1] > -1)
                            m_keep[j] = -1;
                        }
                        //if (m_keep[j+1] == 0) m_keep[j+1] = -1;
                    }
                }
            }

            // refine based on the slope of component i only if the
            // range of s is greater than a fraction 'min_range' of max
            // |s|. This eliminates components that consist of small
            // fluctuations on a constant slope background.

            if ((smax - smin) > m_min_range*ss) {

                // maximum allowable difference in slope between
                // adjacent points.
                dmax = m_curve*(smax - smin); // + 0.5*m_curve*(smax + smin);
                for (j = 0; j < n-2; j++) {
                    r = fabs(s[j+1] - s[j]) / (dmax + m_thresh/dz[j]);
                    if (r > 1.0) {
                        m_c[name] = 1;
                        m_loc[j] = 1;
                        m_loc[j+1] = 1;
                        //if (int(m_loc.size()) + n > m_npmax) goto done;
                    }
                    if (r >= m_prune) {
                        m_keep[j+1] = 1;
                    } else {
                        //writelog(string("r slope = ")+fp2str(r)+"\n");
                        if (m_keep[j+1] == 0) {
                            //if (m_keep[j] > -1 && m_keep[j+2] > -1)
                            m_keep[j+1] = -1;
                        }
                    }
                }
            }

        }
    }

    dz[0] = z[1] - z[0];
    for (j = 1; j < n-1; j++) {
        dz[j] = z[j+1] - z[j];
        if (dz[j] > m_ratio*dz[j-1]) {
            m_loc[j] = 1;
            m_c["point "+int2str(j)] = 1;
        }
        if (dz[j] < dz[j-1]/m_ratio) {
            m_loc[j-1] = 1;
            m_c["point "+int2str(j-1)] = 1;
        }
        //if (m_loc.size() + n > m_npmax) goto done;
    }

    //done:
    //m_did_analysis = true;
    return int(m_loc.size());
}

double Refiner::value(const double* x, size_t i, size_t j)
{
    return x[m_domain->index(i,j)];
}

void Refiner::show()
{
    int nnew = static_cast<int>(m_loc.size());
    if (nnew > 0) {
        r_drawline();
        writelog(string("Refining grid in ") +
                 m_domain->id()+".\n"
                 +"    New points inserted after grid points ");
        map<size_t, int>::const_iterator b = m_loc.begin();
        for (; b != m_loc.end(); ++b) {
            writelog(int2str(b->first)+" ");
        }
        writelog("\n");
        writelog("    to resolve ");
        map<string, int>::const_iterator bb = m_c.begin();
        for (; bb != m_c.end(); ++bb) {
            writelog(string(bb->first)+" ");
        }
        writelog("\n");
    } else if (m_domain->nPoints() > 1) {
        writelog("no new points needed in "+m_domain->id()+"\n");
        //writelog("curve = "+fp2str(m_curve)+"\n");
        //writelog("slope = "+fp2str(m_slope)+"\n");
        //writelog("prune = "+fp2str(m_prune)+"\n");
    }
}

int Refiner::getNewGrid(int n, const doublereal* z,
                        int nn, doublereal* zn)
{
    int j;
    int nnew = static_cast<int>(m_loc.size());
    if (nnew + n > nn) {
        throw CanteraError("Refine::getNewGrid",
                           "array size too small.");
        return -1;
    }

    int jn = 0;
    if (m_loc.empty()) {
        copy(z, z + n,  zn);
        return 0;
    }

    for (j = 0; j < n - 1; j++) {
        zn[jn] = z[j];
        jn++;
        if (has_key(m_loc, j)) {
            zn[jn] = 0.5*(z[j] + z[j+1]);
            jn++;
        }
    }
    zn[jn] = z[n-1];
    return 0;
}
}
<commit_msg>Fixed 1D grid refinement based on curvature requirement<commit_after>#include <map>
#include <algorithm>
#include "cantera/oneD/Domain1D.h"

#include "cantera/oneD/refine.h"

using namespace std;

namespace Cantera
{

template<class M>
bool has_key(const M& m, size_t j)
{
    if (m.find(j) != m.end()) {
        return true;
    }
    return false;
}

static void r_drawline()
{
    string s(78,'#');
    s += '\n';
    writelog(s.c_str());
}

/**
 * Return the square root of machine precision.
 */
static doublereal eps()
{
    doublereal e = 1.0;
    while (1.0 + e != 1.0) {
        e *= 0.5;
    }
    return sqrt(e);
}

Refiner::Refiner(Domain1D& domain) :
    m_ratio(10.0), m_slope(0.8), m_curve(0.8), m_prune(-0.001),
    m_min_range(0.01), m_domain(&domain), m_npmax(3000)
{
    m_nv = m_domain->nComponents();
    m_active.resize(m_nv, true);
    m_thresh = eps();
}

int Refiner::analyze(size_t n, const doublereal* z,
                     const doublereal* x)
{

    if (n >= m_npmax) {
        writelog("max number of grid points reached ("+int2str(m_npmax)+".\n");
        return -2;
    }

    if (m_domain->nPoints() <= 1) {
        //writelog("can't refine a domain with 1 point: "+m_domain->id()+"\n");
        return 0;
    }

    m_loc.clear();
    m_c.clear();
    m_keep.clear();

    m_keep[0] = 1;
    m_keep[n-1] = 1;

    m_nv = m_domain->nComponents();

    // check consistency
    if (n != m_domain->nPoints()) {
        throw CanteraError("analyze","inconsistent");
    }

    /**
     * find locations where cell size ratio is too large.
     */
    size_t j;
    string name;
    doublereal vmin, vmax, smin, smax, aa, ss;
    doublereal dmax, r;
    vector_fp v(n), s(n-1);

    vector_fp dz(n-1);
    for (j = 0; j < n-1; j++) {
        dz[j] = z[j+1] - z[j];
    }

    for (size_t i = 0; i < m_nv; i++) {
        if (m_active[i]) {
            name = m_domain->componentName(i);
            //writelog("refine: examining "+name+"\n");
            // get component i at all points
            for (j = 0; j < n; j++) {
                v[j] = value(x, i, j);
            }

            // slope of component i
            for (j = 0; j < n-1; j++)
                s[j] = (value(x, i, j+1) - value(x, i, j))/
                       (z[j+1] - z[j]);

            // find the range of values and slopes

            vmin = *min_element(v.begin(), v.end());
            vmax = *max_element(v.begin(), v.end());
            smin = *min_element(s.begin(), s.end());
            smax = *max_element(s.begin(), s.end());

            // max absolute values of v and s
            aa = std::max(fabs(vmax), fabs(vmin));
            ss = std::max(fabs(smax), fabs(smin));

            // refine based on component i only if the range of v is
            // greater than a fraction 'min_range' of max |v|. This
            // eliminates components that consist of small fluctuations
            // on a constant background.

            if ((vmax - vmin) > m_min_range*aa) {

                // maximum allowable difference in value between
                // adjacent points.

                dmax = m_slope*(vmax - vmin) + m_thresh;
                for (j = 0; j < n-1; j++) {
                    r = fabs(v[j+1] - v[j])/dmax;
                    if (r > 1.0) {
                        m_loc[j] = 1;
                        m_c[name] = 1;
                        //if (int(m_loc.size()) + n > m_npmax) goto done;
                    }
                    if (r >= m_prune) {
                        m_keep[j] = 1;
                        m_keep[j+1] = 1;
                    } else {
                        //writelog(string("r = ")+fp2str(r)+"\n");
                        if (m_keep[j] == 0) {
                            //if (m_keep[j-1] > -1 && m_keep[j+1] > -1)
                            m_keep[j] = -1;
                        }
                        //if (m_keep[j+1] == 0) m_keep[j+1] = -1;
                    }
                }
            }

            // refine based on the slope of component i only if the
            // range of s is greater than a fraction 'min_range' of max
            // |s|. This eliminates components that consist of small
            // fluctuations on a constant slope background.

            if ((smax - smin) > m_min_range*ss) {

                // maximum allowable difference in slope between
                // adjacent points.
                dmax = m_curve*(smax - smin); // + 0.5*m_curve*(smax + smin);
                for (j = 0; j < n-2; j++) {
                    r = fabs(s[j+1] - s[j]) / (dmax + m_thresh/dz[j]);
                    if (r > 1.0) {
                        m_c[name] = 1;
                        m_loc[j] = 1;
                        m_loc[j+1] = 1;
                        //if (int(m_loc.size()) + n > m_npmax) goto done;
                    }
                    if (r >= m_prune) {
                        m_keep[j+1] = 1;
                    } else {
                        //writelog(string("r slope = ")+fp2str(r)+"\n");
                        if (m_keep[j+1] == 0) {
                            //if (m_keep[j] > -1 && m_keep[j+2] > -1)
                            m_keep[j+1] = -1;
                        }
                    }
                }
            }

        }
    }

    for (j = 1; j < n-1; j++) {
        if (dz[j] > m_ratio*dz[j-1]) {
            m_loc[j] = 1;
            m_c["point "+int2str(j)] = 1;
        }
        if (dz[j] < dz[j-1]/m_ratio) {
            m_loc[j-1] = 1;
            m_c["point "+int2str(j-1)] = 1;
        }
        //if (m_loc.size() + n > m_npmax) goto done;
    }

    //done:
    //m_did_analysis = true;
    return int(m_loc.size());
}

double Refiner::value(const double* x, size_t i, size_t j)
{
    return x[m_domain->index(i,j)];
}

void Refiner::show()
{
    int nnew = static_cast<int>(m_loc.size());
    if (nnew > 0) {
        r_drawline();
        writelog(string("Refining grid in ") +
                 m_domain->id()+".\n"
                 +"    New points inserted after grid points ");
        map<size_t, int>::const_iterator b = m_loc.begin();
        for (; b != m_loc.end(); ++b) {
            writelog(int2str(b->first)+" ");
        }
        writelog("\n");
        writelog("    to resolve ");
        map<string, int>::const_iterator bb = m_c.begin();
        for (; bb != m_c.end(); ++bb) {
            writelog(string(bb->first)+" ");
        }
        writelog("\n");
    } else if (m_domain->nPoints() > 1) {
        writelog("no new points needed in "+m_domain->id()+"\n");
        //writelog("curve = "+fp2str(m_curve)+"\n");
        //writelog("slope = "+fp2str(m_slope)+"\n");
        //writelog("prune = "+fp2str(m_prune)+"\n");
    }
}

int Refiner::getNewGrid(int n, const doublereal* z,
                        int nn, doublereal* zn)
{
    int j;
    int nnew = static_cast<int>(m_loc.size());
    if (nnew + n > nn) {
        throw CanteraError("Refine::getNewGrid",
                           "array size too small.");
        return -1;
    }

    int jn = 0;
    if (m_loc.empty()) {
        copy(z, z + n,  zn);
        return 0;
    }

    for (j = 0; j < n - 1; j++) {
        zn[jn] = z[j];
        jn++;
        if (has_key(m_loc, j)) {
            zn[jn] = 0.5*(z[j] + z[j+1]);
            jn++;
        }
    }
    zn[jn] = z[n-1];
    return 0;
}
}
<|endoftext|>


<commit_before>/** \file    generate_new_journal_alert_stats.cc
 *  \brief   Generates a statics report for our journal alerts.
 *  \note    Additional documentation can be found at
 *           https://github.com/ubtue/ub_tools/wiki/Abonnementservice-f%C3%BCr-Zeitschriftenartikel-in-IxTheo-und-RelBib
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2021 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <vector>
#include <cstdlib>
#include <cstring>
#include "BinaryIO.h"
#include "Compiler.h"
#include "DbConnection.h"
#include "DnsUtil.h"
#include "EmailSender.h"
#include "FileUtil.h"
#include "IniFile.h"
#include "Solr.h"
#include "StringUtil.h"
#include "TextUtil.h"
#include "TimeUtil.h"
#include "UBTools.h"
#include "util.h"
#include "VuFind.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("[solr_host_and_port] user_type report_interval_in_days)\n"
            "  Generates a CSV report about journal subscription statistics.\n"
            "  Should \"solr_host_and_port\" be missing \"" + Solr::DEFAULT_HOST + ":"
            + std::to_string(Solr::DEFAULT_PORT) + "\" will be used.\n"
            "  \"user_type\" must be \"ixtheo\", \"relbib\" or some other realm.\n"
            "  \"report_interval_in_days\" can be a number or the text \"days_in_last_month\".\n");
}

struct Stats {
    unsigned no_of_users_with_subscriptions_;
    double average_number_of_bundle_subscriptions_;
    double average_subscriptions_per_user_;
    unsigned no_of_subscribed_journals_;
    unsigned no_of_journals_for_which_notifications_were_sent_;
    unsigned no_of_subscribed_journals_with_notifications_;
    double average_number_of_notified_articles_per_notified_journal_;
    unsigned report_interval_in_days_;
};

inline bool IsBundle(const std::string &serial_control_number) {
    if (serial_control_number.empty())
        return false;
    return not std::isdigit(serial_control_number[0]);
}

size_t GetBundleSize(const IniFile &bundles_config, const std::string &bundle_name) {
    static std::map<std::string, size_t> bundle_names_to_sizes_map;
    const auto bundle_name_and_size(bundle_names_to_sizes_map.find(bundle_name));
    if (bundle_name_and_size != bundle_names_to_sizes_map.cend())
        return bundle_name_and_size->second;

    const std::string bundle_ppns_string(bundles_config.getString(bundle_name, "ppns", ""));
    if (unlikely(bundle_ppns_string.empty()))
        LOG_ERROR("bundle \"" + bundle_name + "\" not found in \"" + bundles_config.getFilename() + "\"!");

    std::vector<std::string> bundle_ppns;
    StringUtil::SplitThenTrim(bundle_ppns_string, "," , " \t", &bundle_ppns);
    bundle_names_to_sizes_map[bundle_name] = bundle_ppns.size();

    return bundle_ppns.size();
}

void CollectConfigStats(DbConnection * const db_connection, const std::string &user_type, Stats * const stats) {
    db_connection->queryOrDie("SELECT DISTINCT user_id FROM ixtheo_journal_subscriptions WHERE user_id IN (SELECT id FROM "
                              "ixtheo_user WHERE ixtheo_user.user_type = '" + user_type  + "')");
    auto user_ids_result_set(db_connection->getLastResultSet());
    stats->no_of_users_with_subscriptions_ = user_ids_result_set.size();

    const IniFile bundles_config(UBTools::GetTuelibPath() + "journal_alert_bundles.conf");
    unsigned no_of_individual_subscriptions(0), no_of_bundle_subscriptions(0);
    while (const auto user_id_row = user_ids_result_set.getNextRow()) {
        const auto user_id(user_id_row["user_id"]);
        db_connection->queryOrDie("SELECT journal_control_number_or_bundle_name FROM "
                                  "ixtheo_journal_subscriptions WHERE user_id=" + user_id);
        auto journal_control_number_or_bundle_name_result_set(db_connection->getLastResultSet());
        while (const auto journal_control_number_or_bundle_name_row
               = journal_control_number_or_bundle_name_result_set.getNextRow())
        {
            const auto journal_control_number_or_bundle_name(
                journal_control_number_or_bundle_name_row["journal_control_number_or_bundle_name"]);
            if (IsBundle(journal_control_number_or_bundle_name)) {
                ++no_of_bundle_subscriptions;
                no_of_individual_subscriptions += GetBundleSize(bundles_config, journal_control_number_or_bundle_name);
            }
        }
    }

    stats->average_number_of_bundle_subscriptions_ = double(no_of_bundle_subscriptions) / stats->no_of_users_with_subscriptions_;
    stats->average_subscriptions_per_user_ = double(no_of_individual_subscriptions) / stats->no_of_users_with_subscriptions_;
}

void CollectUsageStats(const std::string &user_type, Stats * const stats) {
    const auto USAGE_STATS_PATH(UBTools::GetTuelibPath() + "new_journal_alert.stats");
    const auto usage_stats_file(FileUtil::OpenInputFileOrDie(USAGE_STATS_PATH));

    const double NOW(TimeUtil::GetJulianDayNumber());
    const double TIME_WINDOW(NOW - stats->report_interval_in_days_);

    stats->no_of_subscribed_journals_with_notifications_ = 0;
    stats->average_number_of_notified_articles_per_notified_journal_ = 0.0;
    std::unordered_set<std::string> seen_superior_ppns;
    const auto USAGE_STATS_FILE_SIZE(usage_stats_file->size());
    while (usage_stats_file->tell() < USAGE_STATS_FILE_SIZE) {
        double julian_day_number;
        BinaryIO::ReadOrDie(*usage_stats_file, &julian_day_number);
        std::string logged_user_type;
        BinaryIO::ReadOrDie(*usage_stats_file, &logged_user_type);
        std::string journal_ppn;
        BinaryIO::ReadOrDie(*usage_stats_file, &journal_ppn);
        unsigned notified_count;
        BinaryIO::ReadOrDie(*usage_stats_file, &notified_count);

        if (julian_day_number > TIME_WINDOW and logged_user_type == user_type) {
            if (seen_superior_ppns.find(journal_ppn) == seen_superior_ppns.end()) {
                seen_superior_ppns.emplace(journal_ppn);
                ++(stats->no_of_subscribed_journals_with_notifications_);
            }
            stats->average_number_of_notified_articles_per_notified_journal_ += notified_count;
        }
    }

    stats->average_number_of_notified_articles_per_notified_journal_ /=
        stats->no_of_subscribed_journals_with_notifications_;
}

void GenerateReport(File * const report_file, const Stats &stats) {
    (*report_file) << "\"Report interval in days\"," << stats.report_interval_in_days_ << '\n'
                   << "\"Number of users w/ subscriptions\"," << stats.no_of_users_with_subscriptions_ << '\n'
                   << "\"Average number of subscriptions per user\"," << stats.average_subscriptions_per_user_ << '\n'
                   << "\"Average number of bundle subscriptions per user\","
                   << stats.average_number_of_bundle_subscriptions_ << '\n'
                   << "\"Total number of currently subscribed journals\"," << stats.no_of_subscribed_journals_ << '\n'
                   << "\"Number of subscribed journals w/ notifications\","
                   << stats.no_of_subscribed_journals_with_notifications_ << '\n'
                   << "\"Average number of notified articles per notified journal\","
                   << stats.average_number_of_notified_articles_per_notified_journal_ << '\n';
}

} // unnamed namespace

const std::string REPORT_DIRECTORY("/mnt/ZE020110/FID-Projekte/Statistik/"); // Must end w/ a slash!

// gets user subscriptions for superior works from MySQL
// uses a KeyValueDB instance to prevent entries from being sent multiple times to same user
int Main(int argc, char **argv) {
    if (argc != 4 and argc != 5)
        Usage();

    std::string solr_host_and_port;
    if (argc == 4)
        solr_host_and_port = Solr::DEFAULT_HOST + ":" + std::to_string(Solr::DEFAULT_PORT);
    else {
        solr_host_and_port = argv[1];
        --argc, ++argv;
    }

    const std::string user_type(argv[1]);
    if (user_type != "ixtheo" and user_type != "relbib")
        LOG_ERROR("user_type parameter must be either \"ixtheo\" or \"relbib\"!");

    Stats stats;
    if (std::strcmp(argv[2], "days_in_last_month") != 0)
        stats.report_interval_in_days_ = StringUtil::ToUnsigned(argv[2]);
    else {
        unsigned year, month, day;
        TimeUtil::GetCurrentDate(&year, &month, &day);
        if (month != 1)
            --month;
        else {
            month = 12;
            --year;
        }
        stats.report_interval_in_days_ = TimeUtil::GetDaysInMonth(year, month);
    }

    const std::string email_recipient(argv[3]);
    std::shared_ptr<DbConnection> db_connection(VuFind::GetDbConnection());
    CollectConfigStats(db_connection.get(), user_type, &stats);
    CollectUsageStats(user_type, &stats);

    const auto report_file(FileUtil::OpenOutputFileOrDie(REPORT_DIRECTORY + "new_journal_alert_stats."
                                                         + DnsUtil::GetHostname() + "." + user_type
                                                         + "." + TimeUtil::GetCurrentDateAndTime("%Y-%m-%d") + ".csv"));
    GenerateReport(report_file.get(), stats);

    return EXIT_SUCCESS;
}
<commit_msg>changed usage<commit_after>/** \file    generate_new_journal_alert_stats.cc
 *  \brief   Generates a statics report for our journal alerts.
 *  \note    Additional documentation can be found at
 *           https://github.com/ubtue/ub_tools/wiki/Abonnementservice-f%C3%BCr-Zeitschriftenartikel-in-IxTheo-und-RelBib
 *  \author  Dr. Johannes Ruscheinski
 */

/*
    Copyright (C) 2021 Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <vector>
#include <cstdlib>
#include <cstring>
#include "BinaryIO.h"
#include "Compiler.h"
#include "DbConnection.h"
#include "DnsUtil.h"
#include "EmailSender.h"
#include "FileUtil.h"
#include "IniFile.h"
#include "Solr.h"
#include "StringUtil.h"
#include "TextUtil.h"
#include "TimeUtil.h"
#include "UBTools.h"
#include "util.h"
#include "VuFind.h"

namespace {

[[noreturn]] void Usage() {
    ::Usage("[solr_host_and_port] user_type report_interval_in_days email\n"
            "  Generates a CSV report about journal subscription statistics.\n"
            "  Should \"solr_host_and_port\" be missing \"" + Solr::DEFAULT_HOST + ":"
            + std::to_string(Solr::DEFAULT_PORT) + "\" will be used.\n"
            "  \"user_type\" must be \"ixtheo\", \"relbib\" or some other realm.\n"
            "  \"report_interval_in_days\" can be a number or the text \"days_in_last_month\n"
            "  \"email\" recipient email address.\n");
}

struct Stats {
    unsigned no_of_users_with_subscriptions_;
    double average_number_of_bundle_subscriptions_;
    double average_subscriptions_per_user_;
    unsigned no_of_subscribed_journals_;
    unsigned no_of_journals_for_which_notifications_were_sent_;
    unsigned no_of_subscribed_journals_with_notifications_;
    double average_number_of_notified_articles_per_notified_journal_;
    unsigned report_interval_in_days_;
};

inline bool IsBundle(const std::string &serial_control_number) {
    if (serial_control_number.empty())
        return false;
    return not std::isdigit(serial_control_number[0]);
}

size_t GetBundleSize(const IniFile &bundles_config, const std::string &bundle_name) {
    static std::map<std::string, size_t> bundle_names_to_sizes_map;
    const auto bundle_name_and_size(bundle_names_to_sizes_map.find(bundle_name));
    if (bundle_name_and_size != bundle_names_to_sizes_map.cend())
        return bundle_name_and_size->second;

    const std::string bundle_ppns_string(bundles_config.getString(bundle_name, "ppns", ""));
    if (unlikely(bundle_ppns_string.empty()))
        LOG_ERROR("bundle \"" + bundle_name + "\" not found in \"" + bundles_config.getFilename() + "\"!");

    std::vector<std::string> bundle_ppns;
    StringUtil::SplitThenTrim(bundle_ppns_string, "," , " \t", &bundle_ppns);
    bundle_names_to_sizes_map[bundle_name] = bundle_ppns.size();

    return bundle_ppns.size();
}

void CollectConfigStats(DbConnection * const db_connection, const std::string &user_type, Stats * const stats) {
    db_connection->queryOrDie("SELECT DISTINCT user_id FROM ixtheo_journal_subscriptions WHERE user_id IN (SELECT id FROM "
                              "ixtheo_user WHERE ixtheo_user.user_type = '" + user_type  + "')");
    auto user_ids_result_set(db_connection->getLastResultSet());
    stats->no_of_users_with_subscriptions_ = user_ids_result_set.size();

    const IniFile bundles_config(UBTools::GetTuelibPath() + "journal_alert_bundles.conf");
    unsigned no_of_individual_subscriptions(0), no_of_bundle_subscriptions(0);
    while (const auto user_id_row = user_ids_result_set.getNextRow()) {
        const auto user_id(user_id_row["user_id"]);
        db_connection->queryOrDie("SELECT journal_control_number_or_bundle_name FROM "
                                  "ixtheo_journal_subscriptions WHERE user_id=" + user_id);
        auto journal_control_number_or_bundle_name_result_set(db_connection->getLastResultSet());
        while (const auto journal_control_number_or_bundle_name_row
               = journal_control_number_or_bundle_name_result_set.getNextRow())
        {
            const auto journal_control_number_or_bundle_name(
                journal_control_number_or_bundle_name_row["journal_control_number_or_bundle_name"]);
            if (IsBundle(journal_control_number_or_bundle_name)) {
                ++no_of_bundle_subscriptions;
                no_of_individual_subscriptions += GetBundleSize(bundles_config, journal_control_number_or_bundle_name);
            }
        }
    }

    stats->average_number_of_bundle_subscriptions_ = double(no_of_bundle_subscriptions) / stats->no_of_users_with_subscriptions_;
    stats->average_subscriptions_per_user_ = double(no_of_individual_subscriptions) / stats->no_of_users_with_subscriptions_;
}

void CollectUsageStats(const std::string &user_type, Stats * const stats) {
    const auto USAGE_STATS_PATH(UBTools::GetTuelibPath() + "new_journal_alert.stats");
    const auto usage_stats_file(FileUtil::OpenInputFileOrDie(USAGE_STATS_PATH));

    const double NOW(TimeUtil::GetJulianDayNumber());
    const double TIME_WINDOW(NOW - stats->report_interval_in_days_);

    stats->no_of_subscribed_journals_with_notifications_ = 0;
    stats->average_number_of_notified_articles_per_notified_journal_ = 0.0;
    std::unordered_set<std::string> seen_superior_ppns;
    const auto USAGE_STATS_FILE_SIZE(usage_stats_file->size());
    while (usage_stats_file->tell() < USAGE_STATS_FILE_SIZE) {
        double julian_day_number;
        BinaryIO::ReadOrDie(*usage_stats_file, &julian_day_number);
        std::string logged_user_type;
        BinaryIO::ReadOrDie(*usage_stats_file, &logged_user_type);
        std::string journal_ppn;
        BinaryIO::ReadOrDie(*usage_stats_file, &journal_ppn);
        unsigned notified_count;
        BinaryIO::ReadOrDie(*usage_stats_file, &notified_count);

        if (julian_day_number > TIME_WINDOW and logged_user_type == user_type) {
            if (seen_superior_ppns.find(journal_ppn) == seen_superior_ppns.end()) {
                seen_superior_ppns.emplace(journal_ppn);
                ++(stats->no_of_subscribed_journals_with_notifications_);
            }
            stats->average_number_of_notified_articles_per_notified_journal_ += notified_count;
        }
    }

    stats->average_number_of_notified_articles_per_notified_journal_ /=
        stats->no_of_subscribed_journals_with_notifications_;
}

void GenerateReport(File * const report_file, const Stats &stats) {
    (*report_file) << "\"Report interval in days\"," << stats.report_interval_in_days_ << '\n'
                   << "\"Number of users w/ subscriptions\"," << stats.no_of_users_with_subscriptions_ << '\n'
                   << "\"Average number of subscriptions per user\"," << stats.average_subscriptions_per_user_ << '\n'
                   << "\"Average number of bundle subscriptions per user\","
                   << stats.average_number_of_bundle_subscriptions_ << '\n'
                   << "\"Total number of currently subscribed journals\"," << stats.no_of_subscribed_journals_ << '\n'
                   << "\"Number of subscribed journals w/ notifications\","
                   << stats.no_of_subscribed_journals_with_notifications_ << '\n'
                   << "\"Average number of notified articles per notified journal\","
                   << stats.average_number_of_notified_articles_per_notified_journal_ << '\n';
}

} // unnamed namespace

const std::string REPORT_DIRECTORY("/mnt/ZE020110/FID-Projekte/Statistik/"); // Must end w/ a slash!

// gets user subscriptions for superior works from MySQL
// uses a KeyValueDB instance to prevent entries from being sent multiple times to same user
int Main(int argc, char **argv) {
    if (argc != 4 and argc != 5)
        Usage();

    std::string solr_host_and_port;
    if (argc == 4)
        solr_host_and_port = Solr::DEFAULT_HOST + ":" + std::to_string(Solr::DEFAULT_PORT);
    else {
        solr_host_and_port = argv[1];
        --argc, ++argv;
    }

    const std::string user_type(argv[1]);
    if (user_type != "ixtheo" and user_type != "relbib")
        LOG_ERROR("user_type parameter must be either \"ixtheo\" or \"relbib\"!");

    Stats stats;
    if (std::strcmp(argv[2], "days_in_last_month") != 0)
        stats.report_interval_in_days_ = StringUtil::ToUnsigned(argv[2]);
    else {
        unsigned year, month, day;
        TimeUtil::GetCurrentDate(&year, &month, &day);
        if (month != 1)
            --month;
        else {
            month = 12;
            --year;
        }
        stats.report_interval_in_days_ = TimeUtil::GetDaysInMonth(year, month);
    }

    const std::string email_recipient(argv[3]);
    std::shared_ptr<DbConnection> db_connection(VuFind::GetDbConnection());
    CollectConfigStats(db_connection.get(), user_type, &stats);
    CollectUsageStats(user_type, &stats);

    const auto report_file(FileUtil::OpenOutputFileOrDie(REPORT_DIRECTORY + "new_journal_alert_stats."
                                                         + DnsUtil::GetHostname() + "." + user_type
                                                         + "." + TimeUtil::GetCurrentDateAndTime("%Y-%m-%d") + ".csv"));
    GenerateReport(report_file.get(), stats);

    return EXIT_SUCCESS;
}
<|endoftext|>


<commit_before>/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 - 2017 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *          Alexandre Fernandez <nerf@boboop.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#if HAVE_CONFIG_H
# include "config.h"
#endif

#include "Task.h"

#include "filesystem/IFile.h"
#include "filesystem/IDirectory.h"
#include "File.h"
#include "Folder.h"
#include "Playlist.h"
#include "parser/Task.h"
#include "utils/Filename.h"
#include "utils/Url.h"

namespace medialibrary
{

const std::string policy::TaskTable::Name = "Task";
const std::string policy::TaskTable::PrimaryKeyColumn = "id_task";
int64_t parser::Task::* const policy::TaskTable::PrimaryKey = &parser::Task::m_id;

namespace parser
{

Task::Task( MediaLibraryPtr ml, sqlite::Row& row )
    : currentService( 0 )
    , m_ml( ml )
{
    row >> m_id
        >> m_step
        >> m_retryCount
        >> mrl
        >> m_fileId
        >> m_parentFolderId
        >> m_parentPlaylistId
        >> parentPlaylistIndex;
}

Task::Task( MediaLibraryPtr ml, std::shared_ptr<fs::IFile> fileFs,
            std::shared_ptr<Folder> parentFolder,
            std::shared_ptr<fs::IDirectory> parentFolderFs,
            std::shared_ptr<Playlist> parentPlaylist,
            unsigned int parentPlaylistIndex )
    : fileFs( std::move( fileFs ) )
    , parentFolder( std::move( parentFolder ) )
    , parentFolderFs( std::move( parentFolderFs ) )
    , parentPlaylist( std::move( parentPlaylist ) )
    , parentPlaylistIndex( parentPlaylistIndex )
    , mrl( this->fileFs->mrl() )
    , currentService( 0 )
    , m_ml( ml )
    , m_step( ParserStep::None )
    , m_fileId( 0 )
{
}

void Task::markStepCompleted( ParserStep stepCompleted )
{
    m_step = static_cast<ParserStep>( static_cast<uint8_t>( m_step ) |
                                      static_cast<uint8_t>( stepCompleted ) );
}

void Task::markStepUncompleted( ParserStep stepUncompleted )
{
    m_step = static_cast<ParserStep>( static_cast<uint8_t>( m_step ) &
                                      ( ~ static_cast<uint8_t>( stepUncompleted ) ) );
}

bool Task::saveParserStep()
{
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET step = ?, "
            "retry_count = 0 WHERE id_task = ?";
    if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, m_step, m_id ) == false )
        return false;
    return true;
}

bool Task::isCompleted() const
{
    return m_step == ParserStep::Completed;
}

bool Task::isStepCompleted( Task::ParserStep step ) const
{
    return ( static_cast<uint8_t>( m_step ) & static_cast<uint8_t>( step ) ) != 0;
}

void Task::startParserStep()
{
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "retry_count = retry_count + 1 WHERE id_task = ?";
    sqlite::Tools::executeUpdate( m_ml->getConn(), req, m_id );
}

bool Task::updateFileId()
{
    assert( m_fileId == 0 );
    assert( file != nullptr && file->id() != 0 );
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "file_id = ? WHERE id_task = ?";
    if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, file->id(), m_id ) == false )
        return false;
    m_fileId = file->id();
    return true;
}

int64_t Task::id() const
{
    return m_id;
}

bool Task::restoreLinkedEntities()
{
    LOG_INFO("Restoring linked entities of task ", m_id);
    // MRL will be empty if the task has been resumed from unparsed files
    // parentFolderId == 0 indicates an external file
    if ( mrl.empty() == true && m_parentFolderId == 0 )
    {
        LOG_WARN( "Aborting & removing external file task (#", m_id, ')' );
        destroy( m_ml, m_id );
        return false;
    }
    // First of all, we need to know if the file has been created already
    // ie. have we run the MetadataParser service, at least partially
    file = File::fetch( m_ml, m_fileId );

    // We might re-create tasks without mrl to ease the handling of files on
    // external storage.
    if ( mrl.empty() == true )
    {
        // but we expect those to be created from an existing file after a
        // partial/failed migration. If we don't have a file nor an mrl, we
        // can't really process it.
        if ( file == nullptr )
        {
            assert( !"Can't process a file without a file nor an mrl" );
            return false;
        }
        auto folder = Folder::fetch( m_ml, file->folderId() );
        if ( folder == nullptr )
        {
            assert( !"Can't file the folder associated with a file" );
            // If the folder can't be found in db while the file can, it looks
            // a lot like a sporadic failure, since file are deleted through
            // triggers when its parent folder gets deleted. Just postpone this.
            return false;
        }
        if ( folder->isPresent() == false )
        {
            LOG_WARN( "Postponing rescan of removable file ", file->rawMrl(),
                      " until the device containing it is present again" );
            return false;
        }
        setMrl( file->mrl() );
    }
    auto fsFactory = m_ml->fsFactoryForMrl( mrl );
    if ( fsFactory == nullptr )
        return false;

    try
    {
        parentFolderFs = fsFactory->createDirectory( utils::file::directory( mrl ) );
    }
    catch ( const std::system_error& ex )
    {
        LOG_ERROR( "Failed to restore task: ", ex.what() );
        return false;
    }

    try
    {
        auto files = parentFolderFs->files();
        auto it = std::find_if( begin( files ), end( files ), [this]( std::shared_ptr<fs::IFile> f ) {
            return f->mrl() == mrl;
        });
        if ( it == end( files ) )
        {
            LOG_ERROR( "Failed to restore fs::IFile associated with ", mrl );
            return false;
        }
        fileFs = *it;
    }
    catch ( const std::system_error& ex )
    {
        // If we never found the file yet, we can delete the task. It will be
        // recreated upon next discovery
        if ( file == nullptr )
        {
            LOG_WARN( "Failed to restore file system instances for mrl ", mrl, "."
                      " Removing the task until it gets detected again." );
            destroy( m_ml, m_id );
        }
        else
        {
            // Otherwise we need to postpone it, although most likely we will
            // detect that the file is now missing, and we won't try to restore
            // this task until it comes back (since the task restoration request
            // includes the file.is_present flag)
            LOG_WARN( "Failed to restore file system instances for mrl ", mrl, "."
                      " Postponing the task." );
        }
        return false;
    }

    if ( file != nullptr )
        media = file->media();

    parentFolder = Folder::fetch( m_ml, m_parentFolderId );
    if ( m_parentPlaylistId != 0 )
        parentPlaylist = Playlist::fetch( m_ml, m_parentPlaylistId );

    return true;
}

void Task::setMrl( std::string newMrl )
{
    if ( mrl == newMrl )
        return;
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "mrl = ? WHERE id_task = ?";
    if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, newMrl, m_id ) == false )
        return;
    mrl = std::move( newMrl );
}

void Task::createTable( sqlite::Connection* dbConnection )
{
    std::string req = "CREATE TABLE IF NOT EXISTS " + policy::TaskTable::Name + "("
        "id_task INTEGER PRIMARY KEY AUTOINCREMENT,"
        "step INTEGER NOT NULL DEFAULT 0,"
        "retry_count INTEGER NOT NULL DEFAULT 0,"
        "mrl TEXT,"
        "file_id UNSIGNED INTEGER,"
        "parent_folder_id UNSIGNED INTEGER,"
        "parent_playlist_id INTEGER,"
        "parent_playlist_index UNSIGNED INTEGER,"
        "UNIQUE(mrl, parent_playlist_id) ON CONFLICT FAIL,"
        "FOREIGN KEY (parent_folder_id) REFERENCES " + policy::FolderTable::Name
        + "(id_folder) ON DELETE CASCADE,"
        "FOREIGN KEY (file_id) REFERENCES " + policy::FileTable::Name
        + "(id_file) ON DELETE CASCADE,"
        "FOREIGN KEY (parent_playlist_id) REFERENCES " + policy::PlaylistTable::Name
        + "(id_playlist) ON DELETE CASCADE"
    ")";
    sqlite::Tools::executeRequest( dbConnection, req );
}

void Task::resetRetryCount( MediaLibraryPtr ml )
{
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "retry_count = 0 WHERE step != ?";
    sqlite::Tools::executeUpdate( ml->getConn(), req, parser::Task::ParserStep::Completed );
}

void Task::resetParsing( MediaLibraryPtr ml )
{
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "retry_count = 0, step = ?";
    sqlite::Tools::executeUpdate( ml->getConn(), req, parser::Task::ParserStep::None );
}

std::vector<std::shared_ptr<Task>> Task::fetchUncompleted( MediaLibraryPtr ml )
{
    static const std::string req = "SELECT * FROM " + policy::TaskTable::Name + " t"
        " LEFT JOIN " + policy::FileTable::Name + " f ON f.id_file = t.file_id"
        " WHERE step != ? AND retry_count < 3 AND (f.is_present != 0 OR "
        " t.file_id IS NULL)";
    return Task::fetchAll<Task>( ml, req, parser::Task::ParserStep::Completed );
}

std::shared_ptr<Task>
Task::create( MediaLibraryPtr ml, std::shared_ptr<fs::IFile> fileFs,
              std::shared_ptr<Folder> parentFolder, std::shared_ptr<fs::IDirectory> parentFolderFs,
              std::pair<std::shared_ptr<Playlist>, unsigned int> parentPlaylist )
{
    std::shared_ptr<Task> self = std::make_shared<Task>( ml, std::move( fileFs ),
        std::move( parentFolder ), std::move( parentFolderFs ),
        std::move( parentPlaylist.first ), parentPlaylist.second );
    const std::string req = "INSERT INTO " + policy::TaskTable::Name +
        "(mrl, parent_folder_id, parent_playlist_id, parent_playlist_index) "
        "VALUES(?, ?, ?, ?)";
    if ( insert( ml, self, req, self->mrl, self->parentFolder->id(), sqlite::ForeignKey(
                 self->parentPlaylist ? self->parentPlaylist->id() : 0 ),
                 self->parentPlaylistIndex ) == false )
        return nullptr;
    return self;
}

void Task::recoverUnscannedFiles( MediaLibraryPtr ml )
{
    static const std::string req = "INSERT INTO " + policy::TaskTable::Name +
            "(file_id, parent_folder_id)"
            " SELECT id_file, folder_id FROM " + policy::FileTable::Name +
            " f LEFT JOIN " + policy::TaskTable::Name + " t"
            " ON t.file_id = f.id_file WHERE t.file_id IS NULL"
            " AND f.folder_id IS NOT NULL";
    sqlite::Tools::executeInsert( ml->getConn(), req );
}

}

}
<commit_msg>Task: Don't try to fetch a null file<commit_after>/*****************************************************************************
 * Media Library
 *****************************************************************************
 * Copyright (C) 2015 - 2017 Hugo Beauzée-Luyssen, Videolabs
 *
 * Authors: Hugo Beauzée-Luyssen<hugo@beauzee.fr>
 *          Alexandre Fernandez <nerf@boboop.fr>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

#if HAVE_CONFIG_H
# include "config.h"
#endif

#include "Task.h"

#include "filesystem/IFile.h"
#include "filesystem/IDirectory.h"
#include "File.h"
#include "Folder.h"
#include "Playlist.h"
#include "parser/Task.h"
#include "utils/Filename.h"
#include "utils/Url.h"

namespace medialibrary
{

const std::string policy::TaskTable::Name = "Task";
const std::string policy::TaskTable::PrimaryKeyColumn = "id_task";
int64_t parser::Task::* const policy::TaskTable::PrimaryKey = &parser::Task::m_id;

namespace parser
{

Task::Task( MediaLibraryPtr ml, sqlite::Row& row )
    : currentService( 0 )
    , m_ml( ml )
{
    row >> m_id
        >> m_step
        >> m_retryCount
        >> mrl
        >> m_fileId
        >> m_parentFolderId
        >> m_parentPlaylistId
        >> parentPlaylistIndex;
}

Task::Task( MediaLibraryPtr ml, std::shared_ptr<fs::IFile> fileFs,
            std::shared_ptr<Folder> parentFolder,
            std::shared_ptr<fs::IDirectory> parentFolderFs,
            std::shared_ptr<Playlist> parentPlaylist,
            unsigned int parentPlaylistIndex )
    : fileFs( std::move( fileFs ) )
    , parentFolder( std::move( parentFolder ) )
    , parentFolderFs( std::move( parentFolderFs ) )
    , parentPlaylist( std::move( parentPlaylist ) )
    , parentPlaylistIndex( parentPlaylistIndex )
    , mrl( this->fileFs->mrl() )
    , currentService( 0 )
    , m_ml( ml )
    , m_step( ParserStep::None )
    , m_fileId( 0 )
{
}

void Task::markStepCompleted( ParserStep stepCompleted )
{
    m_step = static_cast<ParserStep>( static_cast<uint8_t>( m_step ) |
                                      static_cast<uint8_t>( stepCompleted ) );
}

void Task::markStepUncompleted( ParserStep stepUncompleted )
{
    m_step = static_cast<ParserStep>( static_cast<uint8_t>( m_step ) &
                                      ( ~ static_cast<uint8_t>( stepUncompleted ) ) );
}

bool Task::saveParserStep()
{
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET step = ?, "
            "retry_count = 0 WHERE id_task = ?";
    if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, m_step, m_id ) == false )
        return false;
    return true;
}

bool Task::isCompleted() const
{
    return m_step == ParserStep::Completed;
}

bool Task::isStepCompleted( Task::ParserStep step ) const
{
    return ( static_cast<uint8_t>( m_step ) & static_cast<uint8_t>( step ) ) != 0;
}

void Task::startParserStep()
{
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "retry_count = retry_count + 1 WHERE id_task = ?";
    sqlite::Tools::executeUpdate( m_ml->getConn(), req, m_id );
}

bool Task::updateFileId()
{
    assert( m_fileId == 0 );
    assert( file != nullptr && file->id() != 0 );
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "file_id = ? WHERE id_task = ?";
    if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, file->id(), m_id ) == false )
        return false;
    m_fileId = file->id();
    return true;
}

int64_t Task::id() const
{
    return m_id;
}

bool Task::restoreLinkedEntities()
{
    LOG_INFO("Restoring linked entities of task ", m_id);
    // MRL will be empty if the task has been resumed from unparsed files
    // parentFolderId == 0 indicates an external file
    if ( mrl.empty() == true && m_parentFolderId == 0 )
    {
        LOG_WARN( "Aborting & removing external file task (#", m_id, ')' );
        destroy( m_ml, m_id );
        return false;
    }
    // First of all, we need to know if the file has been created already
    // ie. have we run the MetadataParser service, at least partially
    if ( m_fileId != 0 )
        file = File::fetch( m_ml, m_fileId );

    // We might re-create tasks without mrl to ease the handling of files on
    // external storage.
    if ( mrl.empty() == true )
    {
        // but we expect those to be created from an existing file after a
        // partial/failed migration. If we don't have a file nor an mrl, we
        // can't really process it.
        if ( file == nullptr )
        {
            assert( !"Can't process a file without a file nor an mrl" );
            return false;
        }
        auto folder = Folder::fetch( m_ml, file->folderId() );
        if ( folder == nullptr )
        {
            assert( !"Can't file the folder associated with a file" );
            // If the folder can't be found in db while the file can, it looks
            // a lot like a sporadic failure, since file are deleted through
            // triggers when its parent folder gets deleted. Just postpone this.
            return false;
        }
        if ( folder->isPresent() == false )
        {
            LOG_WARN( "Postponing rescan of removable file ", file->rawMrl(),
                      " until the device containing it is present again" );
            return false;
        }
        setMrl( file->mrl() );
    }
    auto fsFactory = m_ml->fsFactoryForMrl( mrl );
    if ( fsFactory == nullptr )
        return false;

    try
    {
        parentFolderFs = fsFactory->createDirectory( utils::file::directory( mrl ) );
    }
    catch ( const std::system_error& ex )
    {
        LOG_ERROR( "Failed to restore task: ", ex.what() );
        return false;
    }

    try
    {
        auto files = parentFolderFs->files();
        auto it = std::find_if( begin( files ), end( files ), [this]( std::shared_ptr<fs::IFile> f ) {
            return f->mrl() == mrl;
        });
        if ( it == end( files ) )
        {
            LOG_ERROR( "Failed to restore fs::IFile associated with ", mrl );
            return false;
        }
        fileFs = *it;
    }
    catch ( const std::system_error& ex )
    {
        // If we never found the file yet, we can delete the task. It will be
        // recreated upon next discovery
        if ( file == nullptr )
        {
            LOG_WARN( "Failed to restore file system instances for mrl ", mrl, "."
                      " Removing the task until it gets detected again." );
            destroy( m_ml, m_id );
        }
        else
        {
            // Otherwise we need to postpone it, although most likely we will
            // detect that the file is now missing, and we won't try to restore
            // this task until it comes back (since the task restoration request
            // includes the file.is_present flag)
            LOG_WARN( "Failed to restore file system instances for mrl ", mrl, "."
                      " Postponing the task." );
        }
        return false;
    }

    if ( file != nullptr )
        media = file->media();

    parentFolder = Folder::fetch( m_ml, m_parentFolderId );
    if ( m_parentPlaylistId != 0 )
        parentPlaylist = Playlist::fetch( m_ml, m_parentPlaylistId );

    return true;
}

void Task::setMrl( std::string newMrl )
{
    if ( mrl == newMrl )
        return;
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "mrl = ? WHERE id_task = ?";
    if ( sqlite::Tools::executeUpdate( m_ml->getConn(), req, newMrl, m_id ) == false )
        return;
    mrl = std::move( newMrl );
}

void Task::createTable( sqlite::Connection* dbConnection )
{
    std::string req = "CREATE TABLE IF NOT EXISTS " + policy::TaskTable::Name + "("
        "id_task INTEGER PRIMARY KEY AUTOINCREMENT,"
        "step INTEGER NOT NULL DEFAULT 0,"
        "retry_count INTEGER NOT NULL DEFAULT 0,"
        "mrl TEXT,"
        "file_id UNSIGNED INTEGER,"
        "parent_folder_id UNSIGNED INTEGER,"
        "parent_playlist_id INTEGER,"
        "parent_playlist_index UNSIGNED INTEGER,"
        "UNIQUE(mrl, parent_playlist_id) ON CONFLICT FAIL,"
        "FOREIGN KEY (parent_folder_id) REFERENCES " + policy::FolderTable::Name
        + "(id_folder) ON DELETE CASCADE,"
        "FOREIGN KEY (file_id) REFERENCES " + policy::FileTable::Name
        + "(id_file) ON DELETE CASCADE,"
        "FOREIGN KEY (parent_playlist_id) REFERENCES " + policy::PlaylistTable::Name
        + "(id_playlist) ON DELETE CASCADE"
    ")";
    sqlite::Tools::executeRequest( dbConnection, req );
}

void Task::resetRetryCount( MediaLibraryPtr ml )
{
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "retry_count = 0 WHERE step != ?";
    sqlite::Tools::executeUpdate( ml->getConn(), req, parser::Task::ParserStep::Completed );
}

void Task::resetParsing( MediaLibraryPtr ml )
{
    static const std::string req = "UPDATE " + policy::TaskTable::Name + " SET "
            "retry_count = 0, step = ?";
    sqlite::Tools::executeUpdate( ml->getConn(), req, parser::Task::ParserStep::None );
}

std::vector<std::shared_ptr<Task>> Task::fetchUncompleted( MediaLibraryPtr ml )
{
    static const std::string req = "SELECT * FROM " + policy::TaskTable::Name + " t"
        " LEFT JOIN " + policy::FileTable::Name + " f ON f.id_file = t.file_id"
        " WHERE step != ? AND retry_count < 3 AND (f.is_present != 0 OR "
        " t.file_id IS NULL)";
    return Task::fetchAll<Task>( ml, req, parser::Task::ParserStep::Completed );
}

std::shared_ptr<Task>
Task::create( MediaLibraryPtr ml, std::shared_ptr<fs::IFile> fileFs,
              std::shared_ptr<Folder> parentFolder, std::shared_ptr<fs::IDirectory> parentFolderFs,
              std::pair<std::shared_ptr<Playlist>, unsigned int> parentPlaylist )
{
    std::shared_ptr<Task> self = std::make_shared<Task>( ml, std::move( fileFs ),
        std::move( parentFolder ), std::move( parentFolderFs ),
        std::move( parentPlaylist.first ), parentPlaylist.second );
    const std::string req = "INSERT INTO " + policy::TaskTable::Name +
        "(mrl, parent_folder_id, parent_playlist_id, parent_playlist_index) "
        "VALUES(?, ?, ?, ?)";
    if ( insert( ml, self, req, self->mrl, self->parentFolder->id(), sqlite::ForeignKey(
                 self->parentPlaylist ? self->parentPlaylist->id() : 0 ),
                 self->parentPlaylistIndex ) == false )
        return nullptr;
    return self;
}

void Task::recoverUnscannedFiles( MediaLibraryPtr ml )
{
    static const std::string req = "INSERT INTO " + policy::TaskTable::Name +
            "(file_id, parent_folder_id)"
            " SELECT id_file, folder_id FROM " + policy::FileTable::Name +
            " f LEFT JOIN " + policy::TaskTable::Name + " t"
            " ON t.file_id = f.id_file WHERE t.file_id IS NULL"
            " AND f.folder_id IS NOT NULL";
    sqlite::Tools::executeInsert( ml->getConn(), req );
}

}

}
<|endoftext|>


<commit_before>#include "inputhandler.h"
#include "effects.h"
#include "slog/slog.h"
#include "sdl.h"
#include <utility>

#include "userinput.h"
extern UserInput* user_input;

InputHandler::InputHandler(std::unique_ptr<Image> image)
:   curr_effect_(NULL),
    image_(image.release())
{
    this->RegisterKey(SDLK_q);
    this->RegisterKey(SDLK_ESCAPE);
    this->RegisterKey(SDLK_LEFT);
    this->RegisterKey(SDLK_RIGHT);
}

void InputHandler::OnMessage(SDLKey key, uint8_t type)
{
    SLOG_DEBUG("User input: %d %d", key, type);
    if (type == SDL_KEYUP)
    {
        return;
    }

    if (key == SDLK_ESCAPE)
    {
        user_input->Stop();
        return;
    }

    if (key == SDLK_q)
    {
        this->ResetState();
        SLOG_DEBUG("Leave the effect")
        return;
    }

    if (this->curr_effect_ == NULL)
    {
        auto effect = this->effects_.find(key);
        if (effect != this->effects_.end())
        {
            this->curr_effect_ = effect->second;
            SLOG_DEBUG("Enter the effect")
            return;
        }
    }
    else
    {
        if (this->image_.get() == NULL)
        {
            SLOG_DEBUG("Try to execute effect without image");
            user_input->Stop();
            return;
        }

        SLOG_DEBUG("Apply effect");
        this->curr_effect_(this->image_.get(), key, type);
    }
}

void InputHandler::Bind(SDLKey keypress, Effect effect)
{
    this->effects_.insert(std::make_pair(keypress, effect));
    this->RegisterKey(keypress);
}

void InputHandler::RemoveBind(SDLKey keypress)
{
    auto effect = this->effects_.find(keypress);
    if (effect != this->effects_.end())
    {
        this->effects_.erase(effect);
    }
}
<commit_msg>* Add measurment log<commit_after>#include "inputhandler.h"
#include "effects.h"
#include "slog/slog.h"
#include "sdl.h"
#include <utility>

#include "userinput.h"
extern UserInput* user_input;

InputHandler::InputHandler(std::unique_ptr<Image> image)
:   curr_effect_(NULL),
    image_(image.release())
{
    this->RegisterKey(SDLK_q);
    this->RegisterKey(SDLK_ESCAPE);
    this->RegisterKey(SDLK_LEFT);
    this->RegisterKey(SDLK_RIGHT);
}

void InputHandler::OnMessage(SDLKey key, uint8_t type)
{
    SLOG_DEBUG("User input: %d %d", key, type);
    if (type == SDL_KEYUP)
    {
        return;
    }

    if (key == SDLK_ESCAPE)
    {
        user_input->Stop();
        return;
    }

    if (key == SDLK_q)
    {
        this->ResetState();
        SLOG_DEBUG("Leave the effect")
        return;
    }

    if (this->curr_effect_ == NULL)
    {
        auto effect = this->effects_.find(key);
        if (effect != this->effects_.end())
        {
            this->curr_effect_ = effect->second;
            SLOG_DEBUG("Enter the effect")
            return;
        }
    }
    else
    {
        if (this->image_.get() == NULL)
        {
            SLOG_DEBUG("Try to execute effect without image");
            user_input->Stop();
            return;
        }

        SLOG_DEBUG("Start effect");
        this->curr_effect_(this->image_.get(), key, type);
        SLOG_DEBUG("Finish effect");
    }
}

void InputHandler::Bind(SDLKey keypress, Effect effect)
{
    this->effects_.insert(std::make_pair(keypress, effect));
    this->RegisterKey(keypress);
}

void InputHandler::RemoveBind(SDLKey keypress)
{
    auto effect = this->effects_.find(keypress);
    if (effect != this->effects_.end())
    {
        this->effects_.erase(effect);
    }
}
<|endoftext|>


<commit_before>#include "poddlthread.h"

#include <cinttypes>
#include <curl/curl.h>
#include <iostream>
#include <libgen.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <unistd.h>

#include "config.h"
#include "logger.h"
#include "utils.h"

using namespace newsboat;

namespace podboat {

static size_t my_write_data(void* buffer, size_t size, size_t nmemb,
	void* userp);
static int progress_callback(void* clientp,
	double dltotal,
	double dlnow,
	double ultotal,
	double ulnow);

PodDlThread::PodDlThread(Download* dl_, newsboat::ConfigContainer* c)
	: dl(dl_)
	, f(new std::ofstream())
	, bytecount(0)
	, cfg(c)
{
	static const timeval zero = {0, 0};
	tv1 = zero;
	tv2 = zero;
}

PodDlThread::~PodDlThread() {}

void PodDlThread::operator()()
{
	run();
}

void PodDlThread::run()
{
	// are we resuming previous download?
	bool resumed_download = false;

	gettimeofday(&tv1, nullptr);
	++bytecount;

	CURL* easyhandle = curl_easy_init();
	utils::set_common_curl_options(easyhandle, cfg);

	curl_easy_setopt(easyhandle, CURLOPT_URL, dl->url().c_str());
	curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 0);
	// set up write functions:
	curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, my_write_data);
	curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, this);

	// set up progress notification:
	curl_easy_setopt(easyhandle, CURLOPT_NOPROGRESS, 0);
	curl_easy_setopt(
		easyhandle, CURLOPT_PROGRESSFUNCTION, progress_callback);
	curl_easy_setopt(easyhandle, CURLOPT_PROGRESSDATA, this);

	// set up max download speed
	int max_dl_speed = cfg->get_configvalue_as_int("max-download-speed");
	if (max_dl_speed > 0) {
		curl_easy_setopt(easyhandle,
			CURLOPT_MAX_RECV_SPEED_LARGE,
			(curl_off_t)(max_dl_speed * 1024));
	}

	struct stat sb;
	std::string filename =
		dl->filename() + newsboat::ConfigContainer::PARTIAL_FILE_SUFFIX;

	if (stat(filename.c_str(), &sb) == -1) {
		LOG(Level::INFO,
			"PodDlThread::run: stat failed: starting normal "
			"download");

		// Have to copy the string into a vector in order to be able to
		// get a char* pointer. std::string::c_str() won't do because it
		// returns const char*, whereas ::dirname() needs non-const.
		std::vector<char> directory(filename.begin(), filename.end());
		utils::mkdir_parents(dirname(&directory[0]));

		f->open(filename, std::fstream::out);
		dl->set_offset(0);
		resumed_download = false;
	} else {
		LOG(Level::INFO,
			"PodDlThread::run: stat ok: starting download from %" PRIi64,
			// That field is `long int`, which is at least 32 bits. On x86_64,
			// it's 64 bits. Thus, this cast is either a no-op, or an up-cast
			// which are always safe.
			static_cast<int64_t>(sb.st_size));
		curl_easy_setopt(easyhandle, CURLOPT_RESUME_FROM, sb.st_size);
		dl->set_offset(sb.st_size);
		f->open(filename, std::fstream::out | std::fstream::app);
		resumed_download = true;
	}

	if (f->is_open()) {
		dl->set_status(DlStatus::DOWNLOADING);

		CURLcode success = curl_easy_perform(easyhandle);

		f->close();

		LOG(Level::INFO,
			"PodDlThread::run: curl_easy_perform rc = %u (%s)",
			success,
			curl_easy_strerror(success));

		if (0 == success) {
			LOG(Level::DEBUG,
				"PodDlThread::run: download complete, deleting "
				"temporary suffix");
			rename(filename.c_str(), dl->filename().c_str());
			dl->set_status(DlStatus::READY);
		} else if (dl->status() != DlStatus::CANCELLED) {
			// attempt complete re-download
			if (resumed_download) {
				::unlink(filename.c_str());
				this->run();
			} else {
				dl->set_status(DlStatus::FAILED);
				::unlink(filename.c_str());
			}
		}
	} else {
		dl->set_status(DlStatus::FAILED);
	}

	curl_easy_cleanup(easyhandle);
}

static size_t my_write_data(void* buffer, size_t size, size_t nmemb,
	void* userp)
{
	PodDlThread* thread = static_cast<PodDlThread*>(userp);
	return thread->write_data(buffer, size, nmemb);
}

static int progress_callback(void* clientp,
	double dltotal,
	double dlnow,
	double /* ultotal */,
	double /*ulnow*/)
{
	PodDlThread* thread = static_cast<PodDlThread*>(clientp);
	return thread->progress(dlnow, dltotal);
}

size_t PodDlThread::write_data(void* buffer, size_t size, size_t nmemb)
{
	if (dl->status() == DlStatus::CANCELLED) {
		return 0;
	}
	f->write(static_cast<char*>(buffer), size * nmemb);
	bytecount += (size * nmemb);
	LOG(Level::DEBUG,
		"PodDlThread::write_data: bad = %u size = %" PRIu64,
		f->bad(),
		static_cast<uint64_t>(size * nmemb));
	return f->bad() ? 0 : size * nmemb;
}

int PodDlThread::progress(double dlnow, double dltotal)
{
	if (dl->status() == DlStatus::CANCELLED) {
		return -1;
	}
	gettimeofday(&tv2, nullptr);
	double kbps = compute_kbps();
	if (kbps > 9999.99) {
		kbps = 0.0;
		gettimeofday(&tv1, nullptr);
		bytecount = 0;
	}
	dl->set_kbps(kbps);
	dl->set_progress(dlnow, dltotal);
	return 0;
}

double PodDlThread::compute_kbps()
{
	double t1 = tv1.tv_sec + (tv1.tv_usec / 1000000.0);
	double t2 = tv2.tv_sec + (tv2.tv_usec / 1000000.0);

	double result = (bytecount / (t2 - t1)) / 1024;

	return result;
}

} // namespace podboat
<commit_msg>Add explicit null-termination to poddl directory string<commit_after>#include "poddlthread.h"

#include <cinttypes>
#include <curl/curl.h>
#include <iostream>
#include <libgen.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <unistd.h>

#include "config.h"
#include "logger.h"
#include "utils.h"

using namespace newsboat;

namespace podboat {

static size_t my_write_data(void* buffer, size_t size, size_t nmemb,
	void* userp);
static int progress_callback(void* clientp,
	double dltotal,
	double dlnow,
	double ultotal,
	double ulnow);

PodDlThread::PodDlThread(Download* dl_, newsboat::ConfigContainer* c)
	: dl(dl_)
	, f(new std::ofstream())
	, bytecount(0)
	, cfg(c)
{
	static const timeval zero = {0, 0};
	tv1 = zero;
	tv2 = zero;
}

PodDlThread::~PodDlThread() {}

void PodDlThread::operator()()
{
	run();
}

void PodDlThread::run()
{
	// are we resuming previous download?
	bool resumed_download = false;

	gettimeofday(&tv1, nullptr);
	++bytecount;

	CURL* easyhandle = curl_easy_init();
	utils::set_common_curl_options(easyhandle, cfg);

	curl_easy_setopt(easyhandle, CURLOPT_URL, dl->url().c_str());
	curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 0);
	// set up write functions:
	curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, my_write_data);
	curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, this);

	// set up progress notification:
	curl_easy_setopt(easyhandle, CURLOPT_NOPROGRESS, 0);
	curl_easy_setopt(
		easyhandle, CURLOPT_PROGRESSFUNCTION, progress_callback);
	curl_easy_setopt(easyhandle, CURLOPT_PROGRESSDATA, this);

	// set up max download speed
	int max_dl_speed = cfg->get_configvalue_as_int("max-download-speed");
	if (max_dl_speed > 0) {
		curl_easy_setopt(easyhandle,
			CURLOPT_MAX_RECV_SPEED_LARGE,
			(curl_off_t)(max_dl_speed * 1024));
	}

	struct stat sb;
	std::string filename =
		dl->filename() + newsboat::ConfigContainer::PARTIAL_FILE_SUFFIX;

	if (stat(filename.c_str(), &sb) == -1) {
		LOG(Level::INFO,
			"PodDlThread::run: stat failed: starting normal "
			"download");

		// Have to copy the string into a vector in order to be able to
		// get a char* pointer. std::string::c_str() won't do because it
		// returns const char*, whereas ::dirname() needs non-const.
		std::vector<char> directory(filename.begin(), filename.end());
		directory.push_back('\0');
		utils::mkdir_parents(dirname(&directory[0]));

		f->open(filename, std::fstream::out);
		dl->set_offset(0);
		resumed_download = false;
	} else {
		LOG(Level::INFO,
			"PodDlThread::run: stat ok: starting download from %" PRIi64,
			// That field is `long int`, which is at least 32 bits. On x86_64,
			// it's 64 bits. Thus, this cast is either a no-op, or an up-cast
			// which are always safe.
			static_cast<int64_t>(sb.st_size));
		curl_easy_setopt(easyhandle, CURLOPT_RESUME_FROM, sb.st_size);
		dl->set_offset(sb.st_size);
		f->open(filename, std::fstream::out | std::fstream::app);
		resumed_download = true;
	}

	if (f->is_open()) {
		dl->set_status(DlStatus::DOWNLOADING);

		CURLcode success = curl_easy_perform(easyhandle);

		f->close();

		LOG(Level::INFO,
			"PodDlThread::run: curl_easy_perform rc = %u (%s)",
			success,
			curl_easy_strerror(success));

		if (0 == success) {
			LOG(Level::DEBUG,
				"PodDlThread::run: download complete, deleting "
				"temporary suffix");
			rename(filename.c_str(), dl->filename().c_str());
			dl->set_status(DlStatus::READY);
		} else if (dl->status() != DlStatus::CANCELLED) {
			// attempt complete re-download
			if (resumed_download) {
				::unlink(filename.c_str());
				this->run();
			} else {
				dl->set_status(DlStatus::FAILED);
				::unlink(filename.c_str());
			}
		}
	} else {
		dl->set_status(DlStatus::FAILED);
	}

	curl_easy_cleanup(easyhandle);
}

static size_t my_write_data(void* buffer, size_t size, size_t nmemb,
	void* userp)
{
	PodDlThread* thread = static_cast<PodDlThread*>(userp);
	return thread->write_data(buffer, size, nmemb);
}

static int progress_callback(void* clientp,
	double dltotal,
	double dlnow,
	double /* ultotal */,
	double /*ulnow*/)
{
	PodDlThread* thread = static_cast<PodDlThread*>(clientp);
	return thread->progress(dlnow, dltotal);
}

size_t PodDlThread::write_data(void* buffer, size_t size, size_t nmemb)
{
	if (dl->status() == DlStatus::CANCELLED) {
		return 0;
	}
	f->write(static_cast<char*>(buffer), size * nmemb);
	bytecount += (size * nmemb);
	LOG(Level::DEBUG,
		"PodDlThread::write_data: bad = %u size = %" PRIu64,
		f->bad(),
		static_cast<uint64_t>(size * nmemb));
	return f->bad() ? 0 : size * nmemb;
}

int PodDlThread::progress(double dlnow, double dltotal)
{
	if (dl->status() == DlStatus::CANCELLED) {
		return -1;
	}
	gettimeofday(&tv2, nullptr);
	double kbps = compute_kbps();
	if (kbps > 9999.99) {
		kbps = 0.0;
		gettimeofday(&tv1, nullptr);
		bytecount = 0;
	}
	dl->set_kbps(kbps);
	dl->set_progress(dlnow, dltotal);
	return 0;
}

double PodDlThread::compute_kbps()
{
	double t1 = tv1.tv_sec + (tv1.tv_usec / 1000000.0);
	double t2 = tv2.tv_sec + (tv2.tv_usec / 1000000.0);

	double result = (bytecount / (t2 - t1)) / 1024;

	return result;
}

} // namespace podboat
<|endoftext|>