<commit_before>/*******************************************************************************
 * Copyright (c) 2014, 2016  IBM Corporation and others
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *******************************************************************************/

#ifndef GaussianProcessLight_hpp
#define GaussianProcessLight_hpp

#include <iostream>
#include <limits>
#include <Eigen/Dense>

#include "KernelFunction.hpp"
#include "GaussianProcess.hpp"
#include "SerializeUtils.hpp"

namespace loc{

    class GaussianProcessLight : public GaussianProcess{

    private:
        // variables to be serialized
        std::vector<GaussianProcess> LGPs_;     //Local Gaussian Processes
        std::vector<Eigen::VectorXd> centers_;  //center for each LGP

        double sigmaN_ = 1.0;
        GaussianKernel gaussianKernel_;

    public:
        static const int N_FEATURES = 4;
        constexpr static const double MIN_DENOMINATOR = std::numeric_limits<double>::min() * 1e+16;

        GaussianProcessLight() = default;

        enum ClusteringType{
            GREEDY,
            KMEANS
        };
        ClusteringType clType = KMEANS;
        bool usesOverlap = true;
        int mLocalsMixed_ = 3;

        // A function for serealization
        template<class Archive>
        void serialize(Archive& ar){
            ar(CEREAL_NVP(LGPs_));
            ar(CEREAL_NVP(centers_));
            ar(CEREAL_NVP(sigmaN_));
            ar(CEREAL_NVP(gaussianKernel_));

            std::vector<std::string> names;
            try{
                ar(CEREAL_NVP(mLocalsMixed_));
            }catch(cereal::Exception& e){
                names.push_back("mLocalMixed_");
            }

            if(0<names.size()){
                std::stringstream ss;
                ss << "[notice] parameters not found: ";
                for(const auto& name: names){
                    ss << name << ",";
                }
                std::cerr << ss.str() << std::endl;
            }
        }

        GaussianProcessLight& sigmaN(double sigmaN){
            sigmaN_ = sigmaN;
            return *this;
        }

        double sigmaN() const{
            return sigmaN_;
        }

        //Calculate maximum cluster size based on max complexity of the function predict()
        static const size_t MAX_CLUSTER_SIZE(const size_t MAX_COMPLEXITY,
                                             const size_t MAX_N_OVERLAP,
                                             const size_t N_LOCALS_MIXED) {
            const size_t k = MAX_N_OVERLAP;     //max # cluster overlaps for a sample
            const size_t M = N_LOCALS_MIXED;    //# local models mixed in prediction
            const size_t MAX_AMP = (std::min(k, M) + 1) * std::max(k, M);
            const size_t MAX_CLUSTER_SIZE = MAX_COMPLEXITY / MAX_AMP;

            const size_t MIN_FEASIBLE_CLUSTER_SIZE = 50;
            if (MAX_CLUSTER_SIZE < MIN_FEASIBLE_CLUSTER_SIZE) {
                std::cout << "WARNING: MAX_CLUSTER_SIZE=" << MAX_CLUSTER_SIZE << " is less than MIN_FEASIBLE_CLUSTER_SIZE=" << MIN_FEASIBLE_CLUSTER_SIZE << std::endl;
            }
            return MAX_CLUSTER_SIZE;
        }

        GaussianProcessLight& fit(const Eigen::MatrixXd& X, const Eigen::MatrixXd& Y)
        {
            //Clustering samples
            const size_t TARGET_N_CLUSTER = 1 + (X.rows() / MAX_CLUSTER_SIZE(800, 3, 3));
            CentroidBasedClusteringResult cr;
            if(clType==GREEDY){
                cr = clusteringGreedyWithMagicThreshold(X, Y, 1e-8);
            }else if(clType==KMEANS){
                cr = kMeansClustering(X, Y, TARGET_N_CLUSTER);
            }

            cr.printSummary();

            if(usesOverlap){
            //Improve the clusters
            const double OVERLAP_SCALE = 0.001;
            improveWithOverlap(cr, OVERLAP_SCALE, X, Y);
            cr.printSummary();
            }

            std::cout << "clustered into " << cr.nCluster() << " local models" << std::endl;

            centers_ = cr.centers;

            //Get local models by cluster
            for (auto k=0; k < cr.nCluster(); k++) {
                GaussianProcess gp;
                gp.sigmaN(sigmaN_);
                gp.gaussianKernel(gaussianKernel_);

                gp.fit(cr.XC[k], cr.YC[k]);

                LGPs_.push_back(gp);
            }

            return *this;
        }

        double predict(double x[], int index) const
        {
            std::vector<int> indices(1, index);
            std::vector<double> ypreds = predict(x, indices);
            return ypreds.at(0);
        }

        //TODO change return type: Eigen::VectorXd would be better
        std::vector<double> predict(double x[], const std::vector<int>& indices) const
        {
            const size_t M = mLocalsMixed_;
            const size_t n = centers_.size();

            std::vector<double> weights(n);
            for (size_t i=0; i < n; ++i) {
                const double* c = centers_.at(i).data();
                weights[i] = gaussianKernel_.computeKernel(x, c);
            }

            //indices of k-nearest (=top-k weight) neigbors
            std::vector<size_t> neighbors = top_k(weights, std::min(M, n));

            Eigen::VectorXd sum_wy = Eigen::VectorXd::Zero(indices.size());
            double sum_w = 0.0;
            for (auto m : neighbors) {
                double w = weights.at(m);
                std::vector<double> tmp = LGPs_.at(m).predict(x, indices);
                Eigen::VectorXd y = Eigen::Map<Eigen::VectorXd>(tmp.data(), indices.size());
                sum_wy += w * y;
                sum_w  += w;
            }

            Eigen::VectorXd y_hat = sum_wy;
            if (sum_w > MIN_DENOMINATOR) {
                y_hat /= sum_w;
            } else {
                std::vector<double> top1ypred = LGPs_.at(neighbors.at(0)).predict(x, indices);
                y_hat = Eigen::Map<Eigen::VectorXd>(top1ypred.data(), indices.size());
//                std::cout << "WARN: sum_w~=0 in predict() with " << n << " LGPs"
//                          << " >> predicted only with the nearest local model."<< std::endl;
            }
            std::vector<double> ypreds(y_hat.data(), y_hat.data() + indices.size());
            return ypreds;
        }

        /**
         * Estimate parameters as preparation
         */
        void fitCV(const Eigen::MatrixXd& X, const Eigen::MatrixXd& Y, const Eigen::MatrixXd& Actives)
        {
            GaussianProcess gp;
            gp.sigmaN(sigmaN_);
            gp.gaussianKernel(gaussianKernel_);

            // estimate parameters using GaussianProcess::fitCV
            gp.fitCV(X, Y, Actives);

            // set estimated parameters to this
            sigmaN_ = gp.sigmaN();
            gaussianKernel_ = gp.gaussianKernel();

            this->fit(X, Y);
        }

//        Eigen::VectorXd predictVarianceF(double x[]) const;
//        Eigen::VectorXd predictVarianceF(const Eigen::VectorXd& kstar) const{;
//        double computeLogLikelihood(double x[], const Eigen::VectorXd& y) const;

    private:
        class CentroidBasedClusteringResult{
        public:
            std::vector<Eigen::MatrixXd> XC;
            std::vector<Eigen::MatrixXd> YC;
            std::vector<Eigen::VectorXd> centers = {};
            size_t nCluster() const { return centers.size(); }
            void printSummary() const;
            void printAll() const;
        };

        /**
         * k-means++ clustering
         */
        CentroidBasedClusteringResult kMeansClustering(const Eigen::MatrixXd& X,
                                                       const Eigen::MatrixXd& Y,
                                                       const size_t TARGET_N_CLUSTER) const;

        /**
         * Greedy clustering
         */
        CentroidBasedClusteringResult clusteringGreedyWithMagicThreshold(const Eigen::MatrixXd& X,
                                                                         const Eigen::MatrixXd& Y,
                                                                         const double MAGIC_W_THRESHOLD) const;

        void improveWithOverlap(CentroidBasedClusteringResult& cr,
                                const double OVERLAP_SCALE,
                                const Eigen::MatrixXd& X,
                                const Eigen::MatrixXd& Y) const;

    public:
        // TODO move to an appropriate util class
        static std::vector<size_t> top_k(const std::vector<double>& values, const size_t k)
        {
            assert(k <= values.size());
            size_t n = values.size();
            std::vector<size_t> labels(n);
            for (auto i=0; i < n; ++i) { labels[i] = i; }

            auto comp = [&values](size_t i, size_t j) { return values[i] > values[j]; };

            if (n == k) {
                std::sort(labels.begin(), labels.end(), comp);
                return labels;
            }

            assert(n > k);  //need +1 elements to make heap
            auto first = labels.begin(), last = labels.begin() + k, end = labels.end();
            std::make_heap(first, last + 1, comp);
            std::pop_heap(first, last + 1, comp);
            for (auto it = last + 1; it != end; it++) {
                if (values[*it] <= values[*first]) {
                    continue;
                } else {
                    *last = *it;
                    std::pop_heap(first, last + 1, comp);
                }
            }
            std::sort_heap(first, last, comp);
            return std::vector<size_t>{first, last};
        }

//    private:
//        CentroidBasedClusteringResult aggregativeClustering(const Eigen::MatrixXd& X,
//                                                            const Eigen::MatrixXd& Y,
//                                                            const size_t TARGET_N_CLUSTER) const
//        class CentroidBasedCluster{
//        public:
//            CentroidBasedCluster(size_t i, const Eigen::MatrixXd& X) {
//                labels.push_back(i);
//                center = X.row(i);
//            };
//            std::vector<size_t> labels;
//            Eigen::VectorXd center;
//            void merge(const CentroidBasedCluster& that) {
//                std::copy(that.labels.begin(),that.labels.end(),std::back_inserter(this->labels));
//                std::sort(this->labels.begin(), this->labels.end());
//                double nis = this->labels.size();
//                double nat = that.labels.size();
//                this->center = (nis * this->center + nat * that.center) / (nis + nat);
//            }
//        };

    };
}

#endif /* GaussianProcessLight_hpp */
<commit_msg>fix a variable name<commit_after>/*******************************************************************************
 * Copyright (c) 2014, 2016  IBM Corporation and others
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *******************************************************************************/

#ifndef GaussianProcessLight_hpp
#define GaussianProcessLight_hpp

#include <iostream>
#include <limits>
#include <Eigen/Dense>

#include "KernelFunction.hpp"
#include "GaussianProcess.hpp"
#include "SerializeUtils.hpp"

namespace loc{

    class GaussianProcessLight : public GaussianProcess{

    private:
        // variables to be serialized
        std::vector<GaussianProcess> LGPs_;     //Local Gaussian Processes
        std::vector<Eigen::VectorXd> centers_;  //center for each LGP

        double sigmaN_ = 1.0;
        GaussianKernel gaussianKernel_;

    public:
        static const int N_FEATURES = 4;
        constexpr static const double MIN_DENOMINATOR = std::numeric_limits<double>::min() * 1e+16;

        GaussianProcessLight() = default;

        enum ClusteringType{
            GREEDY,
            KMEANS
        };
        ClusteringType clType = KMEANS;
        bool usesOverlap = true;
        int mLocalsMixed_ = 3;

        // A function for serealization
        template<class Archive>
        void serialize(Archive& ar){
            ar(CEREAL_NVP(LGPs_));
            ar(CEREAL_NVP(centers_));
            ar(CEREAL_NVP(sigmaN_));
            ar(CEREAL_NVP(gaussianKernel_));

            std::vector<std::string> names;
            try{
                ar(CEREAL_NVP(mLocalsMixed_));
            }catch(cereal::Exception& e){
                names.push_back("mLocalsMixed_");
            }

            if(0<names.size()){
                std::stringstream ss;
                ss << "[notice] parameters not found: ";
                for(const auto& name: names){
                    ss << name << ",";
                }
                std::cerr << ss.str() << std::endl;
            }
        }

        GaussianProcessLight& sigmaN(double sigmaN){
            sigmaN_ = sigmaN;
            return *this;
        }

        double sigmaN() const{
            return sigmaN_;
        }

        //Calculate maximum cluster size based on max complexity of the function predict()
        static const size_t MAX_CLUSTER_SIZE(const size_t MAX_COMPLEXITY,
                                             const size_t MAX_N_OVERLAP,
                                             const size_t N_LOCALS_MIXED) {
            const size_t k = MAX_N_OVERLAP;     //max # cluster overlaps for a sample
            const size_t M = N_LOCALS_MIXED;    //# local models mixed in prediction
            const size_t MAX_AMP = (std::min(k, M) + 1) * std::max(k, M);
            const size_t MAX_CLUSTER_SIZE = MAX_COMPLEXITY / MAX_AMP;

            const size_t MIN_FEASIBLE_CLUSTER_SIZE = 50;
            if (MAX_CLUSTER_SIZE < MIN_FEASIBLE_CLUSTER_SIZE) {
                std::cout << "WARNING: MAX_CLUSTER_SIZE=" << MAX_CLUSTER_SIZE << " is less than MIN_FEASIBLE_CLUSTER_SIZE=" << MIN_FEASIBLE_CLUSTER_SIZE << std::endl;
            }
            return MAX_CLUSTER_SIZE;
        }

        GaussianProcessLight& fit(const Eigen::MatrixXd& X, const Eigen::MatrixXd& Y)
        {
            //Clustering samples
            const size_t TARGET_N_CLUSTER = 1 + (X.rows() / MAX_CLUSTER_SIZE(800, 3, 3));
            CentroidBasedClusteringResult cr;
            if(clType==GREEDY){
                cr = clusteringGreedyWithMagicThreshold(X, Y, 1e-8);
            }else if(clType==KMEANS){
                cr = kMeansClustering(X, Y, TARGET_N_CLUSTER);
            }

            cr.printSummary();

            if(usesOverlap){
            //Improve the clusters
            const double OVERLAP_SCALE = 0.001;
            improveWithOverlap(cr, OVERLAP_SCALE, X, Y);
            cr.printSummary();
            }

            std::cout << "clustered into " << cr.nCluster() << " local models" << std::endl;

            centers_ = cr.centers;

            //Get local models by cluster
            for (auto k=0; k < cr.nCluster(); k++) {
                GaussianProcess gp;
                gp.sigmaN(sigmaN_);
                gp.gaussianKernel(gaussianKernel_);

                gp.fit(cr.XC[k], cr.YC[k]);

                LGPs_.push_back(gp);
            }

            return *this;
        }

        double predict(double x[], int index) const
        {
            std::vector<int> indices(1, index);
            std::vector<double> ypreds = predict(x, indices);
            return ypreds.at(0);
        }

        //TODO change return type: Eigen::VectorXd would be better
        std::vector<double> predict(double x[], const std::vector<int>& indices) const
        {
            const size_t M = mLocalsMixed_;
            const size_t n = centers_.size();

            std::vector<double> weights(n);
            for (size_t i=0; i < n; ++i) {
                const double* c = centers_.at(i).data();
                weights[i] = gaussianKernel_.computeKernel(x, c);
            }

            //indices of k-nearest (=top-k weight) neigbors
            std::vector<size_t> neighbors = top_k(weights, std::min(M, n));

            Eigen::VectorXd sum_wy = Eigen::VectorXd::Zero(indices.size());
            double sum_w = 0.0;
            for (auto m : neighbors) {
                double w = weights.at(m);
                std::vector<double> tmp = LGPs_.at(m).predict(x, indices);
                Eigen::VectorXd y = Eigen::Map<Eigen::VectorXd>(tmp.data(), indices.size());
                sum_wy += w * y;
                sum_w  += w;
            }

            Eigen::VectorXd y_hat = sum_wy;
            if (sum_w > MIN_DENOMINATOR) {
                y_hat /= sum_w;
            } else {
                std::vector<double> top1ypred = LGPs_.at(neighbors.at(0)).predict(x, indices);
                y_hat = Eigen::Map<Eigen::VectorXd>(top1ypred.data(), indices.size());
//                std::cout << "WARN: sum_w~=0 in predict() with " << n << " LGPs"
//                          << " >> predicted only with the nearest local model."<< std::endl;
            }
            std::vector<double> ypreds(y_hat.data(), y_hat.data() + indices.size());
            return ypreds;
        }

        /**
         * Estimate parameters as preparation
         */
        void fitCV(const Eigen::MatrixXd& X, const Eigen::MatrixXd& Y, const Eigen::MatrixXd& Actives)
        {
            GaussianProcess gp;
            gp.sigmaN(sigmaN_);
            gp.gaussianKernel(gaussianKernel_);

            // estimate parameters using GaussianProcess::fitCV
            gp.fitCV(X, Y, Actives);

            // set estimated parameters to this
            sigmaN_ = gp.sigmaN();
            gaussianKernel_ = gp.gaussianKernel();

            this->fit(X, Y);
        }

//        Eigen::VectorXd predictVarianceF(double x[]) const;
//        Eigen::VectorXd predictVarianceF(const Eigen::VectorXd& kstar) const{;
//        double computeLogLikelihood(double x[], const Eigen::VectorXd& y) const;

    private:
        class CentroidBasedClusteringResult{
        public:
            std::vector<Eigen::MatrixXd> XC;
            std::vector<Eigen::MatrixXd> YC;
            std::vector<Eigen::VectorXd> centers = {};
            size_t nCluster() const { return centers.size(); }
            void printSummary() const;
            void printAll() const;
        };

        /**
         * k-means++ clustering
         */
        CentroidBasedClusteringResult kMeansClustering(const Eigen::MatrixXd& X,
                                                       const Eigen::MatrixXd& Y,
                                                       const size_t TARGET_N_CLUSTER) const;

        /**
         * Greedy clustering
         */
        CentroidBasedClusteringResult clusteringGreedyWithMagicThreshold(const Eigen::MatrixXd& X,
                                                                         const Eigen::MatrixXd& Y,
                                                                         const double MAGIC_W_THRESHOLD) const;

        void improveWithOverlap(CentroidBasedClusteringResult& cr,
                                const double OVERLAP_SCALE,
                                const Eigen::MatrixXd& X,
                                const Eigen::MatrixXd& Y) const;

    public:
        // TODO move to an appropriate util class
        static std::vector<size_t> top_k(const std::vector<double>& values, const size_t k)
        {
            assert(k <= values.size());
            size_t n = values.size();
            std::vector<size_t> labels(n);
            for (auto i=0; i < n; ++i) { labels[i] = i; }

            auto comp = [&values](size_t i, size_t j) { return values[i] > values[j]; };

            if (n == k) {
                std::sort(labels.begin(), labels.end(), comp);
                return labels;
            }

            assert(n > k);  //need +1 elements to make heap
            auto first = labels.begin(), last = labels.begin() + k, end = labels.end();
            std::make_heap(first, last + 1, comp);
            std::pop_heap(first, last + 1, comp);
            for (auto it = last + 1; it != end; it++) {
                if (values[*it] <= values[*first]) {
                    continue;
                } else {
                    *last = *it;
                    std::pop_heap(first, last + 1, comp);
                }
            }
            std::sort_heap(first, last, comp);
            return std::vector<size_t>{first, last};
        }

//    private:
//        CentroidBasedClusteringResult aggregativeClustering(const Eigen::MatrixXd& X,
//                                                            const Eigen::MatrixXd& Y,
//                                                            const size_t TARGET_N_CLUSTER) const
//        class CentroidBasedCluster{
//        public:
//            CentroidBasedCluster(size_t i, const Eigen::MatrixXd& X) {
//                labels.push_back(i);
//                center = X.row(i);
//            };
//            std::vector<size_t> labels;
//            Eigen::VectorXd center;
//            void merge(const CentroidBasedCluster& that) {
//                std::copy(that.labels.begin(),that.labels.end(),std::back_inserter(this->labels));
//                std::sort(this->labels.begin(), this->labels.end());
//                double nis = this->labels.size();
//                double nat = that.labels.size();
//                this->center = (nis * this->center + nat * that.center) / (nis + nat);
//            }
//        };

    };
}

#endif /* GaussianProcessLight_hpp */
<|endoftext|>


<commit_before>#include <iostream>
#include <map>
#include <mutex>
#include <netinet/in.h>
#include <signal.h>
#include <stdlib.h>
#include <thread>
#include <unistd.h>
#include <string.h>

#include "metacard.h"
#include "utils.h"

using namespace std;

// GLOBAL VARIABLES
int listener_socket;
map<string, uint64_t> balances;
mutex balance_guard;

//Close socket on ^C
void handle_control_c(int s) {
    (void)s; // silence -Wunused-parameter
    close(listener_socket);
    exit(EXIT_SUCCESS);
}

const unsigned char* get_cryptkey(const char* name) {
    if(!strcmp(name, "Alice")) { return Alice_cryptkey; }
    if(!strcmp(name, "Bob")) { return Bob_cryptkey; }
    if(!strcmp(name, "Eve")) { return Eve_cryptkey; }
    return 0;
}

const unsigned char* get_signkey(const char* name) {
    if(!strcmp(name, "Alice")) { return Alice_signkey; }
    if(!strcmp(name, "Bob")) { return Bob_signkey; }
    if(!strcmp(name, "Eve")) { return Eve_signkey; }
    return 0;
}

void handle_connection(int fd) {
    //cout << "handle_connection(" << fd << ")" << endl;
    cts_payload in_payload;
    client_to_server incoming;
    const unsigned char *cryptkey, *signkey;
    do {
        if(read_synchronized(fd, (char*)&incoming, sizeof incoming)) { break; }
        // TODO: handle errors with replies
        if(!(cryptkey = get_cryptkey(incoming.src.username))) { break; }
        if(!(signkey = get_signkey(incoming.src.username))) { break; }
        if(deserialcrypt_cts(cryptkey, signkey, &incoming, &in_payload)) { break; }
        cout << endl;
        hexdump(1, &in_payload, sizeof in_payload);
        cout << endl;
    } while(in_payload.tag != requestLogout);
    close(fd);
}

int bindloop(unsigned short listen_port) {
    //Create listener socket
    listener_socket = socket( AF_INET, SOCK_STREAM, 0 );
    if ( listener_socket < 0 ) {
        cerr << "Socket creation failed" << endl;
        return EXIT_FAILURE;
    }
    signal(SIGINT, &handle_control_c);

    //Bind socket
    struct sockaddr_in server;
    server.sin_family = AF_INET;
    server.sin_addr.s_addr = INADDR_ANY;
    server.sin_port = htons(listen_port);
    int len = sizeof(server);
    if ( bind( listener_socket, (struct sockaddr *) &server, len ) < 0 ) {
        cerr << "Bind failed" << endl;
        return EXIT_FAILURE;
    }
    //Start listening
    listen(listener_socket, MAX_CLIENTS);

    struct sockaddr_in client;
    int fromlen = sizeof(client);

    while(true) {
        int atm_sd = accept(listener_socket, (struct sockaddr *) &client, (socklen_t*) &fromlen);
        if(atm_sd < 0) {
            cerr << "Failed to accept connection" << endl;
            continue;
        }

        thread client_thread(handle_connection, atm_sd);
        client_thread.detach();
    }
}

void print_bankshell_menu() {
    cout << endl;
    cout << "Please enter your desired command" << endl;
    cout << "deposit [username] [amount]" << endl;
    cout << "balance [username]" << endl;
    cout << endl;
}

void handle_deposit(vector<string> tokens) {
    if(tokens.size() != 3) {
        cout << "Expected 2 parameters to 'deposit'." << endl;
        return;
    }
    string username = tokens[1];
    uint64_t amount = strtol(tokens[2].c_str(), NULL, 10);
    lock_guard<mutex> lock(balance_guard);
    balances[username] += amount; // No overflow checking since shell is trusted
}

void handle_balance(vector<string> tokens) {
    if(tokens.size() != 2) {
        cout << "Expected 1 parameter to 'balance'." << endl;
        return;
    }
    string username = tokens[1];
    lock_guard<mutex> lock(balance_guard);
    cout << "Balance for user '" << username << "': " << balances[username] << " cents" << endl;
}

void bankshell() {
    vector<string> tokens;
    while(true){
        print_bankshell_menu();
        prompt:
        print_prompt();
        tokens = get_tokenized_line();
        if(tokens.size() == 0){
            goto prompt;
        }
        if(tokens[0] == "deposit") {
            handle_deposit(tokens);
        }
        else if(tokens[0] == "balance") {
            handle_balance(tokens);
        }
        else if(tokens[0] == "easteregg") {
            cout << "Merry nondenominational seasonal holiday!" << endl;
        }
        else{
            cout << "Unknown command '" << tokens[0] << "'" << endl;
        }
    }
}

int main(int argc, char** argv) {
    // Display a usage message
    if(argc != 2) {
        cout << "Usage: " << argv[0] << " LISTEN_PORT" << endl;
        return EXIT_FAILURE;
    }

    // Get ports
    unsigned short listen_port = (unsigned short) atoi(argv[1]);

    // Assign default balances
    // (doesn't need a mutex since this is before threads are started)
    balances["Alice"] = 100 * CENTS_PER_DOLLAR;
    balances["Bob"] = 50 * CENTS_PER_DOLLAR;
    balances["Eve"] = 0 * CENTS_PER_DOLLAR;

    thread shell = thread(bankshell);
    shell.detach();

    return bindloop(listen_port);
}
<commit_msg>Added plumbing for replies in bank.<commit_after>#include <iostream>
#include <map>
#include <mutex>
#include <netinet/in.h>
#include <signal.h>
#include <stdlib.h>
#include <thread>
#include <unistd.h>
#include <string.h>

#include "metacard.h"
#include "utils.h"

using namespace std;

// GLOBAL VARIABLES
int listener_socket;
map<string, uint64_t> balances;
mutex balance_guard;

//Close socket on ^C
void handle_control_c(int s) {
    (void)s; // silence -Wunused-parameter
    close(listener_socket);
    exit(EXIT_SUCCESS);
}

const unsigned char* get_cryptkey(const char* name) {
    if(!strcmp(name, "Alice")) { return Alice_cryptkey; }
    if(!strcmp(name, "Bob")) { return Bob_cryptkey; }
    if(!strcmp(name, "Eve")) { return Eve_cryptkey; }
    return 0;
}

const unsigned char* get_signkey(const char* name) {
    if(!strcmp(name, "Alice")) { return Alice_signkey; }
    if(!strcmp(name, "Bob")) { return Bob_signkey; }
    if(!strcmp(name, "Eve")) { return Eve_signkey; }
    return 0;
}

// handle_nonce populates the nonce, all the others zero the nonce after checking it
void handle_nonce(nonce_t* nonce, stc_payload* dst) {
}
void handle_balance(nonce_t* nonce, const char* username, const cts_payload* src, stc_payload* dst) {
}
void handle_withdrawl(nonce_t* nonce, const char* username, const cts_payload* src, stc_payload* dst) {
}
void handle_transfer(nonce_t* nonce, const char* username, const cts_payload* src, stc_payload* dst) {
}

void handle_connection(int fd) {
    //cout << "handle_connection(" << fd << ")" << endl;
    cts_payload in_payload;
    client_to_server incoming;
    stc_payload out_payload;
    server_to_client outgoing;
    nonce_t nonce;
    const unsigned char *cryptkey, *signkey;
    do {
        if(read_synchronized(fd, (char*)&incoming, sizeof incoming)) { break; }
        // TODO: handle errors with replies
        if(!(cryptkey = get_cryptkey(incoming.src.username))) { break; }
        if(!(signkey = get_signkey(incoming.src.username))) { break; }
        if(deserialcrypt_cts(cryptkey, signkey, &incoming, &in_payload)) { break; }
        //cout << endl; hexdump(1, &in_payload, sizeof in_payload); cout << endl;
        switch(in_payload.tag) {
            case requestNonce: handle_nonce(&nonce, &out_payload); break;
            case requestBalance: handle_balance(incoming.src.username, &in_payload, &out_payload); break;
            case requestWithdrawl: handle_withdrawl(incoming.src.username, &in_payload, &out_payload); break;
            case requestTransfer: handle_transfer(incoming.src.username, &in_payload, &out_payload); break;
            case requestLogout: goto skip_reply; break;
        }
        reply:
        if(enserialcrypt_stc(cryptkey, signkey, &out_payload, &outgoing)) { break; }
        if(write_synchronized(fd, (char*)&outgoing, sizeof outgoing)) { break; }
        skip_reply:
    } while(in_payload.tag != requestLogout);
    close(fd);
}

int bindloop(unsigned short listen_port) {
    //Create listener socket
    listener_socket = socket( AF_INET, SOCK_STREAM, 0 );
    if ( listener_socket < 0 ) {
        cerr << "Socket creation failed" << endl;
        return EXIT_FAILURE;
    }
    signal(SIGINT, &handle_control_c);

    //Bind socket
    struct sockaddr_in server;
    server.sin_family = AF_INET;
    server.sin_addr.s_addr = INADDR_ANY;
    server.sin_port = htons(listen_port);
    int len = sizeof(server);
    if ( bind( listener_socket, (struct sockaddr *) &server, len ) < 0 ) {
        cerr << "Bind failed" << endl;
        return EXIT_FAILURE;
    }
    //Start listening
    listen(listener_socket, MAX_CLIENTS);

    struct sockaddr_in client;
    int fromlen = sizeof(client);

    while(true) {
        int atm_sd = accept(listener_socket, (struct sockaddr *) &client, (socklen_t*) &fromlen);
        if(atm_sd < 0) {
            cerr << "Failed to accept connection" << endl;
            continue;
        }

        thread client_thread(handle_connection, atm_sd);
        client_thread.detach();
    }
}

void print_bankshell_menu() {
    cout << endl;
    cout << "Please enter your desired command" << endl;
    cout << "deposit [username] [amount]" << endl;
    cout << "balance [username]" << endl;
    cout << endl;
}

void handle_deposit(vector<string> tokens) {
    if(tokens.size() != 3) {
        cout << "Expected 2 parameters to 'deposit'." << endl;
        return;
    }
    string username = tokens[1];
    uint64_t amount = strtol(tokens[2].c_str(), NULL, 10);
    lock_guard<mutex> lock(balance_guard);
    balances[username] += amount; // No overflow checking since shell is trusted
}

void handle_balance(vector<string> tokens) {
    if(tokens.size() != 2) {
        cout << "Expected 1 parameter to 'balance'." << endl;
        return;
    }
    string username = tokens[1];
    lock_guard<mutex> lock(balance_guard);
    cout << "Balance for user '" << username << "': " << balances[username] << " cents" << endl;
}

void bankshell() {
    vector<string> tokens;
    while(true){
        print_bankshell_menu();
        prompt:
        print_prompt();
        tokens = get_tokenized_line();
        if(tokens.size() == 0){
            goto prompt;
        }
        if(tokens[0] == "deposit") {
            handle_deposit(tokens);
        }
        else if(tokens[0] == "balance") {
            handle_balance(tokens);
        }
        else if(tokens[0] == "easteregg") {
            cout << "Merry nondenominational seasonal holiday!" << endl;
        }
        else{
            cout << "Unknown command '" << tokens[0] << "'" << endl;
        }
    }
}

int main(int argc, char** argv) {
    // Display a usage message
    if(argc != 2) {
        cout << "Usage: " << argv[0] << " LISTEN_PORT" << endl;
        return EXIT_FAILURE;
    }

    // Get ports
    unsigned short listen_port = (unsigned short) atoi(argv[1]);

    // Assign default balances
    // (doesn't need a mutex since this is before threads are started)
    balances["Alice"] = 100 * CENTS_PER_DOLLAR;
    balances["Bob"] = 50 * CENTS_PER_DOLLAR;
    balances["Eve"] = 0 * CENTS_PER_DOLLAR;

    thread shell = thread(bankshell);
    shell.detach();

    return bindloop(listen_port);
}
<|endoftext|>


<commit_before>#include "mega.h"
#include "gtest/gtest.h"
#include "test.h"
#include <stdio.h>
#include <fstream>

// If running in Jenkins, we use its working folder.  But for local manual testing, use a convenient location
#ifdef WIN32
    #define LOCAL_TEST_FOLDER "c:\\tmp\\synctests"
#else
    #define LOCAL_TEST_FOLDER (string(getenv("HOME"))+"/synctests_mega_auto")
#endif

using namespace ::mega;

bool gRunningInCI = false;
bool gResumeSessions = false;
bool gTestingInvalidArgs = false;
bool gOutputToCout = false;
int gFseventsFd = -1;
std::string USER_AGENT = "Integration Tests with GoogleTest framework";

std::string getCurrentTimestamp()
{
    using std::chrono::system_clock;
    auto currentTime = std::chrono::system_clock::now();
    char buffer[80];

    auto transformed = currentTime.time_since_epoch().count() / 1000000;

    auto millis = transformed % 1000;

    std::time_t tt;
    tt = system_clock::to_time_t ( currentTime );
    auto timeinfo = localtime (&tt);
    strftime (buffer,80,"%H:%M:%S",timeinfo);
    sprintf(buffer, "%s:%03d",buffer,(int)millis);

    return std::string(buffer);
}

std::string logTime()
{
    return getCurrentTimestamp();
}

class LoggerBroadcastTarget
  : public BroadcastTarget
{
public:
    LoggerBroadcastTarget() = default;

    ~LoggerBroadcastTarget() = default;

    void write(const string& data) override
    {
        LOG_debug << data;
    };
}; // LoggerBroadcastTarget

class StreamBroadcastTarget
  : public BroadcastTarget
{
public:
    explicit
    StreamBroadcastTarget(std::ostream& stream)
      : mOutputStream(stream)
    {
    }

    ~StreamBroadcastTarget() = default;

    void write(const string& data) override
    {
        mOutputStream << getCurrentTimestamp()
                      << " "
                      << data
                      << std::endl;
    }

private:
    std::ostream& mOutputStream;
}; // StreamBroadcastTarget

BroadcastTargetVector gBroadcastTargets;

BroadcastStream out()
{
    auto stream = BroadcastStream(gBroadcastTargets);

    //stream.logTime(withTime);

    return stream;
}

namespace {

class MegaLogger : public Logger
{
public:
    void log(const char* time, int loglevel, const char* source, const char* message
#ifdef ENABLE_LOG_PERFORMANCE
          , const char **directMessages = nullptr, size_t *directMessagesSizes = nullptr, unsigned numberMessages = 0
#endif
    ) override
    {
        std::ostringstream os;

        os << "[";
        os << getCurrentTimestamp();
#ifdef ENABLE_LOG_PERFORMANCE
        os << "] " << SimpleLogger::toStr(static_cast<LogLevel>(loglevel)) << ": ";
        if (message)
        {
            os << message;
        }
        // we can have the message AND the direct messages
        for (unsigned i = 0; i < numberMessages; ++i) os.write(directMessages[i], directMessagesSizes[i]);
#else
        os << "] " << SimpleLogger::toStr(static_cast<LogLevel>(loglevel)) << ": " << message;
#endif
        if (source)
        {
            os << " (" << source << ")";
        }
        os << std::endl;

        if (loglevel <= SimpleLogger::logCurrentLevel)
        {
            if (gRunningInCI)
            {
                if (!mLogFile.is_open())
                {
                    mLogFile.open("test_integration.log");
                }
                mLogFile << os.str() << std::flush;
            }
            else
            {
#ifdef _WIN32
                if (IsDebuggerPresent())
                {
                    OutputDebugStringA(os.str().c_str());
                }
                else
#endif // _WIN32
                {
                    std::cout << os.str() << std::flush;
                }

                if (!gTestingInvalidArgs)
                {
                    if (loglevel <= logError)
                    {
                        ASSERT_GT(loglevel, logError) << os.str();
                    }
                }
            }
        }
    }

private:
    std::ofstream mLogFile;
};

} // anonymous

int main (int argc, char *argv[])
{
    if (!getenv("MEGA_EMAIL") || !getenv("MEGA_PWD") || !getenv("MEGA_EMAIL_AUX") || !getenv("MEGA_PWD_AUX"))
    {
        std::cout << "please set username and password env variables for test" << std::endl;
        return 1;
    }

    std::vector<char*> myargv1(argv, argv + argc);
    std::vector<char*> myargv2;

    for (auto it = myargv1.begin(); it != myargv1.end(); ++it)
    {
        if (std::string(*it) == "--CI")
        {
            gRunningInCI = true;
            argc -= 1;
        }
        else if (std::string(*it).substr(0, 12) == "--USERAGENT:")
        {
            USER_AGENT = std::string(*it).substr(12);
            argc -= 1;
        }
        else if (std::string(*it) == "--COUT")
        {
            gOutputToCout = true;
            argc -= 1;
        }
        else if (std::string(*it).substr(0, 9) == "--APIURL:")
        {
            std::lock_guard<std::mutex> g(g_APIURL_default_mutex);
            g_APIURL_default = std::string(*it).substr(9);
            argc -= 1;
        }
        else if (std::string(*it) == "--RESUMESESSIONS")
        {
            gResumeSessions = true;
            argc -= 1;
        }
#ifdef __APPLE__
        else if (std::string(*it).substr(0, 13) == "--FSEVENTSFD:")
        {
            int fseventsFd = std::stoi(std::string(*it).substr(13));
            if (fcntl(fseventsFd, F_GETFD) == -1 || errno == EBADF) {
                std::cout << "Received bad fsevents fd " << fseventsFd << "\n";
                return 1;
            }

            gFseventsFd = fseventsFd;
            argc -= 1;
        }
#endif
        else
        {
            myargv2.push_back(*it);
        }
    }

    if (gOutputToCout)
    {
        gBroadcastTargets.emplace_back(new StreamBroadcastTarget(std::cout));
    }
    else
    {
        gBroadcastTargets.emplace_back(new LoggerBroadcastTarget());
    }

    MegaLogger megaLogger;

    SimpleLogger::setLogLevel(logMax);
    SimpleLogger::setOutputClass(&megaLogger);

    // delete old test folders, created during previous runs
    TestFS testFS;
    testFS.DeleteTestFolder();
    testFS.DeleteTrashFolder();

#if defined(_WIN32) && defined(NO_READLINE)
    using namespace mega;
    WinConsole* wc = new CONSOLE_CLASS;
    wc->setShellConsole();
#endif

    ::testing::InitGoogleTest(&argc, myargv2.data());
    return RUN_ALL_TESTS();
}

/*
**  TestFS implementation
*/

using namespace std;

fs::path TestFS::GetTestBaseFolder()
{
    const char* jenkins_folder = getenv("WORKSPACE");
    return jenkins_folder ? fs::path(jenkins_folder) : fs::path(LOCAL_TEST_FOLDER);
}

fs::path TestFS::GetTestFolder()
{
#ifdef WIN32
    auto pid = GetCurrentProcessId();
#else
    auto pid = getpid();
#endif

    fs::path testpath = GetTestBaseFolder() / ("pid_" + std::to_string(pid));
    out() << "Local Test folder: " << testpath;
    return testpath;
}

fs::path TestFS::GetTrashFolder()
{
    return GetTestBaseFolder() / "trash";
}

void TestFS::DeleteFolder(fs::path folder)
{
    // rename folder, so that tests can still create one and add to it
    error_code ec;
    fs::path oldpath(folder);
    fs::path newpath(folder);

    for (int i = 10; i--; )
    {
        newpath += "_del"; // this can be improved later if needed
        fs::rename(oldpath, newpath, ec);
        if (!ec) break;
    }

    // if renaming failed, then there's nothing to delete
    if (ec)
    {
        // report failures, other than the case when it didn't exist
        if (ec != errc::no_such_file_or_directory)
        {
            out() << "Renaming " << oldpath << " to " << newpath << " failed."
                 << ec.message();
        }

        return;
    }

    // delete folder in a separate thread
    m_cleaners.emplace_back(thread([=]() mutable // ...mostly for fun, to avoid declaring another ec
        {
            fs::remove_all(newpath, ec);

            if (ec)
            {
                out() << "Deleting " << folder << " failed."
                     << ec.message();
            }
        }));
}

TestFS::~TestFS()
{
    for_each(m_cleaners.begin(), m_cleaners.end(), [](thread& t) { t.join(); });
}

void moveToTrash(const fs::path& p)
{
    fs::path trashpath(TestFS::GetTrashFolder());
    fs::create_directory(trashpath);
    fs::path newpath = trashpath / p.filename();
    for (int i = 2; fs::exists(newpath); ++i)
    {
        newpath = trashpath / fs::u8path(p.filename().stem().u8string() + "_" + to_string(i) + p.extension().u8string());
    }
    fs::rename(p, newpath);
}

fs::path makeNewTestRoot()
{
    fs::path p = TestFS::GetTestFolder();

    if (fs::exists(p))
    {
        moveToTrash(p);
    }
    #ifndef NDEBUG
    bool b =
    #endif
    fs::create_directories(p);
    assert(b);
    return p;
}

<commit_msg>Always output debugging information to COUT.<commit_after>#include "mega.h"
#include "gtest/gtest.h"
#include "test.h"
#include <stdio.h>
#include <fstream>

// If running in Jenkins, we use its working folder.  But for local manual testing, use a convenient location
#ifdef WIN32
    #define LOCAL_TEST_FOLDER "c:\\tmp\\synctests"
#else
    #define LOCAL_TEST_FOLDER (string(getenv("HOME"))+"/synctests_mega_auto")
#endif

using namespace ::mega;

bool gRunningInCI = false;
bool gResumeSessions = false;
bool gTestingInvalidArgs = false;
bool gOutputToCout = false;
int gFseventsFd = -1;
std::string USER_AGENT = "Integration Tests with GoogleTest framework";

std::string getCurrentTimestamp()
{
    using std::chrono::system_clock;
    auto currentTime = std::chrono::system_clock::now();
    char buffer[80];

    auto transformed = currentTime.time_since_epoch().count() / 1000000;

    auto millis = transformed % 1000;

    std::time_t tt;
    tt = system_clock::to_time_t ( currentTime );
    auto timeinfo = localtime (&tt);
    strftime (buffer,80,"%H:%M:%S",timeinfo);
    sprintf(buffer, "%s:%03d",buffer,(int)millis);

    return std::string(buffer);
}

std::string logTime()
{
    return getCurrentTimestamp();
}

class LoggerBroadcastTarget
  : public BroadcastTarget
{
public:
    LoggerBroadcastTarget() = default;

    ~LoggerBroadcastTarget() = default;

    void write(const string& data) override
    {
        LOG_debug << data;
    };
}; // LoggerBroadcastTarget

class StreamBroadcastTarget
  : public BroadcastTarget
{
public:
    explicit
    StreamBroadcastTarget(std::ostream& stream)
      : mOutputStream(stream)
    {
    }

    ~StreamBroadcastTarget() = default;

    void write(const string& data) override
    {
        mOutputStream << getCurrentTimestamp()
                      << " "
                      << data
                      << std::endl;
    }

private:
    std::ostream& mOutputStream;
}; // StreamBroadcastTarget

BroadcastTargetVector gBroadcastTargets;

BroadcastStream out()
{
    auto stream = BroadcastStream(gBroadcastTargets);

    //stream.logTime(withTime);

    return stream;
}

namespace {

class MegaLogger : public Logger
{
public:
    void log(const char* time, int loglevel, const char* source, const char* message
#ifdef ENABLE_LOG_PERFORMANCE
          , const char **directMessages = nullptr, size_t *directMessagesSizes = nullptr, unsigned numberMessages = 0
#endif
    ) override
    {
        std::ostringstream os;

        os << "[";
        os << getCurrentTimestamp();
#ifdef ENABLE_LOG_PERFORMANCE
        os << "] " << SimpleLogger::toStr(static_cast<LogLevel>(loglevel)) << ": ";
        if (message)
        {
            os << message;
        }
        // we can have the message AND the direct messages
        for (unsigned i = 0; i < numberMessages; ++i) os.write(directMessages[i], directMessagesSizes[i]);
#else
        os << "] " << SimpleLogger::toStr(static_cast<LogLevel>(loglevel)) << ": " << message;
#endif
        if (source)
        {
            os << " (" << source << ")";
        }
        os << std::endl;

        if (loglevel <= SimpleLogger::logCurrentLevel)
        {
            if (gRunningInCI)
            {
                if (!mLogFile.is_open())
                {
                    mLogFile.open("test_integration.log");
                }
                mLogFile << os.str() << std::flush;
            }
            else
            {
#ifdef _WIN32
                if (IsDebuggerPresent())
                {
                    OutputDebugStringA(os.str().c_str());
                }
                else
#endif // _WIN32
                {
                    std::cout << os.str() << std::flush;
                }

                if (!gTestingInvalidArgs)
                {
                    if (loglevel <= logError)
                    {
                        ASSERT_GT(loglevel, logError) << os.str();
                    }
                }
            }
        }
    }

private:
    std::ofstream mLogFile;
};

} // anonymous

int main (int argc, char *argv[])
{
    if (!getenv("MEGA_EMAIL") || !getenv("MEGA_PWD") || !getenv("MEGA_EMAIL_AUX") || !getenv("MEGA_PWD_AUX"))
    {
        std::cout << "please set username and password env variables for test" << std::endl;
        return 1;
    }

    std::vector<char*> myargv1(argv, argv + argc);
    std::vector<char*> myargv2;

    for (auto it = myargv1.begin(); it != myargv1.end(); ++it)
    {
        if (std::string(*it) == "--CI")
        {
            gRunningInCI = true;
            argc -= 1;
        }
        else if (std::string(*it).substr(0, 12) == "--USERAGENT:")
        {
            USER_AGENT = std::string(*it).substr(12);
            argc -= 1;
        }
        else if (std::string(*it) == "--COUT")
        {
            gOutputToCout = true;
            argc -= 1;
        }
        else if (std::string(*it).substr(0, 9) == "--APIURL:")
        {
            std::lock_guard<std::mutex> g(g_APIURL_default_mutex);
            g_APIURL_default = std::string(*it).substr(9);
            argc -= 1;
        }
        else if (std::string(*it) == "--RESUMESESSIONS")
        {
            gResumeSessions = true;
            argc -= 1;
        }
#ifdef __APPLE__
        else if (std::string(*it).substr(0, 13) == "--FSEVENTSFD:")
        {
            int fseventsFd = std::stoi(std::string(*it).substr(13));
            if (fcntl(fseventsFd, F_GETFD) == -1 || errno == EBADF) {
                std::cout << "Received bad fsevents fd " << fseventsFd << "\n";
                return 1;
            }

            gFseventsFd = fseventsFd;
            argc -= 1;
        }
#endif
        else
        {
            myargv2.push_back(*it);
        }
    }

    gBroadcastTargets.emplace_back(new StreamBroadcastTarget(std::cout));

    if (gRunningInCI)
    {
        gBroadcastTargets.emplace_back(new LoggerBroadcastTarget());
    }

    MegaLogger megaLogger;

    SimpleLogger::setLogLevel(logMax);
    SimpleLogger::setOutputClass(&megaLogger);

    // delete old test folders, created during previous runs
    TestFS testFS;
    testFS.DeleteTestFolder();
    testFS.DeleteTrashFolder();

#if defined(_WIN32) && defined(NO_READLINE)
    using namespace mega;
    WinConsole* wc = new CONSOLE_CLASS;
    wc->setShellConsole();
#endif

    ::testing::InitGoogleTest(&argc, myargv2.data());
    return RUN_ALL_TESTS();
}

/*
**  TestFS implementation
*/

using namespace std;

fs::path TestFS::GetTestBaseFolder()
{
    const char* jenkins_folder = getenv("WORKSPACE");
    return jenkins_folder ? fs::path(jenkins_folder) : fs::path(LOCAL_TEST_FOLDER);
}

fs::path TestFS::GetTestFolder()
{
#ifdef WIN32
    auto pid = GetCurrentProcessId();
#else
    auto pid = getpid();
#endif

    fs::path testpath = GetTestBaseFolder() / ("pid_" + std::to_string(pid));
    out() << "Local Test folder: " << testpath;
    return testpath;
}

fs::path TestFS::GetTrashFolder()
{
    return GetTestBaseFolder() / "trash";
}

void TestFS::DeleteFolder(fs::path folder)
{
    // rename folder, so that tests can still create one and add to it
    error_code ec;
    fs::path oldpath(folder);
    fs::path newpath(folder);

    for (int i = 10; i--; )
    {
        newpath += "_del"; // this can be improved later if needed
        fs::rename(oldpath, newpath, ec);
        if (!ec) break;
    }

    // if renaming failed, then there's nothing to delete
    if (ec)
    {
        // report failures, other than the case when it didn't exist
        if (ec != errc::no_such_file_or_directory)
        {
            out() << "Renaming " << oldpath << " to " << newpath << " failed."
                 << ec.message();
        }

        return;
    }

    // delete folder in a separate thread
    m_cleaners.emplace_back(thread([=]() mutable // ...mostly for fun, to avoid declaring another ec
        {
            fs::remove_all(newpath, ec);

            if (ec)
            {
                out() << "Deleting " << folder << " failed."
                     << ec.message();
            }
        }));
}

TestFS::~TestFS()
{
    for_each(m_cleaners.begin(), m_cleaners.end(), [](thread& t) { t.join(); });
}

void moveToTrash(const fs::path& p)
{
    fs::path trashpath(TestFS::GetTrashFolder());
    fs::create_directory(trashpath);
    fs::path newpath = trashpath / p.filename();
    for (int i = 2; fs::exists(newpath); ++i)
    {
        newpath = trashpath / fs::u8path(p.filename().stem().u8string() + "_" + to_string(i) + p.extension().u8string());
    }
    fs::rename(p, newpath);
}

fs::path makeNewTestRoot()
{
    fs::path p = TestFS::GetTestFolder();

    if (fs::exists(p))
    {
        moveToTrash(p);
    }
    #ifndef NDEBUG
    bool b =
    #endif
    fs::create_directories(p);
    assert(b);
    return p;
}

<|endoftext|>


<commit_before>/*
 *  This file is part of PlantLight application.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

/*
 * Built for Attiny85 1Mhz, using AVR USBasp programmer.
 * VERSION 0.75
 */

#include <avr/sleep.h>
#include <avr/interrupt.h>

#include <Arduino.h>
#include <TinyWireM.h>
#include <Time.h>
#include <BH1750FVI.h>
#include <DS1307RTC.h>

#define RELAY_SW_OUT            4       // Relay out pin
#define CMD_MAN_IN              1       // Manual light switch
#define WD_TICK_TIME            8       // Watchdog tick time [s] (check WD_MODE or datasheet)
#define WD_WAIT_TIME            16      // Time [s] to count between two sensor read
#define SAMPLE_COUNT            10      // Light sample count (average measurement)
#define LUX_TH                  10      // Lux threshold
#define LUX_TH_HIST             5       // Lux threshold (hysteresis compensation)

// ####################### Prototypes #######################

// Setup watchdog wakeup interrupt
// tick: watchdog tick time [s] (check WD_MODE or datasheet)
void setupWatchdog(float tick);

// Set system into the sleep state,
// system wakes up when watchdog is timed out
void systemSleep();

// Provides a time_t to Time library, wrapper for call
// to RTC.get(), which is not a static function (class member)
time_t timeProvider();

// Display a date in readable format on the serial interface
void printDigits(int digits);
void SerialDisplayDateTime(const time_t &timeToDisplay);

// Sets the RTC to the new time
void setTime();

// Checks the current time of day and month of the year.
// Returns true if the light is now enabled.
boolean checkEnable(const time_t &now);

// Checks actual lux value. Returns true if the conditions
// to turn lights on are met.
boolean checkLightCond(float lux);

// Sample some lux value and calculate the running average over time
float sample();

void printLuxArray();
void cleanLuxArray();

// ####################### Constants ########################

// Watchdog interrupt sleep time
// 0=16ms, 1=32ms, 2=64ms, 3=128ms, 4=250ms, 5=500ms, 6=1 sec, 7=2 sec, 8=4 sec, 9=8sec
const float WD_MODE[10] = {0.016, 0.032, 0.064, 0.128, 0.25, 0.5, 1, 2, 4, 8};

// ####################### Variables ########################

USI_TWI bus;                            // TinyWireM instance (I2C bus)
BH1750FVI BH1750(bus);                  // Light sensor instance
RTC_DS1307 RTC(bus);                    // RTC clock instance

tmDriftInfo di;

volatile uint8_t wdCount = 0;

// Watchdog interrupt count before reading light value
// wd count * wd interval == 4 * 8 s = 32 s
uint16_t wdMatch = 0;
boolean first = false;
static float luxSamples[SAMPLE_COUNT] = {};
static uint8_t readCounter = 0;
static boolean relayState = false, relayStateMem = false;

// ####################### Functions ########################

void setupWatchdog(float tick) {
  // 0=16ms, 1=32ms,2=64ms,3=128ms,4=250ms,5=500ms
  // 6=1 sec,7=2 sec, 8=4 sec, 9= 8sec
  uint8_t wdreg, wdmode = 0;

  for (uint8_t i = 0; i < 10; ++i) {
    if (WD_MODE[i] == tick) {
      wdmode = i;
    }
  }

  if (wdmode > 9)
    wdmode = 9;
  wdreg = wdmode & 7;

  if (wdmode > 7)
    wdreg |= (1 << 5);
  wdreg |= (1 << WDCE);

  MCUSR &= ~(1 << WDRF);

  // Start timed sequence
  WDTCR |= (1 << WDCE) | (1 << WDE);

  // Set new watchdog timeout value
  WDTCR = wdreg;
  WDTCR |= _BV(WDIE);
}

void systemSleep() {
  set_sleep_mode(SLEEP_MODE_PWR_DOWN);  // Sleep mode is set here
  sleep_enable();
  sleep_mode();                         // System actually sleeps here
  sleep_disable();                      // System continues execution here when watchdog timed out
}

// Watchdog Interrupt Service Routine
ISR(WDT_vect) {
  wdCount++;
}

// PCINT Interrupt Service Routine (unused)
ISR(PCINT0_vect) {
  // Don't do anything here but we must include this
  // block of code otherwise the interrupt calls an
  // uninitialized interrupt handler.
}

time_t timeProvider() {
  return RTC.get();
}

void printDigits(int digits) {
  // Utility function for digital clock display: prints preceding colon and leading 0
  if (digits < 10)
    Serial.print('0');
  Serial.print(digits);
}

void SerialDisplayDateTime(const time_t &timeToDisplay) {
  printDigits(day(timeToDisplay));
  Serial.print("/");
  printDigits(month(timeToDisplay));
  Serial.print("/");
  Serial.print(year(timeToDisplay));
  Serial.print(" ");
  printDigits(hour(timeToDisplay));
  Serial.print(":");
  printDigits(minute(timeToDisplay));
  Serial.print(":");
  printDigits(second(timeToDisplay));
  Serial.print(" ");
}

void setTime() {
  tmElements_t tme;
  time_t newTime;

  tme.Year   = 46;
  tme.Month  = 2;
  tme.Day    = 20;
  tme.Hour   = 17;
  tme.Minute = 9;
  tme.Second = 45;
  newTime = makeTime(tme);
  RTC.set(newTime);                             // Set the RTC
  setTime(newTime);                             // Set the system time

  tmDriftInfo diUpdate = RTC.read_DriftInfo();  // Update DriftInfo in RTC
  diUpdate.DriftStart = newTime;
  RTC.write_DriftInfo(diUpdate);

  SerialDisplayDateTime(newTime);
}

float sample() {
  float tmpLux = 0.0;
  // Shift left values in sample array
  for (int i = 0; i < SAMPLE_COUNT - 1; ++i) {
    luxSamples[i] = luxSamples[i + 1];
    tmpLux += luxSamples[i];
  }

  luxSamples[SAMPLE_COUNT - 1] = BH1750.getLightIntensity();  // Get lux value

  tmpLux += luxSamples[SAMPLE_COUNT - 1];

  if (readCounter < SAMPLE_COUNT) {
    readCounter++;
  }

  return tmpLux / readCounter;
}

void printLuxArray() {
  for (int i = 0; i < SAMPLE_COUNT; ++i) {
    Serial.print(luxSamples[i]);
    Serial.print(" ");
  }
}

void cleanLuxArray() {
  readCounter = 0;
  for (int i = 0; i < SAMPLE_COUNT; ++i) {
    luxSamples[i] = 0.0;
  }
}

boolean checkEnable(const time_t &now) {
  int nowH = hour(now);

  if (nowH <= 16 || nowH >= 23) {
    return false;
  } else {
    return true;
  }
}

boolean checkLightCond(float lux) {
  int low = 0, high = 0;
  for (int i = SAMPLE_COUNT / 2 + 1 ; i < SAMPLE_COUNT - 1; ++i) {
    if (luxSamples[i] <= LUX_TH) {
      low++;
    } else {
      high++;
    }
  }

  if (readCounter >= SAMPLE_COUNT / 2 + 1) {

    // Turn light on
    if (high <= 2 && lux <= LUX_TH) {
      return true;
    }

    // Turn light off
    if (relayState && lux <= LUX_TH + LUX_TH_HIST) {
      return true;
    } else {
      return false;
    }
  }
  return false;
}

void setup() {
  Serial.begin(9600);

  pinMode(RELAY_SW_OUT, OUTPUT);
  pinMode(CMD_MAN_IN, INPUT_PULLUP);

  // Setup watchdog timeout to 8 s (mode 9)
  setupWatchdog(WD_TICK_TIME);

  // Watchdog interrupt count before reading light value
  // example: wd count * wd interval == 4 * 8 s = 32 s
  wdMatch = (uint16_t) WD_WAIT_TIME / WD_TICK_TIME;

  // Setup Pin change interrupt on PCINT1
  GIMSK |= _BV(PCIE);
  PCMSK |= _BV(PCINT1);

  // Set the internal registers to reduce power consumes
  ACSR   = (1 << ACD);                  // Shut down the analog comparator
  MCUCR |= (1 << BODS);                 // BOD disabled

  // I2C begin() is called BEFORE sensors library "begin" methods:
  // it is called just once for all the sensors.
  bus.begin();

  // Sensors initialization
  // Light sensor
  BH1750.powerOn();
  BH1750.setMode(BH1750_CONTINUOUS_HIGH_RES_MODE_2);
  BH1750.setMtreg(200);                 // Set measurement time register to high value

  // Real time clock
  setSyncProvider(timeProvider);        // Pointer to function to get the time from the RTC
  setSyncInterval(WD_WAIT_TIME);        // Set system time at every sensor read

  // Update drift info from RTC
  di = RTC.read_DriftInfo();
  di.DriftDays = 1000;                  // RTC drift in seconds/day. 18 s per day is 18000/1000
  di.DriftSeconds = 18000;
  RTC.write_DriftInfo(di);
  setDriftInfo(di);
}

void loop() {
  BH1750.sleep();                       // Send light sensor to sleep
  systemSleep();                        // Send the unit to sleep

  // Manual command to turn light on
  if (!digitalRead(CMD_MAN_IN)) {
    if (!first) {
      // Used only once to set the time with external input
      // setTime();
      first = true;
      cleanLuxArray();
      relayState = relayStateMem = true;
      digitalWrite(RELAY_SW_OUT, relayState);
    }

  } else  {
    // first = true -> First cycle in automatic mode, after manual mode

    if (wdCount >= wdMatch || first) {
      first = false;
      wdCount = 0;

      time_t timeNow = now();

      if (checkEnable(timeNow)) {
        BH1750.wakeUp(BH1750_CONTINUOUS_HIGH_RES_MODE_2);
        delay(300);

        float lux = sample();
        relayState = checkLightCond(lux);

#ifdef DEBUG
        SerialDisplayDateTime(timeNow);
        printLuxArray();
        Serial.print("= ");
        Serial.print(lux);
        Serial.print(" ");
        if (relayState)
          Serial.println("ON");
        else Serial.println("OFF");
#endif
        if (relayState != relayStateMem) {
          digitalWrite(RELAY_SW_OUT, relayState);
          relayStateMem = relayState;
        }
      } else {
        cleanLuxArray();
        relayState = relayStateMem = false;
        digitalWrite(RELAY_SW_OUT, relayState);
      }
    }
  }
}
<commit_msg>Fixed bug in hour condition check <commit_after>/*
 *  This file is part of PlantLight application.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

/*
 * Built for Attiny85 1Mhz, using AVR USBasp programmer.
 * VERSION 0.8
 */

#include <avr/sleep.h>
#include <avr/interrupt.h>

#include <Arduino.h>
#include <TinyWireM.h>
#include <Time.h>
#include <BH1750FVI.h>
#include <DS1307RTC.h>

#define RELAY_SW_OUT            4       // Relay out pin
#define CMD_MAN_IN              1       // Manual light switch
#define WD_TICK_TIME            8       // Watchdog tick time [s] (check WD_MODE or datasheet)
#define WD_WAIT_TIME            16      // Time [s] to count between two sensor read
#define SAMPLE_COUNT            10      // Light sample count (average measurement)
#define LUX_TH                  10      // Lux threshold
#define LUX_TH_HIST             5       // Lux threshold (hysteresis compensation)

// ####################### Prototypes #######################

// Setup watchdog wakeup interrupt
// tick: watchdog tick time [s] (check WD_MODE or datasheet)
void setupWatchdog(float tick);

// Set system into the sleep state,
// system wakes up when watchdog is timed out
void systemSleep();

// Provides a time_t to Time library, wrapper for call
// to RTC.get(), which is not a static function (class member)
time_t timeProvider();

// Display a date in readable format on the serial interface
void printDigits(int digits);
void SerialDisplayDateTime(const time_t &timeToDisplay);

// Sets the RTC to the new time
void setTime();

// Checks the current time of day and month of the year.
// Returns true if the light is now enabled.
boolean checkEnable(const time_t &now);

// Checks actual lux value. Returns true if the conditions
// to turn lights on are met.
boolean checkLightCond(float lux);

// Sample some lux value and calculate the running average over time
float sample();

void printLuxArray();
void cleanLuxArray();

// ####################### Constants ########################

// Watchdog interrupt sleep time
// 0=16ms, 1=32ms, 2=64ms, 3=128ms, 4=250ms, 5=500ms, 6=1 sec, 7=2 sec, 8=4 sec, 9=8sec
const float WD_MODE[10] = {0.016, 0.032, 0.064, 0.128, 0.25, 0.5, 1, 2, 4, 8};

// ####################### Variables ########################

USI_TWI bus;                            // TinyWireM instance (I2C bus)
BH1750FVI BH1750(bus);                  // Light sensor instance
RTC_DS1307 RTC(bus);                    // RTC clock instance

tmDriftInfo di;

volatile uint8_t wdCount = 0;

// Watchdog interrupt count before reading light value
// wd count * wd interval == 4 * 8 s = 32 s
uint16_t wdMatch = 0;
boolean first = false;
static float luxSamples[SAMPLE_COUNT] = {};
static uint8_t readCounter = 0;
static boolean relayState = false, relayStateMem = false;

// ####################### Functions ########################

void setupWatchdog(float tick) {
  // 0=16ms, 1=32ms,2=64ms,3=128ms,4=250ms,5=500ms
  // 6=1 sec,7=2 sec, 8=4 sec, 9= 8sec
  uint8_t wdreg, wdmode = 0;

  for (uint8_t i = 0; i < 10; ++i) {
    if (WD_MODE[i] == tick) {
      wdmode = i;
    }
  }

  if (wdmode > 9)
    wdmode = 9;
  wdreg = wdmode & 7;

  if (wdmode > 7)
    wdreg |= (1 << 5);
  wdreg |= (1 << WDCE);

  MCUSR &= ~(1 << WDRF);

  // Start timed sequence
  WDTCR |= (1 << WDCE) | (1 << WDE);

  // Set new watchdog timeout value
  WDTCR = wdreg;
  WDTCR |= _BV(WDIE);
}

void systemSleep() {
  set_sleep_mode(SLEEP_MODE_PWR_DOWN);  // Sleep mode is set here
  sleep_enable();
  sleep_mode();                         // System actually sleeps here
  sleep_disable();                      // System continues execution here when watchdog timed out
}

// Watchdog Interrupt Service Routine
ISR(WDT_vect) {
  wdCount++;
}

// PCINT Interrupt Service Routine (unused)
ISR(PCINT0_vect) {
  // Don't do anything here but we must include this
  // block of code otherwise the interrupt calls an
  // uninitialized interrupt handler.
}

time_t timeProvider() {
  return RTC.get();
}

void printDigits(int digits) {
  // Utility function for digital clock display: prints preceding colon and leading 0
  if (digits < 10)
    Serial.print('0');
  Serial.print(digits);
}

void SerialDisplayDateTime(const time_t &timeToDisplay) {
  printDigits(day(timeToDisplay));
  Serial.print("/");
  printDigits(month(timeToDisplay));
  Serial.print("/");
  Serial.print(year(timeToDisplay));
  Serial.print(" ");
  printDigits(hour(timeToDisplay));
  Serial.print(":");
  printDigits(minute(timeToDisplay));
  Serial.print(":");
  printDigits(second(timeToDisplay));
  Serial.print(" ");
}

void setTime() {
  tmElements_t tme;
  time_t newTime;

  tme.Year   = 46;
  tme.Month  = 2;
  tme.Day    = 20;
  tme.Hour   = 17;
  tme.Minute = 9;
  tme.Second = 45;
  newTime = makeTime(tme);
  RTC.set(newTime);                             // Set the RTC
  setTime(newTime);                             // Set the system time

  tmDriftInfo diUpdate = RTC.read_DriftInfo();  // Update DriftInfo in RTC
  diUpdate.DriftStart = newTime;
  RTC.write_DriftInfo(diUpdate);

  SerialDisplayDateTime(newTime);
}

float sample() {
  float tmpLux = 0.0;
  // Shift left values in sample array
  for (int i = 0; i < SAMPLE_COUNT - 1; ++i) {
    luxSamples[i] = luxSamples[i + 1];
    tmpLux += luxSamples[i];
  }

  luxSamples[SAMPLE_COUNT - 1] = BH1750.getLightIntensity();  // Get lux value

  tmpLux += luxSamples[SAMPLE_COUNT - 1];

  if (readCounter < SAMPLE_COUNT) {
    readCounter++;
  }

  return tmpLux / readCounter;
}

void printLuxArray() {
  for (int i = 0; i < SAMPLE_COUNT; ++i) {
    Serial.print(luxSamples[i]);
    Serial.print(" ");
  }
}

void cleanLuxArray() {
  readCounter = 0;
  for (int i = 0; i < SAMPLE_COUNT; ++i) {
    luxSamples[i] = 0.0;
  }
}

boolean checkEnable(const time_t &now) {
  return (hour(now) >= 16 && hour(now) <= 22);
}

boolean checkLightCond(float lux) {
  int low = 0, high = 0;
  for (int i = SAMPLE_COUNT / 2 + 1 ; i < SAMPLE_COUNT - 1; ++i) {
    if (luxSamples[i] <= LUX_TH) {
      low++;
    } else {
      high++;
    }
  }

  if (readCounter >= SAMPLE_COUNT / 2 + 1) {

    // Turn light on
    if (high <= 2 && lux <= LUX_TH) {
      return true;
    }

    // Turn light off
    if (relayState && lux <= LUX_TH + LUX_TH_HIST) {
      return true;
    } else {
      return false;
    }
  }
  return false;
}

void setup() {
  Serial.begin(9600);

  pinMode(RELAY_SW_OUT, OUTPUT);
  pinMode(CMD_MAN_IN, INPUT_PULLUP);

  // Setup watchdog timeout to 8 s (mode 9)
  setupWatchdog(WD_TICK_TIME);

  // Watchdog interrupt count before reading light value
  // example: wd count * wd interval == 4 * 8 s = 32 s
  wdMatch = (uint16_t) WD_WAIT_TIME / WD_TICK_TIME;

  // Setup Pin change interrupt on PCINT1
  GIMSK |= _BV(PCIE);
  PCMSK |= _BV(PCINT1);

  // Set the internal registers to reduce power consumes
  ACSR   = (1 << ACD);                  // Shut down the analog comparator
  MCUCR |= (1 << BODS);                 // BOD disabled

  // I2C begin() is called BEFORE sensors library "begin" methods:
  // it is called just once for all the sensors.
  bus.begin();

  // Sensors initialization
  // Light sensor
  BH1750.powerOn();
  BH1750.setMode(BH1750_CONTINUOUS_HIGH_RES_MODE_2);
  BH1750.setMtreg(200);                 // Set measurement time register to high value

  // Real time clock
  setSyncProvider(timeProvider);        // Pointer to function to get the time from the RTC
  setSyncInterval(WD_WAIT_TIME);        // Set system time at every sensor read

  // Update drift info from RTC
  di = RTC.read_DriftInfo();
  di.DriftDays = 1000;                  // RTC drift in seconds/day. 18 s per day is 18000/1000
  di.DriftSeconds = 18000;
  RTC.write_DriftInfo(di);
  setDriftInfo(di);
}

void loop() {
  BH1750.sleep();                       // Send light sensor to sleep
  systemSleep();                        // Send the unit to sleep

  // Manual command to turn light on
  if (!digitalRead(CMD_MAN_IN)) {
    if (!first) {
      // Used only once to set the time with external input
      // setTime();
      first = true;
      cleanLuxArray();
      relayState = relayStateMem = true;
      digitalWrite(RELAY_SW_OUT, relayState);
    }

  } else  {
    // first = true -> First cycle in automatic mode, after manual mode

    if (wdCount >= wdMatch || first) {
      first = false;
      wdCount = 0;

      time_t timeNow = now();

      if (checkEnable(timeNow)) {
        BH1750.wakeUp(BH1750_CONTINUOUS_HIGH_RES_MODE_2);
        delay(300);

        float lux = sample();
        relayState = checkLightCond(lux);

#ifdef DEBUG
        SerialDisplayDateTime(timeNow);
        printLuxArray();
        Serial.print("= ");
        Serial.print(lux);
        Serial.print(" ");
        if (relayState)
          Serial.println("ON");
        else Serial.println("OFF");
#endif
        if (relayState != relayStateMem) {
          digitalWrite(RELAY_SW_OUT, relayState);
          relayStateMem = relayState;
        }
      } else {
        cleanLuxArray();
        relayState = relayStateMem = false;
        digitalWrite(RELAY_SW_OUT, relayState);
      }
    }
  }
}
<|endoftext|>


<commit_before>#include "stdafx.h"

#include "YouTuber.h"

//#define YOUTUBE_EXPERIMENT

#ifdef YOUTUBE_EXPERIMENT

#include <Shlobj.h>

#include <boost/python/exec.hpp>
#include <boost/python/import.hpp>
#include <boost/python/extract.hpp>

#include <boost/log/trivial.hpp>

#include <regex>
#include <fstream>
#include <iterator>
#include <streambuf>
#include <algorithm>
#include <memory>
#include <utility>

#include <tchar.h>

#include "unzip.h"
#include "http_get.h"

#include "MemoryMappedFile.h"

namespace {

// http://thejosephturner.com/blog/post/embedding-python-in-c-applications-with-boostpython-part-2/
// Parses the value of the active python exception
// NOTE SHOULD NOT BE CALLED IF NO EXCEPTION
std::string parse_python_exception()
{

    namespace py = boost::python;

    PyObject *type_ptr = NULL, *value_ptr = NULL, *traceback_ptr = NULL;
    // Fetch the exception info from the Python C API
    PyErr_Fetch(&type_ptr, &value_ptr, &traceback_ptr);

    // Fallback error
    std::string ret("Unfetchable Python error");
    // If the fetch got a type pointer, parse the type into the exception string
    if(type_ptr != NULL){
        py::handle<> h_type(type_ptr);
        py::str type_pstr(h_type);
        // Extract the string from the boost::python object
        py::extract<std::string> e_type_pstr(type_pstr);
        // If a valid string extraction is available, use it
        //  otherwise use fallback
        if(e_type_pstr.check())
            ret = e_type_pstr();
        else
            ret = "Unknown exception type";
    }
    // Do the same for the exception value (the stringification of the exception)
    if(value_ptr != NULL){
        py::handle<> h_val(value_ptr);
        py::str a(h_val);
        py::extract<std::string> returned(a);
        if(returned.check())
            ret +=  ": " + returned();
        else
            ret += std::string(": Unparseable Python error: ");
    }
    // Parse lines from the traceback using the Python traceback module
    if(traceback_ptr != NULL){
        py::handle<> h_tb(traceback_ptr);
        // Load the traceback module and the format_tb function
        py::object tb(py::import("traceback"));
        py::object fmt_tb(tb.attr("format_tb"));
        // Call format_tb to get a list of traceback strings
        py::object tb_list(fmt_tb(h_tb));
        // Join the traceback strings into a single string
        py::object tb_str(py::str("\n").join(tb_list));
        // Extract the string, check the extraction, and fallback in necessary
        py::extract<std::string> returned(tb_str);
        if(returned.check())
            ret += ": " + returned();
        else
            ret += std::string(": Unparseable Python traceback");
    }
    return ret;
}

const char PYTUBE_URL[] = "https://github.com/nficano/pytube/archive/master.zip";

const char SCRIPT_TEMPLATE[] = R"(import sys
sys.path.append("%s")
from pytube import YouTube
def getYoutubeUrl(url):
	return YouTube(url).streams.filter(progressive=True).order_by('resolution').desc().first().url)";

int from_hex(char ch)
{
    return isdigit(ch) ? ch - '0' : tolower(ch) - 'a' + 10;
}

std::string UrlUnescapeString(const std::string& s)
{
    std::istringstream ss(s);
    std::string result;
    std::getline(ss, result, '%');
    std::string buffer;
    while (std::getline(ss, buffer, '%'))
    {
        if (buffer.size() >= 2)
        {
            result += char((from_hex(buffer[0]) << 4) | from_hex(buffer[1])) + buffer.substr(2);
        }
    }
    return result;
}

bool extractYoutubeUrl(std::string& s)
{
    std::regex txt_regex(R"((http(s)?:\/\/)?((w){3}.)?youtu(be|.be)?(\.com)?\/.+)");
    std::string copy = s;
    for (int unescaped = 0; unescaped < 2; ++unescaped)
    {
        std::smatch m;
        if (std::regex_search(copy, m, txt_regex))
        {
            s = copy.substr(m.position());
            return true;
        }
        if (!unescaped)
            copy = UrlUnescapeString(copy);
    }

    return false;
}

bool DownloadAndExtractZip(const char* zipfile, const TCHAR* root)
{
    unzFile uf = unzOpen((voidpf)zipfile);
    if (!uf)
    {
        return false;
    }
    unzGoToFirstFile(uf);
    do {
        char filename[MAX_PATH];
        unzGetCurrentFileInfo(uf, 0, filename, sizeof(filename), 0, 0, 0, 0);

        TCHAR path[MAX_PATH];
        _tcscpy_s(path, root);
        PathAppend(path, CA2T(filename, CP_UTF8));

        auto pathlen = _tcslen(path);
        if (pathlen > 0 && (path[pathlen - 1] == _T('/') || path[pathlen - 1] == _T('\\')))
        {
            if (_tmkdir(path) != 0)
                return false;
        }
        else
        {
            unzOpenCurrentFile(uf);

            std::ofstream f(path, std::ofstream::binary);

            char buf[1024 * 64];
            int r;
            do
            {
                r = unzReadCurrentFile(uf, buf, sizeof(buf));
                if (r > 0)
                {
                    f.write(buf, r);
                }
            } while (r > 0);
            unzCloseCurrentFile(uf);
        }
    } while (unzGoToNextFile(uf) == UNZ_OK);

    unzClose(uf);
    return true;
}

class YouTubeDealer
{
public:
    YouTubeDealer();
    ~YouTubeDealer();

    bool isValid() const { return !!m_obj; }
    std::string getYoutubeUrl(const std::string& url);

private:
    boost::python::object m_obj;
};

YouTubeDealer::YouTubeDealer()
{
    // String buffer for holding the path.
    TCHAR strPath[MAX_PATH]{};

    // Get the special folder path.
    SHGetSpecialFolderPath(
        0,       // Hwnd
        strPath, // String buffer
        CSIDL_LOCAL_APPDATA, // CSLID of folder
        TRUE); // Create if doesn't exist?

    CString localAppdataPath = strPath;

    PathAppend(strPath, _T("pytube-master"));

    if (-1 == _taccess(strPath, 0)
        && (!DownloadAndExtractZip(PYTUBE_URL, localAppdataPath)
            || -1 == _taccess(strPath, 0)))
    {
        return;
    }
    using namespace boost::python;

    Py_Initialize();
    try {
        // Retrieve the main module.
        object main = import("__main__");

        // Retrieve the main module's namespace
        object global(main.attr("__dict__"));

        CT2A const convert(strPath, CP_UTF8);
        LPSTR const pszConvert = convert;
        std::replace(pszConvert, pszConvert + strlen(pszConvert), '\\', '/');

        char script[4096];
        sprintf_s(script, SCRIPT_TEMPLATE, pszConvert);

        // Define function in Python.
        object exec_result = exec(script, global, global);

        // Create a reference to it.
        m_obj = global["getYoutubeUrl"];
        if (!m_obj)
            Py_Finalize();
    }
    catch (const std::exception& ex)
    {
        BOOST_LOG_TRIVIAL(error) << "getYoutubeUrl() bootstrap exception \"" << ex.what() << "\"";
        Py_Finalize();
        return;
    }
    catch (const error_already_set&)
    {
        BOOST_LOG_TRIVIAL(error) << "getYoutubeUrl() bootstrap error \"" << parse_python_exception() << "\"";
        Py_Finalize();
        return;
    }
}

YouTubeDealer::~YouTubeDealer()
{
    if (isValid())
        Py_Finalize();
}

std::string YouTubeDealer::getYoutubeUrl(const std::string& url)
{
    BOOST_LOG_TRIVIAL(trace) << "getYoutubeUrl() url = \"" << url << "\"";
    using namespace boost::python;
    std::string result;
    try
    {
        result = extract<std::string>(m_obj(url));
    }
    catch (const std::exception& ex)
    {
        BOOST_LOG_TRIVIAL(error) << "getYoutubeUrl() exception \"" << ex.what() << "\"";
    }
    catch (const error_already_set&)
    {
        BOOST_LOG_TRIVIAL(error) << "getYoutubeUrl() error \"" << parse_python_exception() << "\"";
    }

    BOOST_LOG_TRIVIAL(trace) << "getYoutubeUrl() returning \"" << result << "\"";

    return result;
}

std::vector<std::string> ParsePlaylist(const char* pData, const char* const pDataEnd)
{
    const char watch[] = "/watch?v=";

    std::vector<std::string> result;

    while ((pData = std::search(pData, pDataEnd, std::begin(watch), std::prev(std::end(watch)))) != pDataEnd)
    {
        const auto localEnd = std::find_if(pData, pDataEnd,
            [](char ch) { return ch == '&' || ch == '"' || ch == '\'' || ch == '\\'; });
        auto el = "https://www.youtube.com" + std::string(pData, localEnd);
        if (std::find(result.begin(), result.end(), el) == result.end())
            result.push_back(std::move(el));
        pData += sizeof(watch) / sizeof(watch[0]) - 1;
    }

    return result;
}

} // namespace

std::vector<std::string> ParsePlaylist(const std::string& url, bool force)
{
    if (!force && url.find("/playlist?list=") == std::string::npos)
        return{};

    CWaitCursor wait;
    CComVariant varBody = HttpGet(url.c_str());
    if ((VT_ARRAY | VT_UI1) != V_VT(&varBody))
        return{};

    auto psa = V_ARRAY(&varBody);
    LONG iLBound, iUBound;
    HRESULT hr = SafeArrayGetLBound(psa, 1, &iLBound);
    if (FAILED(hr))
        return{};
    hr = SafeArrayGetUBound(psa, 1, &iUBound);
    if (FAILED(hr))
        return{};

    const char* pData = nullptr;

    hr = SafeArrayAccessData(psa, (void**)&pData);
    if (FAILED(hr) || !pData)
        return{};

    const char* const pDataEnd = pData + iUBound - iLBound + 1;

    std::unique_ptr<SAFEARRAY, decltype(&SafeArrayUnaccessData)> guard(
        psa, SafeArrayUnaccessData);
    return ParsePlaylist(pData, pDataEnd);
}

std::vector<std::string> ParsePlaylistFile(const TCHAR* fileName)
{
    if (!_tcsstr(fileName, _T("playlist")) && !_tcsstr(fileName, _T("watch"))
		&& (_tcslen(fileName) <= 5 || _tcsicmp(fileName + _tcslen(fileName) - 5, _T(".html")) != 0))
        return{};

    MemoryMappedFile memoryMappedFile;
    if (!memoryMappedFile.MapFlie(fileName))
        return{};
    auto* const pData = static_cast<const char*>(memoryMappedFile.data());
    return ParsePlaylist(pData, pData + memoryMappedFile.size());
}

std::string getYoutubeUrl(std::string url)
{
    if (extractYoutubeUrl(url))
    {
        CWaitCursor wait;
        static YouTubeDealer buddy;
        if (buddy.isValid())
            return buddy.getYoutubeUrl(url);
    }

    return url;
}

#else // YOUTUBE_EXPERIMENT

std::vector<std::string> ParsePlaylist(const std::string&, bool)
{
    return{};
}

std::vector<std::string> ParsePlaylistFile(const TCHAR*)
{
    return{};
}

std::string getYoutubeUrl(std::string url)
{
    return url;
}

#endif // YOUTUBE_EXPERIMENT
<commit_msg>Python script logging<commit_after>#include "stdafx.h"

#include "YouTuber.h"

//#define YOUTUBE_EXPERIMENT

#ifdef YOUTUBE_EXPERIMENT

#include <Shlobj.h>

#include <boost/python/exec.hpp>
#include <boost/python/import.hpp>
#include <boost/python/extract.hpp>
#include <boost/python.hpp>

#include <boost/log/trivial.hpp>
#include <boost/log/sources/channel_logger.hpp>

#include <regex>
#include <fstream>
#include <iterator>
#include <streambuf>
#include <algorithm>
#include <memory>
#include <utility>

#include <tchar.h>

#include "unzip.h"
#include "http_get.h"

#include "MemoryMappedFile.h"

namespace {

// http://thejosephturner.com/blog/post/embedding-python-in-c-applications-with-boostpython-part-2/
// Parses the value of the active python exception
// NOTE SHOULD NOT BE CALLED IF NO EXCEPTION
std::string parse_python_exception()
{

    namespace py = boost::python;

    PyObject *type_ptr = NULL, *value_ptr = NULL, *traceback_ptr = NULL;
    // Fetch the exception info from the Python C API
    PyErr_Fetch(&type_ptr, &value_ptr, &traceback_ptr);

    // Fallback error
    std::string ret("Unfetchable Python error");
    // If the fetch got a type pointer, parse the type into the exception string
    if(type_ptr != NULL){
        py::handle<> h_type(type_ptr);
        py::str type_pstr(h_type);
        // Extract the string from the boost::python object
        py::extract<std::string> e_type_pstr(type_pstr);
        // If a valid string extraction is available, use it
        //  otherwise use fallback
        if(e_type_pstr.check())
            ret = e_type_pstr();
        else
            ret = "Unknown exception type";
    }
    // Do the same for the exception value (the stringification of the exception)
    if(value_ptr != NULL){
        py::handle<> h_val(value_ptr);
        py::str a(h_val);
        py::extract<std::string> returned(a);
        if(returned.check())
            ret +=  ": " + returned();
        else
            ret += std::string(": Unparseable Python error: ");
    }
    // Parse lines from the traceback using the Python traceback module
    if(traceback_ptr != NULL){
        py::handle<> h_tb(traceback_ptr);
        // Load the traceback module and the format_tb function
        py::object tb(py::import("traceback"));
        py::object fmt_tb(tb.attr("format_tb"));
        // Call format_tb to get a list of traceback strings
        py::object tb_list(fmt_tb(h_tb));
        // Join the traceback strings into a single string
        py::object tb_str(py::str("\n").join(tb_list));
        // Extract the string, check the extraction, and fallback in necessary
        py::extract<std::string> returned(tb_str);
        if(returned.check())
            ret += ": " + returned();
        else
            ret += std::string(": Unparseable Python traceback");
    }
    return ret;
}

class LoggerStream
{
public:
    void write(const std::string& what)
    {
        using namespace boost::log;
        BOOST_LOG(sources::channel_logger_mt<>(keywords::channel = "python")) << what;
    }
    void flush() {}
};

const char PYTUBE_URL[] = "https://github.com/nficano/pytube/archive/master.zip";

const char SCRIPT_TEMPLATE[] = R"(import sys
sys.stderr = LoggerStream()
sys.path.append("%s")
from pytube import YouTube
def getYoutubeUrl(url):
	return YouTube(url).streams.filter(progressive=True).order_by('resolution').desc().first().url)";

int from_hex(char ch)
{
    return isdigit(ch) ? ch - '0' : tolower(ch) - 'a' + 10;
}

std::string UrlUnescapeString(const std::string& s)
{
    std::istringstream ss(s);
    std::string result;
    std::getline(ss, result, '%');
    std::string buffer;
    while (std::getline(ss, buffer, '%'))
    {
        if (buffer.size() >= 2)
        {
            result += char((from_hex(buffer[0]) << 4) | from_hex(buffer[1])) + buffer.substr(2);
        }
    }
    return result;
}

bool extractYoutubeUrl(std::string& s)
{
    std::regex txt_regex(R"((http(s)?:\/\/)?((w){3}.)?youtu(be|.be)?(\.com)?\/.+)");
    std::string copy = s;
    for (int unescaped = 0; unescaped < 2; ++unescaped)
    {
        std::smatch m;
        if (std::regex_search(copy, m, txt_regex))
        {
            s = copy.substr(m.position());
            return true;
        }
        if (!unescaped)
            copy = UrlUnescapeString(copy);
    }

    return false;
}

bool DownloadAndExtractZip(const char* zipfile, const TCHAR* root)
{
    unzFile uf = unzOpen((voidpf)zipfile);
    if (!uf)
    {
        return false;
    }
    unzGoToFirstFile(uf);
    do {
        char filename[MAX_PATH];
        unzGetCurrentFileInfo(uf, 0, filename, sizeof(filename), 0, 0, 0, 0);

        TCHAR path[MAX_PATH];
        _tcscpy_s(path, root);
        PathAppend(path, CA2T(filename, CP_UTF8));

        auto pathlen = _tcslen(path);
        if (pathlen > 0 && (path[pathlen - 1] == _T('/') || path[pathlen - 1] == _T('\\')))
        {
            if (_tmkdir(path) != 0)
                return false;
        }
        else
        {
            unzOpenCurrentFile(uf);

            std::ofstream f(path, std::ofstream::binary);

            char buf[1024 * 64];
            int r;
            do
            {
                r = unzReadCurrentFile(uf, buf, sizeof(buf));
                if (r > 0)
                {
                    f.write(buf, r);
                }
            } while (r > 0);
            unzCloseCurrentFile(uf);
        }
    } while (unzGoToNextFile(uf) == UNZ_OK);

    unzClose(uf);
    return true;
}

class YouTubeDealer
{
public:
    YouTubeDealer();
    ~YouTubeDealer();

    bool isValid() const { return !!m_obj; }
    std::string getYoutubeUrl(const std::string& url);

private:
    boost::python::object m_obj;
};

YouTubeDealer::YouTubeDealer()
{
    // String buffer for holding the path.
    TCHAR strPath[MAX_PATH]{};

    // Get the special folder path.
    SHGetSpecialFolderPath(
        0,       // Hwnd
        strPath, // String buffer
        CSIDL_LOCAL_APPDATA, // CSLID of folder
        TRUE); // Create if doesn't exist?

    CString localAppdataPath = strPath;

    PathAppend(strPath, _T("pytube-master"));

    if (-1 == _taccess(strPath, 0)
        && (!DownloadAndExtractZip(PYTUBE_URL, localAppdataPath)
            || -1 == _taccess(strPath, 0)))
    {
        return;
    }
    using namespace boost::python;

    Py_Initialize();
    try {
        // Retrieve the main module.
        object main = import("__main__");

        // Retrieve the main module's namespace
        object global(main.attr("__dict__"));

        global["LoggerStream"] = class_<LoggerStream>("LoggerStream", init<>())
            .def("write", &LoggerStream::write)
            .def("flush", &LoggerStream::flush);

        CT2A const convert(strPath, CP_UTF8);
        LPSTR const pszConvert = convert;
        std::replace(pszConvert, pszConvert + strlen(pszConvert), '\\', '/');

        char script[4096];
        sprintf_s(script, SCRIPT_TEMPLATE, pszConvert);

        // Define function in Python.
        object exec_result = exec(script, global, global);

        // Create a reference to it.
        m_obj = global["getYoutubeUrl"];
        if (!m_obj)
            Py_Finalize();
    }
    catch (const std::exception& ex)
    {
        BOOST_LOG_TRIVIAL(error) << "getYoutubeUrl() bootstrap exception \"" << ex.what() << "\"";
        Py_Finalize();
        return;
    }
    catch (const error_already_set&)
    {
        BOOST_LOG_TRIVIAL(error) << "getYoutubeUrl() bootstrap error \"" << parse_python_exception() << "\"";
        Py_Finalize();
        return;
    }
}

YouTubeDealer::~YouTubeDealer()
{
    if (isValid())
        Py_Finalize();
}

std::string YouTubeDealer::getYoutubeUrl(const std::string& url)
{
    BOOST_LOG_TRIVIAL(trace) << "getYoutubeUrl() url = \"" << url << "\"";
    using namespace boost::python;
    std::string result;
    try
    {
        result = extract<std::string>(m_obj(url));
    }
    catch (const std::exception& ex)
    {
        BOOST_LOG_TRIVIAL(error) << "getYoutubeUrl() exception \"" << ex.what() << "\"";
    }
    catch (const error_already_set&)
    {
        BOOST_LOG_TRIVIAL(error) << "getYoutubeUrl() error \"" << parse_python_exception() << "\"";
    }

    BOOST_LOG_TRIVIAL(trace) << "getYoutubeUrl() returning \"" << result << "\"";

    return result;
}

std::vector<std::string> ParsePlaylist(const char* pData, const char* const pDataEnd)
{
    const char watch[] = "/watch?v=";

    std::vector<std::string> result;

    while ((pData = std::search(pData, pDataEnd, std::begin(watch), std::prev(std::end(watch)))) != pDataEnd)
    {
        const auto localEnd = std::find_if(pData, pDataEnd,
            [](char ch) { return ch == '&' || ch == '"' || ch == '\'' || ch == '\\'; });
        auto el = "https://www.youtube.com" + std::string(pData, localEnd);
        if (std::find(result.begin(), result.end(), el) == result.end())
            result.push_back(std::move(el));
        pData += sizeof(watch) / sizeof(watch[0]) - 1;
    }

    return result;
}

} // namespace

std::vector<std::string> ParsePlaylist(const std::string& url, bool force)
{
    if (!force && url.find("/playlist?list=") == std::string::npos)
        return{};

    CWaitCursor wait;
    CComVariant varBody = HttpGet(url.c_str());
    if ((VT_ARRAY | VT_UI1) != V_VT(&varBody))
        return{};

    auto psa = V_ARRAY(&varBody);
    LONG iLBound, iUBound;
    HRESULT hr = SafeArrayGetLBound(psa, 1, &iLBound);
    if (FAILED(hr))
        return{};
    hr = SafeArrayGetUBound(psa, 1, &iUBound);
    if (FAILED(hr))
        return{};

    const char* pData = nullptr;

    hr = SafeArrayAccessData(psa, (void**)&pData);
    if (FAILED(hr) || !pData)
        return{};

    const char* const pDataEnd = pData + iUBound - iLBound + 1;

    std::unique_ptr<SAFEARRAY, decltype(&SafeArrayUnaccessData)> guard(
        psa, SafeArrayUnaccessData);
    return ParsePlaylist(pData, pDataEnd);
}

std::vector<std::string> ParsePlaylistFile(const TCHAR* fileName)
{
    if (!_tcsstr(fileName, _T("playlist")) && !_tcsstr(fileName, _T("watch"))
		&& (_tcslen(fileName) <= 5 || _tcsicmp(fileName + _tcslen(fileName) - 5, _T(".html")) != 0))
        return{};

    MemoryMappedFile memoryMappedFile;
    if (!memoryMappedFile.MapFlie(fileName))
        return{};
    auto* const pData = static_cast<const char*>(memoryMappedFile.data());
    return ParsePlaylist(pData, pData + memoryMappedFile.size());
}

std::string getYoutubeUrl(std::string url)
{
    if (extractYoutubeUrl(url))
    {
        CWaitCursor wait;
        static YouTubeDealer buddy;
        if (buddy.isValid())
            return buddy.getYoutubeUrl(url);
    }

    return url;
}

#else // YOUTUBE_EXPERIMENT

std::vector<std::string> ParsePlaylist(const std::string&, bool)
{
    return{};
}

std::vector<std::string> ParsePlaylistFile(const TCHAR*)
{
    return{};
}

std::string getYoutubeUrl(std::string url)
{
    return url;
}

#endif // YOUTUBE_EXPERIMENT
<|endoftext|>