<commit_before>// Copyright 2016 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <vector>
#include <iostream>
#include <exception>
#include <algorithm>
#include <iomanip>

#include <configuration.hpp>

#include <ArgumentList.hpp>
#include <InitializeOpenCL.hpp>
#include <Kernel.hpp>
#include <Stencil.hpp>
#include <utils.hpp>
#include <Timer.hpp>
#include <Stats.hpp>

const inputDataType magicValue = 42;

void initializeDeviceMemory(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * output_d, const unsigned int outputSize);

int main(int argc, char * argv[]) {
  bool reInit = true;
  unsigned int nrIterations = 0;
  unsigned int clPlatformID = 0;
  unsigned int clDeviceID = 0;
  unsigned int vectorSize = 0;
  unsigned int maxThreads = 0;
  unsigned int maxItems = 0;
  unsigned int matrixWidth = 0;
  unsigned int padding = 0;
  TuneBench::Stencil2DConf conf;

  try {
    isa::utils::ArgumentList args(argc, argv);

    clPlatformID = args.getSwitchArgument< unsigned int >("-opencl_platform");
    clDeviceID = args.getSwitchArgument< unsigned int >("-opencl_device");
    nrIterations = args.getSwitchArgument< unsigned int >("-iterations");
    vectorSize = args.getSwitchArgument< unsigned int >("-vector");
    padding = args.getSwitchArgument< unsigned int >("-padding");
    maxThreads = args.getSwitchArgument< unsigned int >("-max_threads");
    maxItems = args.getSwitchArgument< unsigned int >("-max_items");
    matrixWidth = args.getSwitchArgument< unsigned int >("-matrix_width");
  } catch ( isa::utils::EmptyCommandLine & err ) {
    std::cerr << argv[0] << " -opencl_platform ... -opencl_device ... -iterations ... -vector ... -padding ... -max_threads ... -max_items ... -matrix_width ..." << std::endl;
    return 1;
  } catch ( std::exception & err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }

  cl::Context clContext;
  std::vector< cl::Platform > * clPlatforms = new std::vector< cl::Platform >();
  std::vector< cl::Device > * clDevices = new std::vector< cl::Device >();
  std::vector< std::vector< cl::CommandQueue > > * clQueues = 0;

  // Allocate host memory
  std::vector< inputDataType > input((matrixWidth + 2) * isa::utils::pad(matrixWidth + 2, padding)), output(matrixWidth * isa::utils::pad(matrixWidth, padding));
  cl::Buffer input_d, output_d;

  std::fill(input.begin(), input.end(), magicValue);

  std::cout << std::fixed << std::endl;
  std::cout << "# matrixWidth localMemory nrThreadsD0 nrThreadsD1 nrItemsD0 nrItemsD1 GFLOP/s time stdDeviation COV" << std::endl << std::endl;

  for ( unsigned int threads = vectorSize; threads <= maxThreads; threads += vectorSize ) {
    conf.setNrThreadsD0(threads);
    for ( unsigned int threads = 1; conf.getNrThreadsD0() * threads <= maxThreads; threads++ ) {
      conf.setNrThreadsD1(threads);
      for ( unsigned int items = 1; items <= maxItems; items++ ) {
        conf.setNrItemsD0(items);
        if ( matrixWidth % (conf.getNrThreadsD0() * conf.getNrItemsD0()) != 0 ) {
          continue;
        }
        for ( unsigned int items = 1; conf.getNrItemsD0() * items <= maxItems; items++ ) {
          conf.setNrItemsD1(items);
          if ( matrixWidth % (conf.getNrThreadsD1() * conf.getNrItemsD1()) != 0 ) {
            continue;
          }

          // Generate kernel
          double gflops = isa::utils::giga(static_cast< uint64_t >(matrixWidth) * matrixWidth * 11);
          cl::Event clEvent;
          cl::Kernel * kernel;
          isa::utils::Timer timer;
          std::string * code = TuneBench::getStencil2DOpenCL(conf, inputDataName, matrixWidth, padding);

          if ( reInit ) {
            delete clQueues;
            clQueues = new std::vector< std::vector< cl::CommandQueue > >();
            isa::OpenCL::initializeOpenCL(clPlatformID, 1, clPlatforms, &clContext, clDevices, clQueues);
            try {
              initializeDeviceMemory(clContext, &(clQueues->at(clDeviceID)[0]), &input, &input_d, &output_d, output.size());
            } catch ( cl::Error & err ) {
              return -1;
            }
            reInit = false;
          }
          try {
            kernel = isa::OpenCL::compile("stencil2D", *code, "-cl-mad-enable -Werror", clContext, clDevices->at(clDeviceID));
          } catch ( cl::Error & err ) {
            std::cerr << err.what() << std::endl;
            delete code;
            break;
          }
          delete code;

          cl::NDRange global(matrixWidth / conf.getNrItemsD0(), matrixWidth / conf.getNrItemsD1());
          cl::NDRange local(conf.getNrThreadsD0(), conf.getNrThreadsD1());

          kernel->setArg(0, input_d);
          kernel->setArg(1, output_d);

          try {
            // Warm-up run
            clQueues->at(clDeviceID)[0].finish();
            clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &clEvent);
            clEvent.wait();
            // Tuning runs
            for ( unsigned int iteration = 0; iteration < nrIterations; iteration++ ) {
              timer.start();
              clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &clEvent);
              clEvent.wait();
              timer.stop();
            }
            clQueues->at(clDeviceID)[0].enqueueReadBuffer(output_d, CL_TRUE, 0, output.size() * sizeof(inputDataType), reinterpret_cast< void * >(output.data()), 0, &clEvent);
            clEvent.wait();
          } catch ( cl::Error & err ) {
            reInit = true;
            std::cerr << "OpenCL kernel execution error (";
            std::cerr << conf.print();
            std::cerr << "): ";
            std::cerr << isa::utils::toString(err.err()) << std::endl;
            delete kernel;
            if ( err.err() == -4 || err.err() == -61 ) {
              return -1;
            }
            break;
          }
          delete kernel;

          bool error = false;
          for ( auto item = output.begin(); item != output.end(); ++item ) {
            if ( !isa::utils::same(*item, (magicValue * 0.25f) + (magicValue * 0.6f) + (magicValue * 0.2f)) ) {
              std::cerr << "Output error (" << conf.print() << ")." << std::endl;
              error = true;
              break;
            }
          }
          if ( error ) {
            continue;
          }

          std::cout << matrixWidth << " ";
          std::cout << conf.print() << " ";
          std::cout << std::setprecision(3);
          std::cout << gflops / timer.getAverageTime() << " ";
          std::cout << std::setprecision(6);
          std::cout << timer.getAverageTime() << " " << timer.getStandardDeviation() << " " << timer.getCoefficientOfVariation() << std::endl;
        }
      }
    }
  }
  std::cout << std::endl;

  return 0;
}

void initializeDeviceMemory(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * output_d, const unsigned int outputSize) {
  try {
    *input_d = cl::Buffer(clContext, CL_MEM_READ_ONLY, input->size() * sizeof(inputDataType), 0, 0);
    *output_d = cl::Buffer(clContext, CL_MEM_WRITE_ONLY, outputSize * sizeof(inputDataType), 0, 0);
    clQueue->enqueueWriteBuffer(*input_d, CL_FALSE, 0, input->size() * sizeof(inputDataType), reinterpret_cast< void * >(input->data()));
    clQueue->finish();
  } catch ( cl::Error & err ) {
    std::cerr << "OpenCL error (memory initialization): " << isa::utils::toString(err.err()) << "." << std::endl;
    throw;
  }
}

<commit_msg>Adding local memory switch.<commit_after>// Copyright 2016 Alessio Sclocco <a.sclocco@vu.nl>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <vector>
#include <iostream>
#include <exception>
#include <algorithm>
#include <iomanip>

#include <configuration.hpp>

#include <ArgumentList.hpp>
#include <InitializeOpenCL.hpp>
#include <Kernel.hpp>
#include <Stencil.hpp>
#include <utils.hpp>
#include <Timer.hpp>
#include <Stats.hpp>

const inputDataType magicValue = 42;

void initializeDeviceMemory(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * output_d, const unsigned int outputSize);

int main(int argc, char * argv[]) {
  bool reInit = true;
  unsigned int nrIterations = 0;
  unsigned int clPlatformID = 0;
  unsigned int clDeviceID = 0;
  unsigned int vectorSize = 0;
  unsigned int maxThreads = 0;
  unsigned int maxItems = 0;
  unsigned int matrixWidth = 0;
  unsigned int padding = 0;
  TuneBench::Stencil2DConf conf;

  try {
    isa::utils::ArgumentList args(argc, argv);

    clPlatformID = args.getSwitchArgument< unsigned int >("-opencl_platform");
    clDeviceID = args.getSwitchArgument< unsigned int >("-opencl_device");
    nrIterations = args.getSwitchArgument< unsigned int >("-iterations");
    vectorSize = args.getSwitchArgument< unsigned int >("-vector");
    padding = args.getSwitchArgument< unsigned int >("-padding");
    maxThreads = args.getSwitchArgument< unsigned int >("-max_threads");
    maxItems = args.getSwitchArgument< unsigned int >("-max_items");
    matrixWidth = args.getSwitchArgument< unsigned int >("-matrix_width");
    conf.setLocalMemory(args.getSwitch("-local"));
  } catch ( isa::utils::EmptyCommandLine & err ) {
    std::cerr << argv[0] << " -opencl_platform ... -opencl_device ... -iterations ... -vector ... -padding ... -max_threads ... -max_items ... -matrix_width ... [-local]" << std::endl;
    return 1;
  } catch ( std::exception & err ) {
    std::cerr << err.what() << std::endl;
    return 1;
  }

  cl::Context clContext;
  std::vector< cl::Platform > * clPlatforms = new std::vector< cl::Platform >();
  std::vector< cl::Device > * clDevices = new std::vector< cl::Device >();
  std::vector< std::vector< cl::CommandQueue > > * clQueues = 0;

  // Allocate host memory
  std::vector< inputDataType > input((matrixWidth + 2) * isa::utils::pad(matrixWidth + 2, padding)), output(matrixWidth * isa::utils::pad(matrixWidth, padding));
  cl::Buffer input_d, output_d;

  std::fill(input.begin(), input.end(), magicValue);

  std::cout << std::fixed << std::endl;
  std::cout << "# matrixWidth localMemory nrThreadsD0 nrThreadsD1 nrItemsD0 nrItemsD1 GFLOP/s time stdDeviation COV" << std::endl << std::endl;

  for ( unsigned int threads = vectorSize; threads <= maxThreads; threads += vectorSize ) {
    conf.setNrThreadsD0(threads);
    for ( unsigned int threads = 1; conf.getNrThreadsD0() * threads <= maxThreads; threads++ ) {
      conf.setNrThreadsD1(threads);
      for ( unsigned int items = 1; items <= maxItems; items++ ) {
        conf.setNrItemsD0(items);
        if ( matrixWidth % (conf.getNrThreadsD0() * conf.getNrItemsD0()) != 0 ) {
          continue;
        }
        for ( unsigned int items = 1; conf.getNrItemsD0() * items <= maxItems; items++ ) {
          conf.setNrItemsD1(items);
          if ( matrixWidth % (conf.getNrThreadsD1() * conf.getNrItemsD1()) != 0 ) {
            continue;
          }

          // Generate kernel
          double gflops = isa::utils::giga(static_cast< uint64_t >(matrixWidth) * matrixWidth * 11);
          cl::Event clEvent;
          cl::Kernel * kernel;
          isa::utils::Timer timer;
          std::string * code = TuneBench::getStencil2DOpenCL(conf, inputDataName, matrixWidth, padding);

          if ( reInit ) {
            delete clQueues;
            clQueues = new std::vector< std::vector< cl::CommandQueue > >();
            isa::OpenCL::initializeOpenCL(clPlatformID, 1, clPlatforms, &clContext, clDevices, clQueues);
            try {
              initializeDeviceMemory(clContext, &(clQueues->at(clDeviceID)[0]), &input, &input_d, &output_d, output.size());
            } catch ( cl::Error & err ) {
              return -1;
            }
            reInit = false;
          }
          try {
            kernel = isa::OpenCL::compile("stencil2D", *code, "-cl-mad-enable -Werror", clContext, clDevices->at(clDeviceID));
          } catch ( cl::Error & err ) {
            std::cerr << err.what() << std::endl;
            delete code;
            break;
          }
          delete code;

          cl::NDRange global(matrixWidth / conf.getNrItemsD0(), matrixWidth / conf.getNrItemsD1());
          cl::NDRange local(conf.getNrThreadsD0(), conf.getNrThreadsD1());

          kernel->setArg(0, input_d);
          kernel->setArg(1, output_d);

          try {
            // Warm-up run
            clQueues->at(clDeviceID)[0].finish();
            clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &clEvent);
            clEvent.wait();
            // Tuning runs
            for ( unsigned int iteration = 0; iteration < nrIterations; iteration++ ) {
              timer.start();
              clQueues->at(clDeviceID)[0].enqueueNDRangeKernel(*kernel, cl::NullRange, global, local, 0, &clEvent);
              clEvent.wait();
              timer.stop();
            }
            clQueues->at(clDeviceID)[0].enqueueReadBuffer(output_d, CL_TRUE, 0, output.size() * sizeof(inputDataType), reinterpret_cast< void * >(output.data()), 0, &clEvent);
            clEvent.wait();
          } catch ( cl::Error & err ) {
            reInit = true;
            std::cerr << "OpenCL kernel execution error (";
            std::cerr << conf.print();
            std::cerr << "): ";
            std::cerr << isa::utils::toString(err.err()) << std::endl;
            delete kernel;
            if ( err.err() == -4 || err.err() == -61 ) {
              return -1;
            }
            break;
          }
          delete kernel;

          bool error = false;
          for ( auto item = output.begin(); item != output.end(); ++item ) {
            if ( !isa::utils::same(*item, (magicValue * 0.25f) + (magicValue * 0.6f) + (magicValue * 0.2f)) ) {
              std::cerr << "Output error (" << conf.print() << ")." << std::endl;
              error = true;
              break;
            }
          }
          if ( error ) {
            continue;
          }

          std::cout << matrixWidth << " ";
          std::cout << conf.print() << " ";
          std::cout << std::setprecision(3);
          std::cout << gflops / timer.getAverageTime() << " ";
          std::cout << std::setprecision(6);
          std::cout << timer.getAverageTime() << " " << timer.getStandardDeviation() << " " << timer.getCoefficientOfVariation() << std::endl;
        }
      }
    }
  }
  std::cout << std::endl;

  return 0;
}

void initializeDeviceMemory(cl::Context & clContext, cl::CommandQueue * clQueue, std::vector< inputDataType > * input, cl::Buffer * input_d, cl::Buffer * output_d, const unsigned int outputSize) {
  try {
    *input_d = cl::Buffer(clContext, CL_MEM_READ_ONLY, input->size() * sizeof(inputDataType), 0, 0);
    *output_d = cl::Buffer(clContext, CL_MEM_WRITE_ONLY, outputSize * sizeof(inputDataType), 0, 0);
    clQueue->enqueueWriteBuffer(*input_d, CL_FALSE, 0, input->size() * sizeof(inputDataType), reinterpret_cast< void * >(input->data()));
    clQueue->finish();
  } catch ( cl::Error & err ) {
    std::cerr << "OpenCL error (memory initialization): " << isa::utils::toString(err.err()) << "." << std::endl;
    throw;
  }
}

<|endoftext|>


<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "statuswebserver.h"
#include <vespa/storageframework/storageframework.h>
#include <vespa/storageapi/message/persistence.h>
#include <vespa/fastlib/net/url.h>
#include <vespa/vespalib/util/host_name.h>
#include <vespa/vespalib/util/exceptions.h>
#include <vespa/vespalib/component/vtag.h>
#include <sstream>

#include <vespa/log/log.h>
LOG_SETUP(".status");

namespace storage {

StatusWebServer::StatusWebServer(
        framework::ComponentRegister& componentRegister,
        framework::StatusReporterMap& reporterMap,
        const config::ConfigUri & configUri)
    : _reporterMap(reporterMap),
      _workerMonitor(),
      _port(0),
      _httpServer(),
      _configFetcher(configUri.getContext()),
      _queuedRequests(),
      _component(std::make_unique<framework::Component>(componentRegister, "Status")),
      _thread()
{
    _configFetcher.subscribe<vespa::config::content::core::StorStatusConfig>(configUri.getConfigId(), this);
    _configFetcher.start();
    framework::MilliSecTime maxProcessingTime(60 * 60 * 1000);
    framework::MilliSecTime maxWaitTime(10 * 1000);
    _thread = _component->startThread(*this, maxProcessingTime, maxWaitTime);

}

StatusWebServer::~StatusWebServer()
{
    // Avoid getting config during shutdown
    _configFetcher.close();

    if (_httpServer.get() != 0) {
        LOG(debug, "Shutting down status web server on port %u", _httpServer->getListenPort());
    }
    // Delete http server to ensure that no more incoming requests reach us.
    _httpServer.reset(0);

    // Stop internal thread such that we don't process anymore web requests
    if (_thread.get() != 0) {
        _thread->interruptAndJoin(&_workerMonitor);
    }
}

void StatusWebServer::configure(std::unique_ptr<vespa::config::content::core::StorStatusConfig> config)
{
    int newPort = config->httpport;
        // If server is already running, ignore config updates that doesn't
        // alter port, or suggests random port.
    if (_httpServer.get() != 0) {
        if (newPort == 0 || newPort == _port) return;
    }
        // Try to create new server before destroying old.
    LOG(info, "Starting status web server on port %u.", newPort);
    std::unique_ptr<WebServer> server;
        // Negative port number means don't run the web server
    if (newPort >= 0) {
        server.reset(new WebServer(*this, newPort));
        server->SetKeepAlive(false);

        bool started = false;
        switch (server->Start()) {
            case FASTLIB_SUCCESS:
                started = true;
                break;
            case FASTLIB_HTTPSERVER_BADLISTEN:
                LOG(warning, "Listen failed on port %u", newPort);
                break;
            case FASTLIB_HTTPSERVER_NEWTHREADFAILED:
                LOG(warning, "Failed starting thread for status server on port %u", newPort);
                break;
            case FASTLIB_HTTPSERVER_ALREADYSTARTED:
                LOG(warning, "Failed starting status server on port %u (already started?)", newPort);
                break;
            default:
                LOG(warning, "Failed starting status server on port %u (unknown reason)", newPort);
                break;
        }
        if (!started) {
            std::ostringstream ost;
            ost << "Failed to start status HTTP server using port " << newPort << ".";
            if (_httpServer.get() != 0) {
                ost << " Status server still running on port " << _port << " instead of suggested port " << newPort;
            }
            LOG(fatal, "Failed to start status HTTP server using port %u. Old port was %u. Exiting.", newPort, _port);
            std::quick_exit(67);
        }
            // Now that we know config update went well, update internal state
        _port = server->getListenPort();
        LOG(config, "Status pages now available on port %u", _port);
        if (_httpServer.get() != 0) {
            LOG(debug, "Shutting down old status server.");
            _httpServer.reset(0);
            LOG(debug, "Done shutting down old status server.");
        }
    } else if (_httpServer.get() != 0) {
        LOG(info, "No longer running status server as negative port was given "
                  "in config, indicating not to run a server.");
    }
    _httpServer = std::move(server);
}

StatusWebServer::WebServer::WebServer(StatusWebServer& status, uint16_t port)
    : Fast_HTTPServer(port, NULL, 100, false, 128*1024, 10),
      _status(status),
      _serverSpec(vespalib::make_string("%s:%d", vespalib::HostName::get().c_str(), port))
{
}

namespace {
    /** Utility class for printing HTTP errors. */
    struct HttpErrorWriter {
        std::ostream& _out;

        HttpErrorWriter(std::ostream& out, vespalib::stringref error)
            : _out(out)
        {
            _out << "HTTP/1.1 " << error << "\r\n"
                    "Connection: Close\r\n"
                    "Content-type: text/html\r\n\r\n"
                    "<title>" << error << "</title>\r\n"
                    "
" << error << "
\r\n"
                    "
";
        }

        template<typename T>
        HttpErrorWriter& operator<<(const T& t) {
            _out << t;
            return *this;
        }

        ~HttpErrorWriter() {
            _out << "
\r\n"
                    "\r\n";
        }
    };
}

void
StatusWebServer::WebServer::onGetRequest(const string & tmpurl, const string &serverSpec, Fast_HTTPConnection& conn)
{
    Fast_URL urlCodec;
    int bufLength = tmpurl.length() * 2 + 10;
    char * encodedUrl = new char[bufLength];
    strcpy(encodedUrl, tmpurl.c_str());
    char decodedUrl[bufLength];
    urlCodec.DecodeQueryString(encodedUrl);
    urlCodec.decode(encodedUrl, decodedUrl, bufLength);
    delete [] encodedUrl;

    string url = decodedUrl;

    LOG(debug, "Status got get request '%s'", url.c_str());
    framework::HttpUrlPath urlpath(url.c_str(),
            StatusWebServer::getServerSpec(serverSpec, getServerSpec()));
    std::string link(urlpath.getPath());
    if (link.size() > 0 && link[0] == '/') link = link.substr(1);
        // Only allow crucial components not locking to answer directly.
        // (We want deadlockdetector status page to be available during a
        // deadlock
    if (link == "" || link == "deadlockdetector") {
        std::ostringstream ost;
        _status.handlePage(urlpath, ost);
        conn.Output(ost.str().c_str());
    } else {
            // Route other status requests that can possibly deadlock to a
            // worker thread.
        vespalib::MonitorGuard monitor(_status._workerMonitor);
        _status._queuedRequests.emplace_back(std::make_shared<HttpRequest>(url.c_str(), urlpath.getServerSpec()));
        HttpRequest* req = _status._queuedRequests.back().get();
        framework::SecondTime timeout(urlpath.get("timeout", 30u));
        framework::SecondTime timeoutTime(_status._component->getClock().getTimeInSeconds() + timeout);
        monitor.signal();
        while (true) {
            monitor.wait(100);
            bool done = false;
            if (req->_result.get()) {
                conn.Output(req->_result->c_str());
                LOG(debug, "Finished status request for '%s'", req->_url.c_str());
                done = true;
            } else {
                if (_status._component->getClock().getTimeInSeconds() > timeoutTime)
                {
                    std::ostringstream ost;
                    {
                        HttpErrorWriter writer(ost, "500 Internal Server Error");
                        writer << "Request " << url.c_str() << " timed out "
                               << "after " << timeout << " seconds.";
                    }
                    LOG(debug, "HTTP status request failed: %s. %zu requests queued",
                        ost.str().c_str(), _status._queuedRequests.size() - 1);
                    conn.Output(ost.str().c_str());
                    done = true;
                }
            }
            if (done) {
                for (std::list<HttpRequest::SP>::iterator it
                        = _status._queuedRequests.begin();
                     it != _status._queuedRequests.end(); ++it)
                {
                    if (it->get() == req) {
                        _status._queuedRequests.erase(it);
                        break;
                    }
                }
                break;
            }
        }
    }
}

namespace {
    class IndexPageReporter : public framework::HtmlStatusReporter {
        std::ostringstream ost;
        void reportHtmlStatus(std::ostream& out,const framework::HttpUrlPath&) const override {
            out << ost.str();
        }

    public:
        IndexPageReporter() : framework::HtmlStatusReporter("", "Index page") {}

        template<typename T>
        IndexPageReporter& operator<<(const T& t) { ost << t; return *this; }
    };
}

void
StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, std::ostream& out)
{
    vespalib::string link(urlpath.getPath());
    if (link.size() > 0 && link[0] == '/') link = link.substr(1);

    size_t slashPos = link.find('/');
    if (slashPos != std::string::npos) link = link.substr(0, slashPos);

    bool pageExisted = false;
    if ( ! link.empty()) {
        const framework::StatusReporter *reporter = _reporterMap.getStatusReporter(link);
        if (reporter != nullptr) {
            try {
                pageExisted = reporter->reportHttpHeader(out, urlpath);
                if (pageExisted) {
                    pageExisted = reporter->reportStatus(out, urlpath);
                }
            } catch (std::exception &e) {
                HttpErrorWriter writer(out, "500 Internal Server Error");
                writer << "
" << e.what() << "
";
                pageExisted = true;
            } catch (...) {
                HttpErrorWriter writer(out, "500 Internal Server Error");
                writer << "Unknown exception";
                pageExisted = true;
            }
            if (pageExisted) {
                LOG(spam, "Status finished request");
                return;
            }
        }
    }
    if (!pageExisted && link.size() > 0) {
        HttpErrorWriter writer(out, "404 Not found");
    } else {
        IndexPageReporter indexRep;
        indexRep << "
Binary version of Vespa: "
                 << vespalib::Vtag::currentVersion.toString()
                 << "
\n";
        {
            for (const framework::StatusReporter * reporter : _reporterMap.getStatusReporters()) {
                indexRep << "<a href=\"" << reporter->getId() << "\">"
                         << reporter->getName() << "
\n";
            }
        }
        indexRep.reportHttpHeader(out, urlpath);
        indexRep.reportStatus(out, urlpath);
    }
    LOG(spam, "Status finished request");
}

vespalib::string
StatusWebServer::getServerSpec(const vespalib::string &specFromRequest,
                               const vespalib::string &specFromServer)
{
    if (specFromRequest.empty()) {
        // This is a fallback in case the request spec is not given (HTTP 1.0 header)
        return specFromServer;
    }
    return specFromRequest;
}

void
StatusWebServer::run(framework::ThreadHandle& thread)
{
    while (!thread.interrupted()) {
        HttpRequest::SP request;
        {
            vespalib::MonitorGuard monitor(_workerMonitor);
            for (const HttpRequest::SP & cur : _queuedRequests) {
                if ( ! cur->_result ) {
                    request = cur;
                    break;
                }
            }
            if (!request.get()) {
                monitor.wait(10 * 1000);
                thread.registerTick(framework::WAIT_CYCLE);
                continue;
            }
        }
        framework::HttpUrlPath urlpath(request->_url, request->_serverSpec);
        std::ostringstream ost;
        handlePage(urlpath, ost);
        // If the same request is still in front of the queue
        // (it hasn't timed out), add the result to it.
        vespalib::MonitorGuard monitor(_workerMonitor);
        request->_result.reset(new vespalib::string(ost.str()));
        monitor.signal();
        thread.registerTick(framework::PROCESS_CYCLE);
    }
}

} // storage
<commit_msg>Request shutdown instead of doing a std::quick_exit.<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "statuswebserver.h"
#include <vespa/storageframework/storageframework.h>
#include <vespa/storageapi/message/persistence.h>
#include <vespa/fastlib/net/url.h>
#include <vespa/vespalib/util/host_name.h>
#include <vespa/vespalib/util/exceptions.h>
#include <vespa/vespalib/component/vtag.h>

#include <vespa/log/log.h>
LOG_SETUP(".status");

namespace storage {

StatusWebServer::StatusWebServer(
        framework::ComponentRegister& componentRegister,
        framework::StatusReporterMap& reporterMap,
        const config::ConfigUri & configUri)
    : _reporterMap(reporterMap),
      _workerMonitor(),
      _port(0),
      _httpServer(),
      _configFetcher(configUri.getContext()),
      _queuedRequests(),
      _component(std::make_unique<framework::Component>(componentRegister, "Status")),
      _thread()
{
    _configFetcher.subscribe<vespa::config::content::core::StorStatusConfig>(configUri.getConfigId(), this);
    _configFetcher.start();
    framework::MilliSecTime maxProcessingTime(60 * 60 * 1000);
    framework::MilliSecTime maxWaitTime(10 * 1000);
    _thread = _component->startThread(*this, maxProcessingTime, maxWaitTime);

}

StatusWebServer::~StatusWebServer()
{
    // Avoid getting config during shutdown
    _configFetcher.close();

    if (_httpServer.get() != 0) {
        LOG(debug, "Shutting down status web server on port %u", _httpServer->getListenPort());
    }
    // Delete http server to ensure that no more incoming requests reach us.
    _httpServer.reset(0);

    // Stop internal thread such that we don't process anymore web requests
    if (_thread.get() != 0) {
        _thread->interruptAndJoin(&_workerMonitor);
    }
}

void StatusWebServer::configure(std::unique_ptr<vespa::config::content::core::StorStatusConfig> config)
{
    int newPort = config->httpport;
        // If server is already running, ignore config updates that doesn't
        // alter port, or suggests random port.
    if (_httpServer.get() != 0) {
        if (newPort == 0 || newPort == _port) return;
    }
        // Try to create new server before destroying old.
    LOG(info, "Starting status web server on port %u.", newPort);
    std::unique_ptr<WebServer> server;
        // Negative port number means don't run the web server
    if (newPort >= 0) {
        server.reset(new WebServer(*this, newPort));
        server->SetKeepAlive(false);

        bool started = false;
        switch (server->Start()) {
            case FASTLIB_SUCCESS:
                started = true;
                break;
            case FASTLIB_HTTPSERVER_BADLISTEN:
                LOG(warning, "Listen failed on port %u", newPort);
                break;
            case FASTLIB_HTTPSERVER_NEWTHREADFAILED:
                LOG(warning, "Failed starting thread for status server on port %u", newPort);
                break;
            case FASTLIB_HTTPSERVER_ALREADYSTARTED:
                LOG(warning, "Failed starting status server on port %u (already started?)", newPort);
                break;
            default:
                LOG(warning, "Failed starting status server on port %u (unknown reason)", newPort);
                break;
        }
        if (!started) {
            std::ostringstream ost;
            ost << "Failed to start status HTTP server using port " << newPort << ".";
            if (_httpServer.get() != 0) {
                ost << " Status server still running on port " << _port << " instead of suggested port " << newPort;
            }
            LOG(fatal, "%s.", ost.str().c_str());
            _component->requestShutdown(ost.str());
        }
            // Now that we know config update went well, update internal state
        _port = server->getListenPort();
        LOG(config, "Status pages now available on port %u", _port);
        if (_httpServer.get() != 0) {
            LOG(debug, "Shutting down old status server.");
            _httpServer.reset(0);
            LOG(debug, "Done shutting down old status server.");
        }
    } else if (_httpServer.get() != 0) {
        LOG(info, "No longer running status server as negative port was given "
                  "in config, indicating not to run a server.");
    }
    _httpServer = std::move(server);
}

StatusWebServer::WebServer::WebServer(StatusWebServer& status, uint16_t port)
    : Fast_HTTPServer(port, NULL, 100, false, 128*1024, 10),
      _status(status),
      _serverSpec(vespalib::make_string("%s:%d", vespalib::HostName::get().c_str(), port))
{
}

namespace {
    /** Utility class for printing HTTP errors. */
    struct HttpErrorWriter {
        std::ostream& _out;

        HttpErrorWriter(std::ostream& out, vespalib::stringref error)
            : _out(out)
        {
            _out << "HTTP/1.1 " << error << "\r\n"
                    "Connection: Close\r\n"
                    "Content-type: text/html\r\n\r\n"
                    "<title>" << error << "</title>\r\n"
                    "
" << error << "
\r\n"
                    "
";
        }

        template<typename T>
        HttpErrorWriter& operator<<(const T& t) {
            _out << t;
            return *this;
        }

        ~HttpErrorWriter() {
            _out << "
\r\n"
                    "\r\n";
        }
    };
}

void
StatusWebServer::WebServer::onGetRequest(const string & tmpurl, const string &serverSpec, Fast_HTTPConnection& conn)
{
    Fast_URL urlCodec;
    int bufLength = tmpurl.length() * 2 + 10;
    char * encodedUrl = new char[bufLength];
    strcpy(encodedUrl, tmpurl.c_str());
    char decodedUrl[bufLength];
    urlCodec.DecodeQueryString(encodedUrl);
    urlCodec.decode(encodedUrl, decodedUrl, bufLength);
    delete [] encodedUrl;

    string url = decodedUrl;

    LOG(debug, "Status got get request '%s'", url.c_str());
    framework::HttpUrlPath urlpath(url.c_str(),
            StatusWebServer::getServerSpec(serverSpec, getServerSpec()));
    std::string link(urlpath.getPath());
    if (link.size() > 0 && link[0] == '/') link = link.substr(1);
        // Only allow crucial components not locking to answer directly.
        // (We want deadlockdetector status page to be available during a
        // deadlock
    if (link == "" || link == "deadlockdetector") {
        std::ostringstream ost;
        _status.handlePage(urlpath, ost);
        conn.Output(ost.str().c_str());
    } else {
            // Route other status requests that can possibly deadlock to a
            // worker thread.
        vespalib::MonitorGuard monitor(_status._workerMonitor);
        _status._queuedRequests.emplace_back(std::make_shared<HttpRequest>(url.c_str(), urlpath.getServerSpec()));
        HttpRequest* req = _status._queuedRequests.back().get();
        framework::SecondTime timeout(urlpath.get("timeout", 30u));
        framework::SecondTime timeoutTime(_status._component->getClock().getTimeInSeconds() + timeout);
        monitor.signal();
        while (true) {
            monitor.wait(100);
            bool done = false;
            if (req->_result.get()) {
                conn.Output(req->_result->c_str());
                LOG(debug, "Finished status request for '%s'", req->_url.c_str());
                done = true;
            } else {
                if (_status._component->getClock().getTimeInSeconds() > timeoutTime)
                {
                    std::ostringstream ost;
                    {
                        HttpErrorWriter writer(ost, "500 Internal Server Error");
                        writer << "Request " << url.c_str() << " timed out "
                               << "after " << timeout << " seconds.";
                    }
                    LOG(debug, "HTTP status request failed: %s. %zu requests queued",
                        ost.str().c_str(), _status._queuedRequests.size() - 1);
                    conn.Output(ost.str().c_str());
                    done = true;
                }
            }
            if (done) {
                for (std::list<HttpRequest::SP>::iterator it
                        = _status._queuedRequests.begin();
                     it != _status._queuedRequests.end(); ++it)
                {
                    if (it->get() == req) {
                        _status._queuedRequests.erase(it);
                        break;
                    }
                }
                break;
            }
        }
    }
}

namespace {
    class IndexPageReporter : public framework::HtmlStatusReporter {
        std::ostringstream ost;
        void reportHtmlStatus(std::ostream& out,const framework::HttpUrlPath&) const override {
            out << ost.str();
        }

    public:
        IndexPageReporter() : framework::HtmlStatusReporter("", "Index page") {}

        template<typename T>
        IndexPageReporter& operator<<(const T& t) { ost << t; return *this; }
    };
}

void
StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, std::ostream& out)
{
    vespalib::string link(urlpath.getPath());
    if (link.size() > 0 && link[0] == '/') link = link.substr(1);

    size_t slashPos = link.find('/');
    if (slashPos != std::string::npos) link = link.substr(0, slashPos);

    bool pageExisted = false;
    if ( ! link.empty()) {
        const framework::StatusReporter *reporter = _reporterMap.getStatusReporter(link);
        if (reporter != nullptr) {
            try {
                pageExisted = reporter->reportHttpHeader(out, urlpath);
                if (pageExisted) {
                    pageExisted = reporter->reportStatus(out, urlpath);
                }
            } catch (std::exception &e) {
                HttpErrorWriter writer(out, "500 Internal Server Error");
                writer << "
" << e.what() << "
";
                pageExisted = true;
            } catch (...) {
                HttpErrorWriter writer(out, "500 Internal Server Error");
                writer << "Unknown exception";
                pageExisted = true;
            }
            if (pageExisted) {
                LOG(spam, "Status finished request");
                return;
            }
        }
    }
    if (!pageExisted && link.size() > 0) {
        HttpErrorWriter writer(out, "404 Not found");
    } else {
        IndexPageReporter indexRep;
        indexRep << "
Binary version of Vespa: "
                 << vespalib::Vtag::currentVersion.toString()
                 << "
\n";
        {
            for (const framework::StatusReporter * reporter : _reporterMap.getStatusReporters()) {
                indexRep << "<a href=\"" << reporter->getId() << "\">"
                         << reporter->getName() << "
\n";
            }
        }
        indexRep.reportHttpHeader(out, urlpath);
        indexRep.reportStatus(out, urlpath);
    }
    LOG(spam, "Status finished request");
}

vespalib::string
StatusWebServer::getServerSpec(const vespalib::string &specFromRequest,
                               const vespalib::string &specFromServer)
{
    if (specFromRequest.empty()) {
        // This is a fallback in case the request spec is not given (HTTP 1.0 header)
        return specFromServer;
    }
    return specFromRequest;
}

void
StatusWebServer::run(framework::ThreadHandle& thread)
{
    while (!thread.interrupted()) {
        HttpRequest::SP request;
        {
            vespalib::MonitorGuard monitor(_workerMonitor);
            for (const HttpRequest::SP & cur : _queuedRequests) {
                if ( ! cur->_result ) {
                    request = cur;
                    break;
                }
            }
            if (!request.get()) {
                monitor.wait(10 * 1000);
                thread.registerTick(framework::WAIT_CYCLE);
                continue;
            }
        }
        framework::HttpUrlPath urlpath(request->_url, request->_serverSpec);
        std::ostringstream ost;
        handlePage(urlpath, ost);
        // If the same request is still in front of the queue
        // (it hasn't timed out), add the result to it.
        vespalib::MonitorGuard monitor(_workerMonitor);
        request->_result.reset(new vespalib::string(ost.str()));
        monitor.signal();
        thread.registerTick(framework::PROCESS_CYCLE);
    }
}

} // storage
<|endoftext|>


<commit_before>#include "testing/testing.hpp"

#include "map/framework.hpp"

#include "platform/http_request.hpp"
#include "platform/local_country_file_utils.hpp"
#include "platform/platform.hpp"
#include "platform/platform_tests_support/scoped_dir.hpp"
#include "platform/platform_tests_support/write_dir_changer.hpp"

#include "coding/file_name_utils.hpp"

#include "storage/storage.hpp"

#include "base/assert.hpp"

#include "std/condition_variable.hpp"
#include "std/mutex.hpp"
#include "std/set.hpp"
#include "std/unique_ptr.hpp"

using namespace platform;
using namespace storage;

namespace
{

string const kTestWebServer = "http://new-search.mapswithme.com/";

string const kMapTestDir = "map-tests";

TCountryId const kGroupCountryId = "New Zealand";
TCountriesSet const kLeafCountriesIds = {"Tokelau",
                                         "New Zealand North_Auckland",
                                         "New Zealand North_Wellington",
                                         "New Zealand South_Canterbury",
                                         "New Zealand South_Southland"};

string GetMwmFilePath(string const & version, TCountryId const & countryId)
{
  return my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
                               countryId + DATA_FILE_EXTENSION);
}

string GetMwmDownloadingFilePath(string const & version, TCountryId const & countryId)
{
  return my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
                               countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION DOWNLOADING_FILE_EXTENSION);
}

string GetMwmResumeFilePath(string const & version, TCountryId const & countryId)
{
  return my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
                               countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION RESUME_FILE_EXTENSION);
}

void DownloadGroup(Storage & storage, bool oneByOne)
{
  Platform & platform = GetPlatform();

  string const version = strings::to_string(storage.GetCurrentDataVersion());

  //  Get children nodes for the group node
  TCountriesVec v;
  storage.GetChildren(kGroupCountryId, v);
  TCountriesSet const children(v.begin(), v.end());
  v.clear();

  TCountriesSet changed;
  auto onChangeCountryFn = [&](TCountryId const & countryId)
  {
    TEST(children.find(countryId) != children.end(), ());
    changed.insert(countryId);
    if (!storage.IsDownloadInProgress())
    {
      // end waiting when all chilren will be downloaded
      testing::StopEventLoop();
    }
  };

  TCountriesSet downloaded;
  auto onProgressFn = [&](TCountryId const & countryId, TLocalAndRemoteSize const & mapSize)
  {
    TEST(children.find(countryId) != children.end(), ());
    if (mapSize.first == mapSize.second)
    {
      auto const res = downloaded.insert(countryId);
      TEST_EQUAL(res.second, true, ()); // every child is downloaded only once
    }
  };

  int const subsrcibtionId = storage.Subscribe(onChangeCountryFn, onProgressFn);

  // Check group node is not downloaded
  storage.GetDownloadedChildren(storage.GetRootId(), v);
  TEST(v.empty(), ());

  // Check children nodes are not downloaded
  storage.GetDownloadedChildren(kGroupCountryId, v);
  TEST(v.empty(), ());

  // Check status for the all children nodes is set to ENotDownloaded
  size_t totalGroupSize = 0;
  for (auto const & countryId : children)
  {
    TEST_EQUAL(Status::ENotDownloaded, storage.CountryStatusEx(countryId), ());
    NodeAttrs attrs;
    storage.GetNodeAttrs(countryId, attrs);
    TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
    TEST_GREATER(attrs.m_mwmSize, 0, ());
    totalGroupSize += attrs.m_mwmSize;
  }

  // Check status for the group node is set to ENotDownloaded
  NodeAttrs attrs;
  storage.GetNodeAttrs(kGroupCountryId, attrs);
  TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
  TEST_EQUAL(attrs.m_mwmSize, totalGroupSize, ());
  attrs = NodeAttrs();

  // Check there is no mwm or any other files for the children nodes
  for (auto const & countryId : children)
  {
    string const mwmFullPath = GetMwmFilePath(version, countryId);
    string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId);
    string const resumeFullPath = GetMwmResumeFilePath(version, countryId);
    TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
    TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
    TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
  }

  // Download the group
  if (oneByOne)
  {
    for (auto const & countryId : children)
      storage.DownloadNode(countryId);
  }
  else
  {
    storage.DownloadNode(kGroupCountryId);
  }
  // wait for downloading of all children
  testing::RunEventLoop();

  // Check all children nodes have been downloaded and changed.
  TEST_EQUAL(changed, children, ());
  TEST_EQUAL(downloaded, children, ());

  // Check status for the group node is set to EOnDisk
  storage.GetNodeAttrs(kGroupCountryId, attrs);
  TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());

  // Check status for the all children nodes is set to EOnDisk
  for (auto const & countryId : children)
  {
    TEST_EQUAL(Status::EOnDisk, storage.CountryStatusEx(countryId), ());
    NodeAttrs attrs;
    storage.GetNodeAttrs(countryId, attrs);
    TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());
  }

  // Check there is only mwm files are present and no any other for the children nodes
  for (auto const & countryId : children)
  {
    string const mwmFullPath = GetMwmFilePath(version, countryId);
    string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId);
    string const resumeFullPath = GetMwmResumeFilePath(version, countryId);
    TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
    TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
    TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
  }

  // Check group is downloaded
  storage.GetDownloadedChildren(storage.GetRootId(), v);
  TEST_EQUAL(v, TCountriesVec({kGroupCountryId}), ());
  v.clear();

  // Check all group children are downloaded
  storage.GetDownloadedChildren(kGroupCountryId, v);
  TEST_EQUAL(children, TCountriesSet(v.begin(), v.end()), ());
  v.clear();

  storage.Unsubscribe(subsrcibtionId);
}

void DeleteGroup(Storage & storage, bool oneByOne)
{
  Platform & platform = GetPlatform();

  string const version = strings::to_string(storage.GetCurrentDataVersion());

  //  Get children nodes for the group node
  TCountriesVec v;
  storage.GetChildren(kGroupCountryId, v);
  TCountriesSet const children(v.begin(), v.end());
  v.clear();

  // Check group node is downloaded
  storage.GetDownloadedChildren(storage.GetRootId(), v);
  TEST_EQUAL(v, TCountriesVec({kGroupCountryId}), ());
  v.clear();

  // Check children nodes are downloaded
  storage.GetDownloadedChildren(kGroupCountryId, v);
  TEST_EQUAL(children, TCountriesSet(v.begin(), v.end()), ());
  v.clear();

  // Check there are mwm files for the children nodes
  for (auto const & countryId : children)
  {
    string const mwmFullPath = GetMwmFilePath(version, countryId);
    TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
  }

  // Delete the group
  if (oneByOne)
  {
    for (auto const & countryId : children)
      storage.DeleteNode(countryId);
  }
  else
  {
    storage.DeleteNode(kGroupCountryId);
  }

  // Check state for the group node is set to UpToDate and NoError
  NodeAttrs attrs;
  storage.GetNodeAttrs(kGroupCountryId, attrs);
  TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());

  // Check state for the all children nodes is set to UpToDate and NoError
  for (auto const & countryId : children)
  {
    TEST_EQUAL(Status::ENotDownloaded, storage.CountryStatusEx(countryId), ());
    NodeAttrs attrs;
    storage.GetNodeAttrs(countryId, attrs);
    TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
  }

  // Check there are no mwm files for the children nodes
  for (auto const & countryId : children)
  {
    string const mwmFullPath = GetMwmFilePath(version, countryId);
    TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
  }

  // Check group is not downloaded
  storage.GetDownloadedChildren(storage.GetRootId(), v);
  TEST(v.empty(), ());

  // Check all children nodes are not downloaded
  storage.GetDownloadedChildren(kGroupCountryId, v);
  TEST(v.empty(), ());
}

void TestDownloadDelete(bool downloadOneByOne, bool deleteOneByOne)
{
  WritableDirChanger writableDirChanger(kMapTestDir);

  Storage storage(COUNTRIES_MIGRATE_FILE);

  TEST(version::IsSingleMwm(storage.GetCurrentDataVersion()), ());
  string const version = strings::to_string(storage.GetCurrentDataVersion());

  auto onUpdatedFn = [&](LocalCountryFile const & localCountryFile)
  {
    TCountryId const countryId = localCountryFile.GetCountryName();
    TEST(kLeafCountriesIds.find(countryId) != kLeafCountriesIds.end(), ());
  };

  storage.Init(onUpdatedFn);
  storage.RegisterAllLocalMaps();
  storage.SetDownloadingUrlsForTesting({kTestWebServer});

  tests_support::ScopedDir cleanupVersionDir(version);

  // Check children for the kGroupCountryId
  TCountriesVec children;
  storage.GetChildren(kGroupCountryId, children);
  TEST_EQUAL(TCountriesSet(children.begin(), children.end()), kLeafCountriesIds, ());

  DownloadGroup(storage, downloadOneByOne);

  DeleteGroup(storage, deleteOneByOne);
}

} // namespace

UNIT_TEST(SmallMwms_GroupDownloadDelete_Test1)
{
  TestDownloadDelete(false, false);
}

UNIT_TEST(SmallMwms_GroupDownloadDelete_Test2)
{
  TestDownloadDelete(false, true);
}

UNIT_TEST(SmallMwms_GroupDownloadDelete_Test3)
{
  TestDownloadDelete(true, false);
}

UNIT_TEST(SmallMwms_GroupDownloadDelete_Test4)
{
  TestDownloadDelete(true, true);
}
<commit_msg>[new downloader] Fix comments to describe how the function actually works<commit_after>#include "testing/testing.hpp"

#include "map/framework.hpp"

#include "platform/http_request.hpp"
#include "platform/local_country_file_utils.hpp"
#include "platform/platform.hpp"
#include "platform/platform_tests_support/scoped_dir.hpp"
#include "platform/platform_tests_support/write_dir_changer.hpp"

#include "coding/file_name_utils.hpp"

#include "storage/storage.hpp"

#include "base/assert.hpp"

#include "std/condition_variable.hpp"
#include "std/mutex.hpp"
#include "std/set.hpp"
#include "std/unique_ptr.hpp"

using namespace platform;
using namespace storage;

namespace
{

string const kTestWebServer = "http://new-search.mapswithme.com/";

string const kMapTestDir = "map-tests";

TCountryId const kGroupCountryId = "New Zealand";
TCountriesSet const kLeafCountriesIds = {"Tokelau",
                                         "New Zealand North_Auckland",
                                         "New Zealand North_Wellington",
                                         "New Zealand South_Canterbury",
                                         "New Zealand South_Southland"};

string GetMwmFilePath(string const & version, TCountryId const & countryId)
{
  return my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
                               countryId + DATA_FILE_EXTENSION);
}

string GetMwmDownloadingFilePath(string const & version, TCountryId const & countryId)
{
  return my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
                               countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION DOWNLOADING_FILE_EXTENSION);
}

string GetMwmResumeFilePath(string const & version, TCountryId const & countryId)
{
  return my::JoinFoldersToPath({GetPlatform().WritableDir(), version},
                               countryId + DATA_FILE_EXTENSION READY_FILE_EXTENSION RESUME_FILE_EXTENSION);
}

void DownloadGroup(Storage & storage, bool oneByOne)
{
  Platform & platform = GetPlatform();

  string const version = strings::to_string(storage.GetCurrentDataVersion());

  //  Get children nodes for the group node
  TCountriesVec v;
  storage.GetChildren(kGroupCountryId, v);
  TCountriesSet const children(v.begin(), v.end());
  v.clear();

  TCountriesSet changed;
  auto onChangeCountryFn = [&](TCountryId const & countryId)
  {
    TEST(children.find(countryId) != children.end(), ());
    changed.insert(countryId);
    if (!storage.IsDownloadInProgress())
    {
      // end waiting when all chilren will be downloaded
      testing::StopEventLoop();
    }
  };

  TCountriesSet downloaded;
  auto onProgressFn = [&](TCountryId const & countryId, TLocalAndRemoteSize const & mapSize)
  {
    TEST(children.find(countryId) != children.end(), ());
    if (mapSize.first == mapSize.second)
    {
      auto const res = downloaded.insert(countryId);
      TEST_EQUAL(res.second, true, ()); // every child is downloaded only once
    }
  };

  int const subsrcibtionId = storage.Subscribe(onChangeCountryFn, onProgressFn);

  // Check group node is not downloaded
  storage.GetDownloadedChildren(storage.GetRootId(), v);
  TEST(v.empty(), ());

  // Check children nodes are not downloaded
  storage.GetDownloadedChildren(kGroupCountryId, v);
  TEST(v.empty(), ());

  // Check status for the all children nodes is set to ENotDownloaded
  size_t totalGroupSize = 0;
  for (auto const & countryId : children)
  {
    TEST_EQUAL(Status::ENotDownloaded, storage.CountryStatusEx(countryId), ());
    NodeAttrs attrs;
    storage.GetNodeAttrs(countryId, attrs);
    TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
    TEST_GREATER(attrs.m_mwmSize, 0, ());
    totalGroupSize += attrs.m_mwmSize;
  }

  // Check status for the group node is set to ENotDownloaded
  NodeAttrs attrs;
  storage.GetNodeAttrs(kGroupCountryId, attrs);
  TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
  TEST_EQUAL(attrs.m_mwmSize, totalGroupSize, ());
  attrs = NodeAttrs();

  // Check there is no mwm or any other files for the children nodes
  for (auto const & countryId : children)
  {
    string const mwmFullPath = GetMwmFilePath(version, countryId);
    string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId);
    string const resumeFullPath = GetMwmResumeFilePath(version, countryId);
    TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
    TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
    TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
  }

  // Download the group
  if (oneByOne)
  {
    for (auto const & countryId : children)
      storage.DownloadNode(countryId);
  }
  else
  {
    storage.DownloadNode(kGroupCountryId);
  }
  // wait for downloading of all children
  testing::RunEventLoop();

  // Check all children nodes have been downloaded and changed.
  TEST_EQUAL(changed, children, ());
  TEST_EQUAL(downloaded, children, ());

  // Check status for the group node is set to EOnDisk
  storage.GetNodeAttrs(kGroupCountryId, attrs);
  TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());

  // Check status for the all children nodes is set to EOnDisk
  for (auto const & countryId : children)
  {
    TEST_EQUAL(Status::EOnDisk, storage.CountryStatusEx(countryId), ());
    NodeAttrs attrs;
    storage.GetNodeAttrs(countryId, attrs);
    TEST_EQUAL(NodeStatus::OnDisk, attrs.m_status, ());
  }

  // Check there is only mwm files are present and no any other for the children nodes
  for (auto const & countryId : children)
  {
    string const mwmFullPath = GetMwmFilePath(version, countryId);
    string const downloadingFullPath = GetMwmDownloadingFilePath(version, countryId);
    string const resumeFullPath = GetMwmResumeFilePath(version, countryId);
    TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
    TEST(!platform.IsFileExistsByFullPath(downloadingFullPath), ());
    TEST(!platform.IsFileExistsByFullPath(resumeFullPath), ());
  }

  // Check group is downloaded
  storage.GetDownloadedChildren(storage.GetRootId(), v);
  TEST_EQUAL(v, TCountriesVec({kGroupCountryId}), ());
  v.clear();

  // Check all group children are downloaded
  storage.GetDownloadedChildren(kGroupCountryId, v);
  TEST_EQUAL(children, TCountriesSet(v.begin(), v.end()), ());
  v.clear();

  storage.Unsubscribe(subsrcibtionId);
}

void DeleteGroup(Storage & storage, bool oneByOne)
{
  Platform & platform = GetPlatform();

  string const version = strings::to_string(storage.GetCurrentDataVersion());

  //  Get children nodes for the group node
  TCountriesVec v;
  storage.GetChildren(kGroupCountryId, v);
  TCountriesSet const children(v.begin(), v.end());
  v.clear();

  // Check group node is downloaded
  storage.GetDownloadedChildren(storage.GetRootId(), v);
  TEST_EQUAL(v, TCountriesVec({kGroupCountryId}), ());
  v.clear();

  // Check children nodes are downloaded
  storage.GetDownloadedChildren(kGroupCountryId, v);
  TEST_EQUAL(children, TCountriesSet(v.begin(), v.end()), ());
  v.clear();

  // Check there are mwm files for the children nodes
  for (auto const & countryId : children)
  {
    string const mwmFullPath = GetMwmFilePath(version, countryId);
    TEST(platform.IsFileExistsByFullPath(mwmFullPath), ());
  }

  // Delete the group
  if (oneByOne)
  {
    for (auto const & countryId : children)
      storage.DeleteNode(countryId);
  }
  else
  {
    storage.DeleteNode(kGroupCountryId);
  }

  // Check state for the group node is set to NotDownloaded and NoError
  NodeAttrs attrs;
  storage.GetNodeAttrs(kGroupCountryId, attrs);
  TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());

  // Check state for the all children nodes is set to NotDownloaded and NoError
  for (auto const & countryId : children)
  {
    TEST_EQUAL(Status::ENotDownloaded, storage.CountryStatusEx(countryId), ());
    NodeAttrs attrs;
    storage.GetNodeAttrs(countryId, attrs);
    TEST_EQUAL(NodeStatus::NotDownloaded, attrs.m_status, ());
  }

  // Check there are no mwm files for the children nodes
  for (auto const & countryId : children)
  {
    string const mwmFullPath = GetMwmFilePath(version, countryId);
    TEST(!platform.IsFileExistsByFullPath(mwmFullPath), ());
  }

  // Check group is not downloaded
  storage.GetDownloadedChildren(storage.GetRootId(), v);
  TEST(v.empty(), ());

  // Check all children nodes are not downloaded
  storage.GetDownloadedChildren(kGroupCountryId, v);
  TEST(v.empty(), ());
}

void TestDownloadDelete(bool downloadOneByOne, bool deleteOneByOne)
{
  WritableDirChanger writableDirChanger(kMapTestDir);

  Storage storage(COUNTRIES_MIGRATE_FILE);

  TEST(version::IsSingleMwm(storage.GetCurrentDataVersion()), ());
  string const version = strings::to_string(storage.GetCurrentDataVersion());

  auto onUpdatedFn = [&](LocalCountryFile const & localCountryFile)
  {
    TCountryId const countryId = localCountryFile.GetCountryName();
    TEST(kLeafCountriesIds.find(countryId) != kLeafCountriesIds.end(), ());
  };

  storage.Init(onUpdatedFn);
  storage.RegisterAllLocalMaps();
  storage.SetDownloadingUrlsForTesting({kTestWebServer});

  tests_support::ScopedDir cleanupVersionDir(version);

  // Check children for the kGroupCountryId
  TCountriesVec children;
  storage.GetChildren(kGroupCountryId, children);
  TEST_EQUAL(TCountriesSet(children.begin(), children.end()), kLeafCountriesIds, ());

  DownloadGroup(storage, downloadOneByOne);

  DeleteGroup(storage, deleteOneByOne);
}

} // namespace

UNIT_TEST(SmallMwms_GroupDownloadDelete_Test1)
{
  TestDownloadDelete(false, false);
}

UNIT_TEST(SmallMwms_GroupDownloadDelete_Test2)
{
  TestDownloadDelete(false, true);
}

UNIT_TEST(SmallMwms_GroupDownloadDelete_Test3)
{
  TestDownloadDelete(true, false);
}

UNIT_TEST(SmallMwms_GroupDownloadDelete_Test4)
{
  TestDownloadDelete(true, true);
}
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
#include <string>

using namespace std;

//Function to count the lines of the script to be converted
int linecounter(string dir){
    int counter = 0;

    string line;

    //Create new file stream to file to be converted
    ifstream file(dir);

    while (getline(file, line)){
        counter++;}

    return counter;
}

//Accepts a string and returns the value of the key(s) in digispark language
//Some of the keys are encoded using their usage IDs. Refer to http://www.usb.org/developers/hidpage/Hut1_12v2.pdf  (Page 53-59)
string stringprocessor(string key){

    if((key == "A") || (key == "a")){
        key = "KEY_A";
        return key;
    }

    else if((key == "B") || (key == "b")){
        key = "KEY_B";
        return key;
    }

    else if((key == "C") || (key == "c")){
        key = "KEY_C";
        return key;
    }

    else if((key == "D") || (key == "d")){
        key = "KEY_D";
        return key;
    }

    else if((key == "E") || (key == "e")){
        key = "KEY_E";
        return key;
    }

    else if((key == "F") || (key == "f")){
        key = "KEY_F";
        return key;
    }

    else if((key == "G") || (key == "g")){
        key = "KEY_G";
        return key;
    }

    else if((key == "H") || (key == "h")){
        key = "KEY_H";
        return key;
    }

    else if((key == "I") || (key == "i")){
        key = "KEY_I";
        return key;
    }

    else if((key == "J") || (key == "j")){
        key = "KEY_J";
        return key;
    }

    else if((key == "K") || (key == "k")){
        key = "KEY_K";
        return key;
    }

    else if((key == "L") || (key == "l")){
        key = "KEY_L";
        return key;
    }

    else if((key == "M") || (key == "m")){
        key = "KEY_M";
        return key;
    }

    else if((key == "N") || (key == "n")){
        key = "KEY_N";
        return key;
    }

    else if((key == "O") || (key == "o")){
        key = "KEY_O";
        return key;
    }

    else if((key == "P") || (key == "p")){
        key = "KEY_P";
        return key;
    }

    else if((key == "Q") || (key == "q")){
        key = "KEY_Q";
        return key;
    }

    else if((key == "R") || (key == "r")){
        key = "KEY_R";
        return key;
    }

    else if((key == "S") || (key == "s")){
        key = "KEY_S";
        return key;
    }

    else if((key == "T") || (key == "t")){
        key = "KEY_T";
        return key;
    }

    else if((key == "U") || (key == "u")){
        key = "KEY_U";
        return key;
    }

    else if((key == "V") || (key == "v")){
        key = "KEY_V";
        return key;
    }

    else if((key == "W") || (key == "w")){
        key = "KEY_W";
        return key;
    }

    else if((key == "X") || (key == "x")){
        key = "KEY_X";
        return key;
    }

    else if((key == "Y") || (key == "y")){
        key = "KEY_Y";
        return key;
    }

    else if((key == "Z") || (key == "z")){
        key = "KEY_Z";
        return key;
    }

    else if(key=="1"){
        key = "KEY_1";
        return key;
    }

    else if(key=="2"){
        key = "KEY_2";
        return key;
    }

    else if(key=="3"){
        key = "KEY_3";
        return key;
    }

    else if(key=="4"){
        key = "KEY_4";
        return key;
    }

    else if(key=="5"){
        key = "KEY_5";
        return key;
    }

    else if(key=="6"){
        key = "KEY_6";
        return key;
    }

    else if(key=="7"){
        key = "KEY_7";
        return key;
    }

    else if(key=="8"){
        key = "KEY_8";
        return key;
    }

    else if(key=="9"){
        key = "KEY_9";
        return key;
    }

    else if(key=="0"){
        key = "KEY_0";
        return key;
    }

    else if(key=="ENTER"){
        key = "KEY_ENTER";
        return key;
    }

    else if(key=="F1"){
        key = "KEY_F1";
        return key;
    }

    else if(key=="F2"){
        key = "KEY_F2";
        return key;
    }

    else if(key=="F3"){
        key = "KEY_F3";
        return key;
    }

    else if(key=="F4"){
        key = "KEY_F4";
        return key;
    }

    else if(key=="F5"){
        key = "KEY_F5";
        return key;
    }

    else if(key=="F6"){
        key = "KEY_F6";
        return key;
    }

    else if(key=="F7"){
        key = "KEY_F7";
        return key;
    }

    else if(key=="F8"){
        key = "KEY_F8";
        return key;
    }

    else if(key=="F9"){
        key = "KEY_F9";
        return key;
    }

    else if(key=="F10"){
        key = "KEY_F10";
        return key;
    }

    else if(key=="F11"){
        key = "KEY_F11";
        return key;
    }

    else if(key=="F12"){
        key = "KEY_F12";
        return key;
    }

    else if((key=="GUI") || (key=="WINDOWS")){
        key = "0, MOD_GUI_LEFT"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="APP") || (key=="MENU")){
        key = "101"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if(key=="SHIFT"){
        key = "MOD_SHIFT_LEFT";
        return key;
    }

    else if(key=="ALT"){
        key = "MOD_ALT_LEFT";
        return key;
    }

    else if((key=="CONTROL") || (key=="CTRL")){
        key = "MOD_CONTROL_LEFT"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="DOWNARROW") || (key=="DOWN")){
        key = "84"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="UPARROW") || (key=="UP")){
        key = "83"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="LEFTARROW") || (key=="LEFT")){
        key = "79"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="RIGHTARROW") || (key=="RIGHT")){
        key = "89"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="BREAK") || (key=="PAUSE")){
        key = "126"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if(key=="CAPSLOCK"){
        key = "30";
        return key;
    }

    else if((key=="ESC") || (key=="ESCAPE")){
        key = "110";
        return key;
    }

    else if(key=="DELETE"){
        key = "15";
        return key;
    }

    else if(key=="END"){
        key = "81";
        return key;
    }

    else if(key=="HOME"){
        key = "80";
        return key;
    }

    else if(key=="NUMLOCK"){
        key = "90";
        return key;
    }

    else if(key=="PAGEUP"){
        key = "85";
        return key;
    }

    else if(key=="PAGEDOWN"){
        key = "86";
        return key;
    }

    else if(key=="PRINTSCREEN"){
        key = "124";
        return key;
    }

    else if(key=="SCROLLLOCK"){
        key = "125";
        return key;
    }

    else if(key=="SPACE"){
        key = "61";
        return key;
    }

    else if(key=="TAB"){
        key = "16";
        return key;
    }
}

int main(){

    //Eye-candy
    cout<<"  _      _      _"<<endl;
    cout<<">(.)__ <(.)__ =(.)__"<<endl;
    cout<<" (___/  (___/  (___/"<<endl;
    cout<<endl;
    cout<<endl;
    cout<<"Welcome to digiQuack, the easy DuckyScript to Digispark payload converter!"<<endl;

    //Ask user to enter path to file to be converted
    string dir;
    cout<<"Enter the path of the file to be converted:"<<endl;
    cin>>dir;

    //Open input file stream
    ifstream filein(dir);

    //Check if file stream opened - if file exists and is a valid input
    if(!filein){
        cout<<"File not found or invalid input."<<endl;
        return -1;
    }

    //Check if the input file is empty
    if ( filein.peek() == std::ifstream::traits_type::eof() )
    {
        cout<<"File supplied is empty"<<endl;
        return -1;

    }

    //Create a counter as big as the number of lines in the input file
    int counter = linecounter(dir);

    //Open converted file fstream and create text file
    ofstream fileout;
    fileout.open ("converted.txt");

    //Create a string variable to store each individual ducky command/line to process in the for loop
    string command;

    //Create a trigger for the default delay command - if default delay command encounter, inject converted script with delay lines
    int delay = 0;

    //Insert this line in the beginning of the converted script to make sure that (on older systems) the first keystroke after a delay isn't missed
    fileout << "DigiKeyboard.sendKeyStroke(0);\n";

    //Create temporary string variables to use for string manipulation in the following for-loop
    string tmpstring;
    string tmpstring2;
    string fincom; //Use this variable to keep track of the last converted command to use in the repeat function
    string key; //Variable to be used in the keystroke conversion if clause underneath
    int replays = 0; //Replays counter for the REPLAY feature if clause underneath

    //Open a for loop to process each line/command of the input script
    for(int i=0; i<counter; i++){

        //Set command string equal to current line
        getline(filein, command);

        //Check for DEFAULT_DELAY or DEFAULTDELAY, set the delay variable and re-examine for-loop (Note: writing the delay commands happens at the end of the for-loop).
        tmpstring = tmpstring.assign(command,0,13);
        if((tmpstring.compare("DEFAULTDELAY ")==0) || (tmpstring.compare("DEFAULT_DELAY")==0)){
            command.erase(0,13);
            delay = stoi(command);
            continue;
        }

        //Convert duckyscript comments to Arduino IDE comments: Extract 3 first characters, if == REM, replace them with //, write to output and re-examine for-loop - this helps avoid placing unnecessary delay commands
        tmpstring = tmpstring.assign(command,0,3);
        if(tmpstring.compare("REM") == 0){
            command.replace(0,4,"//");
            fileout << command << "\n";
            continue;
        }

        //Convert duckyscript STRING commands to digispark print commands (Note, used F to store large strings to flash since the digispark has few RAM)
        tmpstring = tmpstring.assign(command,0,6);
        if(tmpstring.compare("STRING") == 0){
            command.replace(0,7,"DigiKeyboard.print(F(\"");
            command += "\"));";
            fincom = command; //See use of fincom above
            fileout << command << "\n";
            command = ""; //Empty the string so it won't trigger the keystroke processor
        }

        //Convert DELAY command (Not DEFAULTDELAY)
        tmpstring = tmpstring.assign(command,0,5);
        if(tmpstring.compare("DELAY") == 0){
            command.erase(0,6);
            fincom = "DigiKeyboard.delay(" + command + ");";
            fileout << fincom << "\n";
            command = ""; //command variable is set to null to bypass keystroke processor but still access default delay
        }

        //REPLAY feature (Note: at the end, command variable is set to null to bypass keystroke processor but still access default delay)
        tmpstring = tmpstring.assign(command,0,5);
        if(tmpstring.compare("REPLAY") == 0){
            command.erase(0,6);
            replays = stoi(command);
            for(int j=0; j < replays; j++){
                fileout << fincom << "\n";
            }
            command = "";
        }

        //Convert keystroke commands by calling the string processor function
        if(!command.empty()){
            key = command.substr(0, command.find(' '));
            tmpstring = "DigiKeyboard.sendKeyStroke(" + stringprocessor(key);
            command.erase(0, command.find(' ')+1);
            if(!command.empty()){
                tmpstring = tmpstring + ",";
                tmpstring = tmpstring + stringprocessor(command);
            }
            tmpstring = tmpstring + ");";
            fincom = tmpstring;
            fileout << fincom << "\n";
        }

        //If default delay trigger is activated, inject output file with a delay command
        if(delay!=0){
            fileout << "DigiKeyboard.delay(" << delay << ");\n";
        }
    }

    //Ask if user wants to execute the script just once - Use the counter variable instead of initialising a new one since counter won't be used anymore.
    cout << "Script has been converted successfully. Do you want the DigiSpark to execute the script once or loop infinitely? Select 0 for infinity and 1 for a single execution." << endl;
    cout << "[0/1]:";
    cin >> counter;

    if(counter){
        fileout << "exit();";
    }

    //Close converted script txt fstream
    fileout.close();
    return 0;
}
<commit_msg>Fixed typo and removed a blank line<commit_after>#include <iostream>
#include <fstream>
#include <string>

using namespace std;

//Function to count the lines of the script to be converted
int linecounter(string dir){
    int counter = 0;

    string line;

    //Create new file stream to file to be converted
    ifstream file(dir);

    while (getline(file, line)){
        counter++;}

    return counter;
}

//Accepts a string and returns the value of the key(s) in digispark language
//Some of the keys are encoded using their usage IDs. Refer to http://www.usb.org/developers/hidpage/Hut1_12v2.pdf  (Page 53-59)
string stringprocessor(string key){

    if((key == "A") || (key == "a")){
        key = "KEY_A";
        return key;
    }

    else if((key == "B") || (key == "b")){
        key = "KEY_B";
        return key;
    }

    else if((key == "C") || (key == "c")){
        key = "KEY_C";
        return key;
    }

    else if((key == "D") || (key == "d")){
        key = "KEY_D";
        return key;
    }

    else if((key == "E") || (key == "e")){
        key = "KEY_E";
        return key;
    }

    else if((key == "F") || (key == "f")){
        key = "KEY_F";
        return key;
    }

    else if((key == "G") || (key == "g")){
        key = "KEY_G";
        return key;
    }

    else if((key == "H") || (key == "h")){
        key = "KEY_H";
        return key;
    }

    else if((key == "I") || (key == "i")){
        key = "KEY_I";
        return key;
    }

    else if((key == "J") || (key == "j")){
        key = "KEY_J";
        return key;
    }

    else if((key == "K") || (key == "k")){
        key = "KEY_K";
        return key;
    }

    else if((key == "L") || (key == "l")){
        key = "KEY_L";
        return key;
    }

    else if((key == "M") || (key == "m")){
        key = "KEY_M";
        return key;
    }

    else if((key == "N") || (key == "n")){
        key = "KEY_N";
        return key;
    }

    else if((key == "O") || (key == "o")){
        key = "KEY_O";
        return key;
    }

    else if((key == "P") || (key == "p")){
        key = "KEY_P";
        return key;
    }

    else if((key == "Q") || (key == "q")){
        key = "KEY_Q";
        return key;
    }

    else if((key == "R") || (key == "r")){
        key = "KEY_R";
        return key;
    }

    else if((key == "S") || (key == "s")){
        key = "KEY_S";
        return key;
    }

    else if((key == "T") || (key == "t")){
        key = "KEY_T";
        return key;
    }

    else if((key == "U") || (key == "u")){
        key = "KEY_U";
        return key;
    }

    else if((key == "V") || (key == "v")){
        key = "KEY_V";
        return key;
    }

    else if((key == "W") || (key == "w")){
        key = "KEY_W";
        return key;
    }

    else if((key == "X") || (key == "x")){
        key = "KEY_X";
        return key;
    }

    else if((key == "Y") || (key == "y")){
        key = "KEY_Y";
        return key;
    }

    else if((key == "Z") || (key == "z")){
        key = "KEY_Z";
        return key;
    }

    else if(key=="1"){
        key = "KEY_1";
        return key;
    }

    else if(key=="2"){
        key = "KEY_2";
        return key;
    }

    else if(key=="3"){
        key = "KEY_3";
        return key;
    }

    else if(key=="4"){
        key = "KEY_4";
        return key;
    }

    else if(key=="5"){
        key = "KEY_5";
        return key;
    }

    else if(key=="6"){
        key = "KEY_6";
        return key;
    }

    else if(key=="7"){
        key = "KEY_7";
        return key;
    }

    else if(key=="8"){
        key = "KEY_8";
        return key;
    }

    else if(key=="9"){
        key = "KEY_9";
        return key;
    }

    else if(key=="0"){
        key = "KEY_0";
        return key;
    }

    else if(key=="ENTER"){
        key = "KEY_ENTER";
        return key;
    }

    else if(key=="F1"){
        key = "KEY_F1";
        return key;
    }

    else if(key=="F2"){
        key = "KEY_F2";
        return key;
    }

    else if(key=="F3"){
        key = "KEY_F3";
        return key;
    }

    else if(key=="F4"){
        key = "KEY_F4";
        return key;
    }

    else if(key=="F5"){
        key = "KEY_F5";
        return key;
    }

    else if(key=="F6"){
        key = "KEY_F6";
        return key;
    }

    else if(key=="F7"){
        key = "KEY_F7";
        return key;
    }

    else if(key=="F8"){
        key = "KEY_F8";
        return key;
    }

    else if(key=="F9"){
        key = "KEY_F9";
        return key;
    }

    else if(key=="F10"){
        key = "KEY_F10";
        return key;
    }

    else if(key=="F11"){
        key = "KEY_F11";
        return key;
    }

    else if(key=="F12"){
        key = "KEY_F12";
        return key;
    }

    else if((key=="GUI") || (key=="WINDOWS")){
        key = "0, MOD_GUI_LEFT"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="APP") || (key=="MENU")){
        key = "101"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if(key=="SHIFT"){
        key = "MOD_SHIFT_LEFT";
        return key;
    }

    else if(key=="ALT"){
        key = "MOD_ALT_LEFT";
        return key;
    }

    else if((key=="CONTROL") || (key=="CTRL")){
        key = "MOD_CONTROL_LEFT"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="DOWNARROW") || (key=="DOWN")){
        key = "84"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="UPARROW") || (key=="UP")){
        key = "83"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="LEFTARROW") || (key=="LEFT")){
        key = "79"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="RIGHTARROW") || (key=="RIGHT")){
        key = "89"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if((key=="BREAK") || (key=="PAUSE")){
        key = "126"; //Doesn't work without the 0, for some reason
        return key;
    }

    else if(key=="CAPSLOCK"){
        key = "30";
        return key;
    }

    else if((key=="ESC") || (key=="ESCAPE")){
        key = "110";
        return key;
    }

    else if(key=="DELETE"){
        key = "15";
        return key;
    }

    else if(key=="END"){
        key = "81";
        return key;
    }

    else if(key=="HOME"){
        key = "80";
        return key;
    }

    else if(key=="NUMLOCK"){
        key = "90";
        return key;
    }

    else if(key=="PAGEUP"){
        key = "85";
        return key;
    }

    else if(key=="PAGEDOWN"){
        key = "86";
        return key;
    }

    else if(key=="PRINTSCREEN"){
        key = "124";
        return key;
    }

    else if(key=="SCROLLLOCK"){
        key = "125";
        return key;
    }

    else if(key=="SPACE"){
        key = "61";
        return key;
    }

    else if(key=="TAB"){
        key = "16";
        return key;
    }
}

int main(){

    //Eye-candy
    cout<<"  _      _      _"<<endl;
    cout<<">(.)__ <(.)__ =(.)__"<<endl;
    cout<<" (___/  (___/  (___/"<<endl;
    cout<<endl;
    cout<<endl;
    cout<<"Welcome to digiQuack, the easy DuckyScript to Digispark payload converter!"<<endl;

    //Ask user to enter path to file to be converted
    string dir;
    cout<<"Enter the path of the file to be converted:"<<endl;
    cin>>dir;

    //Open input file stream
    ifstream filein(dir);

    //Check if file stream opened - if file exists and is a valid input
    if(!filein){
        cout<<"File not found or invalid input."<<endl;
        return -1;
    }

    //Check if the input file is empty
    if ( filein.peek() == std::ifstream::traits_type::eof() )
    {
        cout<<"File supplied is empty"<<endl;
        return -1;

    }

    //Create a counter as big as the number of lines in the input file
    int counter = linecounter(dir);

    //Open converted file fstream and create text file
    ofstream fileout;
    fileout.open ("converted.txt");

    //Create a string variable to store each individual ducky command/line to process in the for loop
    string command;

    //Create a trigger for the default delay command - if default delay command is encountered, inject converted script with delay lines
    int delay = 0;

    //Insert this line in the beginning of the converted script to make sure that (on older systems) the first keystroke after a delay isn't missed
    fileout << "DigiKeyboard.sendKeyStroke(0);\n";

    //Create temporary string variables to use for string manipulation in the following for-loop
    string tmpstring;
    string tmpstring2;
    string fincom; //Use this variable to keep track of the last converted command to use in the repeat function
    string key; //Variable to be used in the keystroke conversion if clause underneath
    int replays = 0; //Replays counter for the REPLAY feature if clause underneath

    //Open a for loop to process each line/command of the input script
    for(int i=0; i<counter; i++){

        //Set command string equal to current line
        getline(filein, command);

        //Check for DEFAULT_DELAY or DEFAULTDELAY, set the delay variable and re-examine for-loop (Note: writing the delay commands happens at the end of the for-loop).
        tmpstring = tmpstring.assign(command,0,13);
        if((tmpstring.compare("DEFAULTDELAY ")==0) || (tmpstring.compare("DEFAULT_DELAY")==0)){
            command.erase(0,13);
            delay = stoi(command);
            continue;
        }

        //Convert duckyscript comments to Arduino IDE comments: Extract 3 first characters, if == REM, replace them with //, write to output and re-examine for-loop - this helps avoid placing unnecessary delay commands
        tmpstring = tmpstring.assign(command,0,3);
        if(tmpstring.compare("REM") == 0){
            command.replace(0,4,"//");
            fileout << command << "\n";
            continue;
        }

        //Convert duckyscript STRING commands to digispark print commands (Note, used F to store large strings to flash since the digispark has few RAM)
        tmpstring = tmpstring.assign(command,0,6);
        if(tmpstring.compare("STRING") == 0){
            command.replace(0,7,"DigiKeyboard.print(F(\"");
            command += "\"));";
            fincom = command; //See use of fincom above
            fileout << command << "\n";
            command = ""; //Empty the string so it won't trigger the keystroke processor
        }

        //Convert DELAY command (Not DEFAULTDELAY)
        tmpstring = tmpstring.assign(command,0,5);
        if(tmpstring.compare("DELAY") == 0){
            command.erase(0,6);
            fincom = "DigiKeyboard.delay(" + command + ");";
            fileout << fincom << "\n";
            command = ""; //command variable is set to null to bypass keystroke processor but still access default delay
        }

        //REPLAY feature (Note: at the end, command variable is set to null to bypass keystroke processor but still access default delay)
        tmpstring = tmpstring.assign(command,0,5);
        if(tmpstring.compare("REPLAY") == 0){
            command.erase(0,6);
            replays = stoi(command);
            for(int j=0; j < replays; j++){
                fileout << fincom << "\n";
            }
            command = "";
        }

        //Convert keystroke commands by calling the string processor function
        if(!command.empty()){
            key = command.substr(0, command.find(' '));
            tmpstring = "DigiKeyboard.sendKeyStroke(" + stringprocessor(key);
            command.erase(0, command.find(' ')+1);
            if(!command.empty()){
                tmpstring = tmpstring + ",";
                tmpstring = tmpstring + stringprocessor(command);
            }
            tmpstring = tmpstring + ");";
            fincom = tmpstring;
            fileout << fincom << "\n";
        }

        //If default delay trigger is activated, inject output file with a delay command
        if(delay!=0){
            fileout << "DigiKeyboard.delay(" << delay << ");\n";
        }
    }

    //Ask if user wants to execute the script just once - Use the counter variable instead of initialising a new one since counter won't be used anymore.
    cout << "Script has been converted successfully. Do you want the DigiSpark to execute the script once or loop infinitely? Select 0 for infinity and 1 for a single execution." << endl;
    cout << "[0/1]:";
    cin >> counter;

    if(counter){
        fileout << "exit();";
    }

    //Close converted script txt fstream
    fileout.close();
    return 0;
}
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: XMLRangeHelper.hxx,v $
 *
 *  $Revision: 1.2 $
 *
 *  last change: $Author: vg $ $Date: 2007-05-22 18:25:05 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/
#ifndef CHART2_XMLRANGEHELPER_HXX
#define CHART2_XMLRANGEHELPER_HXX

#ifndef _SAL_TYPES_H_
#include <sal/types.h>
#endif

#ifndef _RTL_USTRING_HXX_
#include <rtl/ustring.hxx>
#endif

namespace chart
{
namespace XMLRangeHelper
{

struct Cell
{
    sal_Int32 nColumn;
    sal_Int32 nRow;
    bool bRelativeColumn;
    bool bRelativeRow;
    bool bIsEmpty;

    Cell() :
            nColumn(0),
            nRow(0),
            bRelativeColumn(false),
            bRelativeRow(false),
            bIsEmpty(true)
    {}

    inline bool empty() const { return bIsEmpty; }
};

struct CellRange
{
    Cell aUpperLeft;
    Cell aLowerRight;
    ::rtl::OUString aTableName;
};

CellRange getCellRangeFromXMLString( const ::rtl::OUString & rXMLString );

::rtl::OUString getXMLStringFromCellRange( const CellRange & rRange );

} //  namespace XMLRangeHelper
} //  namespace chart

// CHART2_XMLRANGEHELPER_HXX
#endif
<commit_msg>INTEGRATION: CWS changefileheader (1.2.126); FILE MERGED 2008/04/01 15:04:14 thb 1.2.126.2: #i85898# Stripping all external header guards 2008/03/28 16:43:58 rt 1.2.126.1: #i87441# Change license header to LPGL v3.<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: XMLRangeHelper.hxx,v $
 * $Revision: 1.3 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/
#ifndef CHART2_XMLRANGEHELPER_HXX
#define CHART2_XMLRANGEHELPER_HXX

#include <sal/types.h>
#include <rtl/ustring.hxx>

namespace chart
{
namespace XMLRangeHelper
{

struct Cell
{
    sal_Int32 nColumn;
    sal_Int32 nRow;
    bool bRelativeColumn;
    bool bRelativeRow;
    bool bIsEmpty;

    Cell() :
            nColumn(0),
            nRow(0),
            bRelativeColumn(false),
            bRelativeRow(false),
            bIsEmpty(true)
    {}

    inline bool empty() const { return bIsEmpty; }
};

struct CellRange
{
    Cell aUpperLeft;
    Cell aLowerRight;
    ::rtl::OUString aTableName;
};

CellRange getCellRangeFromXMLString( const ::rtl::OUString & rXMLString );

::rtl::OUString getXMLStringFromCellRange( const CellRange & rRange );

} //  namespace XMLRangeHelper
} //  namespace chart

// CHART2_XMLRANGEHELPER_HXX
#endif
<|endoftext|>