<commit_before>#if ENABLE_S3

#include "s3.hh"
#include "s3-binary-cache-store.hh"
#include "nar-info.hh"
#include "nar-info-disk-cache.hh"
#include "globals.hh"
#include "compression.hh"
#include "download.hh"
#include "istringstream_nocopy.hh"

#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
#include <aws/core/auth/AWSCredentialsProvider.h>
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/client/ClientConfiguration.h>
#include <aws/core/client/DefaultRetryStrategy.h>
#include <aws/core/utils/logging/FormattedLogSystem.h>
#include <aws/core/utils/logging/LogMacros.h>
#include <aws/core/utils/threading/Executor.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/CreateBucketRequest.h>
#include <aws/s3/model/GetBucketLocationRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/transfer/TransferManager.h>

using namespace Aws::Transfer;

namespace nix {

struct S3Error : public Error
{
    Aws::S3::S3Errors err;
    S3Error(Aws::S3::S3Errors err, const FormatOrString & fs)
        : Error(fs), err(err) { };
};

/* Helper: given an Outcome<R, E>, return R in case of success, or
   throw an exception in case of an error. */
template<typename R, typename E>
R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome)
{
    if (!outcome.IsSuccess())
        throw S3Error(
            outcome.GetError().GetErrorType(),
            fs.s + ": " + outcome.GetError().GetMessage());
    return outcome.GetResultWithOwnership();
}

class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem
{
    using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem;

    void ProcessFormattedStatement(Aws::String && statement) override
    {
        debug("AWS: %s", chomp(statement));
    }
};

static void initAWS()
{
    static std::once_flag flag;
    std::call_once(flag, []() {
        Aws::SDKOptions options;

        /* We install our own OpenSSL locking function (see
           shared.cc), so don't let aws-sdk-cpp override it. */
        options.cryptoOptions.initAndCleanupOpenSSL = false;

        if (verbosity >= lvlDebug) {
            options.loggingOptions.logLevel =
                verbosity == lvlDebug
                ? Aws::Utils::Logging::LogLevel::Debug
                : Aws::Utils::Logging::LogLevel::Trace;
            options.loggingOptions.logger_create_fn = [options]() {
                return std::make_shared<AwsLogger>(options.loggingOptions.logLevel);
            };
        }

        Aws::InitAPI(options);
    });
}

S3Helper::S3Helper(const std::string & profile, const std::string & region)
    : config(makeConfig(region))
    , client(make_ref<Aws::S3::S3Client>(
            profile == ""
            ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
                std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
            : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
                std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
            *config,
            // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
            false,
#else
            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
#endif
            false))
{
}

/* Log AWS retries. */
class RetryStrategy : public Aws::Client::DefaultRetryStrategy
{
    bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
    {
        auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
        if (retry)
            printError("AWS error '%s' (%s), will retry in %d ms",
                error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries));
        return retry;
    }
};

ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region)
{
    initAWS();
    auto res = make_ref<Aws::Client::ClientConfiguration>();
    res->region = region;
    res->requestTimeoutMs = 600 * 1000;
    res->retryStrategy = std::make_shared<RetryStrategy>();
    res->caFile = settings.caFile;
    return res;
}

S3Helper::DownloadResult S3Helper::getObject(
    const std::string & bucketName, const std::string & key)
{
    debug("fetching 's3://%s/%s'...", bucketName, key);

    auto request =
        Aws::S3::Model::GetObjectRequest()
        .WithBucket(bucketName)
        .WithKey(key);

    request.SetResponseStreamFactory([&]() {
        return Aws::New<std::stringstream>("STRINGSTREAM");
    });

    DownloadResult res;

    auto now1 = std::chrono::steady_clock::now();

    try {

        auto result = checkAws(fmt("AWS error fetching '%s'", key),
            client->GetObject(request));

        res.data = decodeContent(
            result.GetContentEncoding(),
            make_ref<std::string>(
                dynamic_cast<std::stringstream &>(result.GetBody()).str()));

    } catch (S3Error & e) {
        if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
    }

    auto now2 = std::chrono::steady_clock::now();

    res.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();

    return res;
}

struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
{
    const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
    const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
    const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
    const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
    const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
    const Setting<uint64_t> bufferSize{
        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads. defaults to 5Mb"};

    std::string bucketName;

    Stats stats;

    S3Helper s3Helper;

    S3BinaryCacheStoreImpl(
        const Params & params, const std::string & bucketName)
        : S3BinaryCacheStore(params)
        , bucketName(bucketName)
        , s3Helper(profile, region)
    {
        diskCache = getNarInfoDiskCache();
    }

    std::string getUri() override
    {
        return "s3://" + bucketName;
    }

    void init() override
    {
        if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {

            /* Create the bucket if it doesn't already exists. */
            // FIXME: HeadBucket would be more appropriate, but doesn't return
            // an easily parsed 404 message.
            auto res = s3Helper.client->GetBucketLocation(
                Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));

            if (!res.IsSuccess()) {
                if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
                    throw Error(format("AWS error checking bucket '%s': %s") % bucketName % res.GetError().GetMessage());

                printInfo("creating S3 bucket '%s'...", bucketName);

                // Stupid S3 bucket locations.
                auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration();
                if (s3Helper.config->region != "us-east-1")
                    bucketConfig.SetLocationConstraint(
                        Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(
                            s3Helper.config->region));

                checkAws(format("AWS error creating bucket '%s'") % bucketName,
                    s3Helper.client->CreateBucket(
                        Aws::S3::Model::CreateBucketRequest()
                        .WithBucket(bucketName)
                        .WithCreateBucketConfiguration(bucketConfig)));
            }

            BinaryCacheStore::init();

            diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
        }
    }

    const Stats & getS3Stats() override
    {
        return stats;
    }

    /* This is a specialisation of isValidPath() that optimistically
       fetches the .narinfo file, rather than first checking for its
       existence via a HEAD request. Since .narinfos are small, doing
       a GET is unlikely to be slower than HEAD. */
    bool isValidPathUncached(const Path & storePath) override
    {
        try {
            queryPathInfo(storePath);
            return true;
        } catch (InvalidPath & e) {
            return false;
        }
    }

    bool fileExists(const std::string & path) override
    {
        stats.head++;

        auto res = s3Helper.client->HeadObject(
            Aws::S3::Model::HeadObjectRequest()
            .WithBucket(bucketName)
            .WithKey(path));

        if (!res.IsSuccess()) {
            auto & error = res.GetError();
            if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
                || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
                // If bucket listing is disabled, 404s turn into 403s
                || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
                return false;
            throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
        }

        return true;
    }

    void uploadFile(const std::string & path, const std::string & data,
        const std::string & mimeType,
        const std::string & contentEncoding)
    {
        auto stream = std::make_shared<istringstream_nocopy>(data);

        auto maxThreads = std::thread::hardware_concurrency();

        auto executor =
            std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);

        TransferManagerConfiguration transferConfig(executor.get());

        transferConfig.s3Client = s3Helper.client;
        transferConfig.bufferSize = bufferSize;

        if (contentEncoding != "")
            transferConfig.createMultipartUploadTemplate.SetContentEncoding(
                contentEncoding);

        transferConfig.uploadProgressCallback =
            [&](const TransferManager *transferManager,
                const std::shared_ptr<const TransferHandle>
                    &transferHandle) {
              //FIXME: find a way to properly abort the multipart upload.
              checkInterrupt();
              printTalkative("upload progress ('%s'): '%d' of '%d' bytes",
                             path,
                             transferHandle->GetBytesTransferred(),
                             transferHandle->GetBytesTotalSize());
            };

        transferConfig.transferStatusUpdatedCallback =
            [&](const TransferManager *,
                const std::shared_ptr<const TransferHandle>
                    &transferHandle) {
              switch (transferHandle->GetStatus()) {
                  case TransferStatus::COMPLETED:
                      printTalkative("upload of '%s' completed", path);
                      stats.put++;
                      stats.putBytes += data.size();
                      break;
                  case TransferStatus::IN_PROGRESS:
                      break;
                  case TransferStatus::FAILED:
                      throw Error("AWS error: failed to upload 's3://%s/%s'",
                                  bucketName, path);
                      break;
                  default:
                      throw Error("AWS error: transfer status of 's3://%s/%s' "
                                  "in unexpected state",
                                  bucketName, path);
              };
            };

        std::shared_ptr<TransferManager> transferManager =
            TransferManager::Create(transferConfig);

        auto now1 = std::chrono::steady_clock::now();

        std::shared_ptr<TransferHandle> transferHandle =
            transferManager->UploadFile(stream, bucketName, path, mimeType,
                                        Aws::Map<Aws::String, Aws::String>());

        transferHandle->WaitUntilFinished();

        auto now2 = std::chrono::steady_clock::now();

        auto duration =
            std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
                .count();

        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
                  bucketName % path % data.size() % duration);

        stats.putTimeMs += duration;
    }

    void upsertFile(const std::string & path, const std::string & data,
        const std::string & mimeType) override
    {
        if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
            uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
        else if (lsCompression != "" && hasSuffix(path, ".ls"))
            uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
        else if (logCompression != "" && hasPrefix(path, "log/"))
            uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
        else
            uploadFile(path, data, mimeType, "");
    }

    void getFile(const std::string & path,
        std::function<void(std::shared_ptr<std::string>)> success,
        std::function<void(std::exception_ptr exc)> failure) override
    {
        sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
            stats.get++;

            auto res = s3Helper.getObject(bucketName, path);

            stats.getBytes += res.data ? res.data->size() : 0;
            stats.getTimeMs += res.durationMs;

            if (res.data)
                printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
                    bucketName, path, res.data->size(), res.durationMs);

            return res.data;
        });
    }

    PathSet queryAllValidPaths() override
    {
        PathSet paths;
        std::string marker;

        do {
            debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker);

            auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName,
                s3Helper.client->ListObjects(
                    Aws::S3::Model::ListObjectsRequest()
                    .WithBucket(bucketName)
                    .WithDelimiter("/")
                    .WithMarker(marker)));

            auto & contents = res.GetContents();

            debug(format("got %d keys, next marker '%s'")
                % contents.size() % res.GetNextMarker());

            for (auto object : contents) {
                auto & key = object.GetKey();
                if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
                paths.insert(storeDir + "/" + key.substr(0, key.size() - 8));
            }

            marker = res.GetNextMarker();
        } while (!marker.empty());

        return paths;
    }

};

static RegisterStoreImplementation regStore([](
    const std::string & uri, const Store::Params & params)
    -> std::shared_ptr<Store>
{
    if (std::string(uri, 0, 5) != "s3://") return 0;
    auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
    store->init();
    return store;
});

}

#endif
<commit_msg>share the executor between multiple copy threads.<commit_after>#if ENABLE_S3

#include "s3.hh"
#include "s3-binary-cache-store.hh"
#include "nar-info.hh"
#include "nar-info-disk-cache.hh"
#include "globals.hh"
#include "compression.hh"
#include "download.hh"
#include "istringstream_nocopy.hh"

#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
#include <aws/core/auth/AWSCredentialsProvider.h>
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/client/ClientConfiguration.h>
#include <aws/core/client/DefaultRetryStrategy.h>
#include <aws/core/utils/logging/FormattedLogSystem.h>
#include <aws/core/utils/logging/LogMacros.h>
#include <aws/core/utils/threading/Executor.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/CreateBucketRequest.h>
#include <aws/s3/model/GetBucketLocationRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/transfer/TransferManager.h>

using namespace Aws::Transfer;

namespace nix {

struct S3Error : public Error
{
    Aws::S3::S3Errors err;
    S3Error(Aws::S3::S3Errors err, const FormatOrString & fs)
        : Error(fs), err(err) { };
};

/* Helper: given an Outcome<R, E>, return R in case of success, or
   throw an exception in case of an error. */
template<typename R, typename E>
R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome)
{
    if (!outcome.IsSuccess())
        throw S3Error(
            outcome.GetError().GetErrorType(),
            fs.s + ": " + outcome.GetError().GetMessage());
    return outcome.GetResultWithOwnership();
}

class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem
{
    using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem;

    void ProcessFormattedStatement(Aws::String && statement) override
    {
        debug("AWS: %s", chomp(statement));
    }
};

static void initAWS()
{
    static std::once_flag flag;
    std::call_once(flag, []() {
        Aws::SDKOptions options;

        /* We install our own OpenSSL locking function (see
           shared.cc), so don't let aws-sdk-cpp override it. */
        options.cryptoOptions.initAndCleanupOpenSSL = false;

        if (verbosity >= lvlDebug) {
            options.loggingOptions.logLevel =
                verbosity == lvlDebug
                ? Aws::Utils::Logging::LogLevel::Debug
                : Aws::Utils::Logging::LogLevel::Trace;
            options.loggingOptions.logger_create_fn = [options]() {
                return std::make_shared<AwsLogger>(options.loggingOptions.logLevel);
            };
        }

        Aws::InitAPI(options);
    });
}

S3Helper::S3Helper(const std::string & profile, const std::string & region)
    : config(makeConfig(region))
    , client(make_ref<Aws::S3::S3Client>(
            profile == ""
            ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
                std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
            : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
                std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
            *config,
            // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
            false,
#else
            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
#endif
            false))
{
}

/* Log AWS retries. */
class RetryStrategy : public Aws::Client::DefaultRetryStrategy
{
    bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
    {
        auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
        if (retry)
            printError("AWS error '%s' (%s), will retry in %d ms",
                error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries));
        return retry;
    }
};

ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region)
{
    initAWS();
    auto res = make_ref<Aws::Client::ClientConfiguration>();
    res->region = region;
    res->requestTimeoutMs = 600 * 1000;
    res->retryStrategy = std::make_shared<RetryStrategy>();
    res->caFile = settings.caFile;
    return res;
}

S3Helper::DownloadResult S3Helper::getObject(
    const std::string & bucketName, const std::string & key)
{
    debug("fetching 's3://%s/%s'...", bucketName, key);

    auto request =
        Aws::S3::Model::GetObjectRequest()
        .WithBucket(bucketName)
        .WithKey(key);

    request.SetResponseStreamFactory([&]() {
        return Aws::New<std::stringstream>("STRINGSTREAM");
    });

    DownloadResult res;

    auto now1 = std::chrono::steady_clock::now();

    try {

        auto result = checkAws(fmt("AWS error fetching '%s'", key),
            client->GetObject(request));

        res.data = decodeContent(
            result.GetContentEncoding(),
            make_ref<std::string>(
                dynamic_cast<std::stringstream &>(result.GetBody()).str()));

    } catch (S3Error & e) {
        if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
    }

    auto now2 = std::chrono::steady_clock::now();

    res.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();

    return res;
}

struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
{
    const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
    const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
    const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
    const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
    const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
    const Setting<uint64_t> bufferSize{
        this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads. defaults to 5Mb"};

    std::string bucketName;

    Stats stats;

    S3Helper s3Helper;

    S3BinaryCacheStoreImpl(
        const Params & params, const std::string & bucketName)
        : S3BinaryCacheStore(params)
        , bucketName(bucketName)
        , s3Helper(profile, region)
    {
        diskCache = getNarInfoDiskCache();
    }

    std::string getUri() override
    {
        return "s3://" + bucketName;
    }

    void init() override
    {
        if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {

            /* Create the bucket if it doesn't already exists. */
            // FIXME: HeadBucket would be more appropriate, but doesn't return
            // an easily parsed 404 message.
            auto res = s3Helper.client->GetBucketLocation(
                Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));

            if (!res.IsSuccess()) {
                if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET)
                    throw Error(format("AWS error checking bucket '%s': %s") % bucketName % res.GetError().GetMessage());

                printInfo("creating S3 bucket '%s'...", bucketName);

                // Stupid S3 bucket locations.
                auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration();
                if (s3Helper.config->region != "us-east-1")
                    bucketConfig.SetLocationConstraint(
                        Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(
                            s3Helper.config->region));

                checkAws(format("AWS error creating bucket '%s'") % bucketName,
                    s3Helper.client->CreateBucket(
                        Aws::S3::Model::CreateBucketRequest()
                        .WithBucket(bucketName)
                        .WithCreateBucketConfiguration(bucketConfig)));
            }

            BinaryCacheStore::init();

            diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority);
        }
    }

    const Stats & getS3Stats() override
    {
        return stats;
    }

    /* This is a specialisation of isValidPath() that optimistically
       fetches the .narinfo file, rather than first checking for its
       existence via a HEAD request. Since .narinfos are small, doing
       a GET is unlikely to be slower than HEAD. */
    bool isValidPathUncached(const Path & storePath) override
    {
        try {
            queryPathInfo(storePath);
            return true;
        } catch (InvalidPath & e) {
            return false;
        }
    }

    bool fileExists(const std::string & path) override
    {
        stats.head++;

        auto res = s3Helper.client->HeadObject(
            Aws::S3::Model::HeadObjectRequest()
            .WithBucket(bucketName)
            .WithKey(path));

        if (!res.IsSuccess()) {
            auto & error = res.GetError();
            if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
                || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
                // If bucket listing is disabled, 404s turn into 403s
                || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
                return false;
            throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
        }

        return true;
    }

    void uploadFile(const std::string & path, const std::string & data,
        const std::string & mimeType,
        const std::string & contentEncoding)
    {
        auto stream = std::make_shared<istringstream_nocopy>(data);

        auto maxThreads = std::thread::hardware_concurrency();

        static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
            executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);

        TransferManagerConfiguration transferConfig(executor.get());

        transferConfig.s3Client = s3Helper.client;
        transferConfig.bufferSize = bufferSize;

        if (contentEncoding != "")
            transferConfig.createMultipartUploadTemplate.SetContentEncoding(
                contentEncoding);

        transferConfig.uploadProgressCallback =
            [&](const TransferManager *transferManager,
                const std::shared_ptr<const TransferHandle>
                    &transferHandle) {
              //FIXME: find a way to properly abort the multipart upload.
              checkInterrupt();
              printTalkative("upload progress ('%s'): '%d' of '%d' bytes",
                             path,
                             transferHandle->GetBytesTransferred(),
                             transferHandle->GetBytesTotalSize());
            };

        transferConfig.transferStatusUpdatedCallback =
            [&](const TransferManager *,
                const std::shared_ptr<const TransferHandle>
                    &transferHandle) {
              switch (transferHandle->GetStatus()) {
                  case TransferStatus::COMPLETED:
                      printTalkative("upload of '%s' completed", path);
                      stats.put++;
                      stats.putBytes += data.size();
                      break;
                  case TransferStatus::IN_PROGRESS:
                      break;
                  case TransferStatus::FAILED:
                      throw Error("AWS error: failed to upload 's3://%s/%s'",
                                  bucketName, path);
                      break;
                  default:
                      throw Error("AWS error: transfer status of 's3://%s/%s' "
                                  "in unexpected state",
                                  bucketName, path);
              };
            };

        std::shared_ptr<TransferManager> transferManager =
            TransferManager::Create(transferConfig);

        auto now1 = std::chrono::steady_clock::now();

        std::shared_ptr<TransferHandle> transferHandle =
            transferManager->UploadFile(stream, bucketName, path, mimeType,
                                        Aws::Map<Aws::String, Aws::String>());

        transferHandle->WaitUntilFinished();

        auto now2 = std::chrono::steady_clock::now();

        auto duration =
            std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
                .count();

        printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
                  bucketName % path % data.size() % duration);

        stats.putTimeMs += duration;
    }

    void upsertFile(const std::string & path, const std::string & data,
        const std::string & mimeType) override
    {
        if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
            uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
        else if (lsCompression != "" && hasSuffix(path, ".ls"))
            uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
        else if (logCompression != "" && hasPrefix(path, "log/"))
            uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
        else
            uploadFile(path, data, mimeType, "");
    }

    void getFile(const std::string & path,
        std::function<void(std::shared_ptr<std::string>)> success,
        std::function<void(std::exception_ptr exc)> failure) override
    {
        sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
            stats.get++;

            auto res = s3Helper.getObject(bucketName, path);

            stats.getBytes += res.data ? res.data->size() : 0;
            stats.getTimeMs += res.durationMs;

            if (res.data)
                printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
                    bucketName, path, res.data->size(), res.durationMs);

            return res.data;
        });
    }

    PathSet queryAllValidPaths() override
    {
        PathSet paths;
        std::string marker;

        do {
            debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker);

            auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName,
                s3Helper.client->ListObjects(
                    Aws::S3::Model::ListObjectsRequest()
                    .WithBucket(bucketName)
                    .WithDelimiter("/")
                    .WithMarker(marker)));

            auto & contents = res.GetContents();

            debug(format("got %d keys, next marker '%s'")
                % contents.size() % res.GetNextMarker());

            for (auto object : contents) {
                auto & key = object.GetKey();
                if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
                paths.insert(storeDir + "/" + key.substr(0, key.size() - 8));
            }

            marker = res.GetNextMarker();
        } while (!marker.empty());

        return paths;
    }

};

static RegisterStoreImplementation regStore([](
    const std::string & uri, const Store::Params & params)
    -> std::shared_ptr<Store>
{
    if (std::string(uri, 0, 5) != "s3://") return 0;
    auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
    store->init();
    return store;
});

}

#endif
<|endoftext|>


<commit_before>#include "pch.h"
#include "DemoScene.h"
#include "resource.h"

using namespace std;
using D2D1::ColorF;
using D2D1::Matrix3x2F;

ref class Test
{
public:
	static System::Random ^r = gcnew System::Random(System::Guid::NewGuid().GetHashCode());
};

RandomDemoScene::RandomDemoScene() :
	_ui{ 0, SampleCount - 1 }
{
}

void RandomDemoScene::CreateDeviceResources(CHwndRenderTarget *target)
{
	_black = new CD2DSolidColorBrush(target, ColorF(ColorF::Black));
	_white = new CD2DSolidColorBrush(target, ColorF(ColorF::White));
}

void RandomDemoScene::CreateDeviceSizeResources(CHwndRenderTarget *)
{
}

void RandomDemoScene::Render(CHwndRenderTarget * target)
{
	auto size = target->GetSize();
	auto unitH = size.height * 0.8f / (SampleTotal / SampleCount);
	auto baseH = size.height * 0.1f;
	auto baseW = size.width / SampleCount;

	double sum = std::accumulate(begin(_result), end(_result), 0.0);
	double mean = sum / _result.size();
	double sq_sum = inner_product(begin(_result), end(_result), begin(_result), 0.0);

	double stdev = sqrt(sq_sum / _result.size() - mean * mean);
	DrawStd(target, stdev, 10.0f, 0.0f);
	_minStd = min(stdev, _minStd);
	DrawStd(target, _minStd, 10.0f, 10.0f);
	_maxStd = max(stdev, _maxStd);
	DrawStd(target, _maxStd, 10.0f, 20.0f);
	auto stdavg = (_sumStd += stdev) / _iterateCount;
	DrawStd(target, stdavg, 10.0f, 30.0f);

	DrawStd(target, _iterateCount/100.0, 10.0f, 50.0f);

	for (int i = 0; i < SampleCount; ++i)
	{
		auto count = _result[i];
		CD2DRectF rect{
			i * baseW + 1, baseH,
			(i + 1)* baseW - 1, baseH + count * unitH
		};
		target->FillRectangle(rect, _black);
	}
}

void RandomDemoScene::Update()
{
	_result.clear();
	_result.assign(SampleCount, 0);
	for (auto i = 0; i < SampleTotal; ++i)
	{
		_result[GetNextRand()]++;
	}
	_iterateCount++;
}

std::unique_ptr<RandomDemoScene> RandomDemoScene::Make(int type)
{
	switch (type)
	{
	case ID_TEST_RAND:
		return make_unique<RandDemoScene>();
	case ID_TEST_RD:
		return make_unique<RdDemoScene>();
	case ID_TEST_MT19937:
		return make_unique<Mt19937DemoScene>();
	case ID_TEST_MINSTD:
		return make_unique<MinstdRandDemoScene>();
	case ID_TEST_RANLUX:
		return make_unique<RanluxDemoScene>();
	case ID_TEST_CSRANDOM:
		return make_unique<CSDemoScene>();
	case ID_TEST_MT19937_64:
		return make_unique<Mt1993764DemoScene>();
	}
	throw make_exception_ptr("unknown type");
}

void RandomDemoScene::DrawStd(CHwndRenderTarget* target, double std, float height, float y)
{
	auto size = target->GetSize();
	CD2DRectF stdevRect{ 0.f, y, float(std / 150 * size.width), y + height };
	target->FillRectangle(stdevRect, _black);
}

int RandDemoScene::GetNextRand()
{
	return rand() % SampleCount;
}

int RdDemoScene::GetNextRand()
{
	return _ui(_rd);
}

int Mt19937DemoScene::GetNextRand()
{
	return _ui(_mt);
}

int RanluxDemoScene::GetNextRand()
{
	return _ui(_rd);
}

int MinstdRandDemoScene::GetNextRand()
{
	return _ui(_rd);
}

int CSDemoScene::GetNextRand()
{
	return Test::r->Next() % SampleCount;
}

int Mt1993764DemoScene::GetNextRand()
{
	return _ui(_mt);
}
<commit_msg>specify rate<commit_after>#include "pch.h"
#include "DemoScene.h"
#include "resource.h"

using namespace std;
using D2D1::ColorF;
using D2D1::Matrix3x2F;

ref class Test
{
public:
	static System::Random ^r = gcnew System::Random(System::Guid::NewGuid().GetHashCode());
};

RandomDemoScene::RandomDemoScene() :
	_ui{ 0, SampleCount - 1 }
{
}

void RandomDemoScene::CreateDeviceResources(CHwndRenderTarget *target)
{
	_black = new CD2DSolidColorBrush(target, ColorF(ColorF::Black));
	_white = new CD2DSolidColorBrush(target, ColorF(ColorF::White));
}

void RandomDemoScene::CreateDeviceSizeResources(CHwndRenderTarget *)
{
}

void RandomDemoScene::Render(CHwndRenderTarget * target)
{
	auto size = target->GetSize();
	auto unitH = size.height * 0.8f / (SampleTotal / SampleCount);
	auto baseH = size.height * 0.1f;
	auto baseW = size.width / SampleCount;

	double sum = std::accumulate(begin(_result), end(_result), 0.0);
	double mean = sum / _result.size();
	double sq_sum = inner_product(begin(_result), end(_result), begin(_result), 0.0);

	double stdev = sqrt(sq_sum / _result.size() - mean * mean);
	DrawStd(target, stdev, 10.0f, 0.0f);
	_minStd = min(stdev, _minStd);
	DrawStd(target, _minStd, 10.0f, 10.0f);
	_maxStd = max(stdev, _maxStd);
	DrawStd(target, _maxStd, 10.0f, 20.0f);
	auto stdavg = (_sumStd += stdev) / _iterateCount;
	DrawStd(target, stdavg, 10.0f, 30.0f);

	//rand		66.70	(154W)
	//rd		65.33	(172W)
	//mt19937	65.11	(152W)
	//Random	65.21	(152W)

	DrawStd(target, _iterateCount / 100.0, 10.0f, 50.0f);

	for (int i = 0; i < SampleCount; ++i)
	{
		auto count = _result[i];
		CD2DRectF rect{
			i * baseW + 1, baseH,
			(i + 1)* baseW - 1, baseH + count * unitH
		};
		target->FillRectangle(rect, _black);
	}
}

void RandomDemoScene::Update()
{
	_result.clear();
	_result.assign(SampleCount, 0);
	for (auto i = 0; i < SampleTotal; ++i)
	{
		_result[GetNextRand()]++;
	}
	_iterateCount++;
}

std::unique_ptr<RandomDemoScene> RandomDemoScene::Make(int type)
{
	switch (type)
	{
	case ID_TEST_RAND:
		return make_unique<RandDemoScene>();
	case ID_TEST_RD:
		return make_unique<RdDemoScene>();
	case ID_TEST_MT19937:
		return make_unique<Mt19937DemoScene>();
	case ID_TEST_MINSTD:
		return make_unique<MinstdRandDemoScene>();
	case ID_TEST_RANLUX:
		return make_unique<RanluxDemoScene>();
	case ID_TEST_CSRANDOM:
		return make_unique<CSDemoScene>();
	case ID_TEST_MT19937_64:
		return make_unique<Mt1993764DemoScene>();
	}
	throw make_exception_ptr("unknown type");
}

void RandomDemoScene::DrawStd(CHwndRenderTarget* target, double std, float height, float y)
{
	auto size = target->GetSize();
	CD2DRectF stdevRect{ 0.f, y, float(std / 150 * size.width), y + height };
	target->FillRectangle(stdevRect, _black);
}

int RandDemoScene::GetNextRand()
{
	return rand() % SampleCount;
}

int RdDemoScene::GetNextRand()
{
	return _ui(_rd);
}

int Mt19937DemoScene::GetNextRand()
{
	return _ui(_mt);
}

int RanluxDemoScene::GetNextRand()
{
	return _ui(_rd);
}

int MinstdRandDemoScene::GetNextRand()
{
	return _ui(_rd);
}

int CSDemoScene::GetNextRand()
{
	return Test::r->Next() % SampleCount;
}

int Mt1993764DemoScene::GetNextRand()
{
	return _ui(_mt);
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "database.hh"
#include "db/system_keyspace.hh"
#include "db/timeout_clock.hh"
#include "dht/i_partitioner.hh"
#include "flat_mutation_reader.hh"
#include "mutation_fragment.hh"
#include "mutation_reader.hh"
#include "query-request.hh"
#include "schema.hh"
#include "tracing/tracing.hh"

#include <boost/range/iterator_range.hpp>

#include <iterator>
#include <memory>

namespace db::view {

// Allows a user to query the views_builds_in_progress system table
// in terms of the scylla_views_builds_in_progress one, which is
// a superset of the former. When querying, we don't have to adjust
// the clustering key, but we have to adjust the requested regular
// columns. When reading the results from the scylla_views_builds_in_progress
// table, we adjust the clustering key (we shed the cpu_id column) and map
// back the regular columns.
class build_progress_virtual_reader {
    database& _db;

    struct build_progress_reader : flat_mutation_reader::impl {
        column_id _scylla_next_token_col;
        column_id _scylla_generation_number_col;
        column_id _legacy_last_token_col;
        column_id _legacy_generation_number_col;
        const query::partition_slice& _legacy_slice;
        query::partition_slice _slice;
        flat_mutation_reader _underlying;

        build_progress_reader(
                schema_ptr legacy_schema,
                column_family& scylla_views_build_progress,
                const dht::partition_range& range,
                const query::partition_slice& slice,
                const io_priority_class& pc,
                tracing::trace_state_ptr trace_state,
                streamed_mutation::forwarding fwd,
                mutation_reader::forwarding fwd_mr)
                : flat_mutation_reader::impl(std::move(legacy_schema))
                , _scylla_next_token_col(scylla_views_build_progress.schema()->get_column_definition("next_token")->id)
                , _scylla_generation_number_col(scylla_views_build_progress.schema()->get_column_definition("generation_number")->id)
                , _legacy_last_token_col(_schema->get_column_definition("last_token")->id)
                , _legacy_generation_number_col(_schema->get_column_definition("generation_number")->id)
                , _legacy_slice(slice)
                , _slice(adjust_partition_slice())
                , _underlying(scylla_views_build_progress.make_reader(
                        scylla_views_build_progress.schema(),
                        range,
                        slice,
                        pc,
                        std::move(trace_state),
                        fwd,
                        fwd_mr)) {
        }

        const schema& underlying_schema() const {
            return *_underlying.schema();
        }

        query::partition_slice adjust_partition_slice() {
            auto slice = _legacy_slice;
            std::vector<column_id> adjusted_columns;
            for (auto col_id : slice.regular_columns) {
                if (col_id == _legacy_last_token_col) {
                    adjusted_columns.push_back(_scylla_next_token_col);
                } else if (col_id == _legacy_generation_number_col) {
                    adjusted_columns.push_back(_scylla_generation_number_col);
                }
            }
            slice.regular_columns = std::move(adjusted_columns);
            return slice;
        }

        clustering_key adjust_ckey(clustering_key& underlying_ck) {
            if (underlying_ck.size(underlying_schema()) < 3) {
                return std::move(underlying_ck);
            }
            // Drop the cpu_id from the clustering key
            auto end = underlying_ck.begin(underlying_schema());
            std::advance(end, 1);
            auto r = boost::make_iterator_range(underlying_ck.begin(underlying_schema()), std::move(end));
            return clustering_key_prefix::from_exploded(r);
        }

        virtual future<> fill_buffer(db::timeout_clock::time_point timeout) override {
            return _underlying.fill_buffer(timeout).then([this] {
                _end_of_stream = _underlying.is_end_of_stream();
                while (!_underlying.is_buffer_empty()) {
                    auto mf = _underlying.pop_mutation_fragment();
                    if (mf.is_clustering_row()) {
                        auto scylla_in_progress_row = std::move(mf).as_clustering_row();
                        auto legacy_in_progress_row = row();
                        // Drop the first_token from the regular columns
                        scylla_in_progress_row.cells().for_each_cell([&, this] (column_id id, atomic_cell_or_collection& c) {
                            if (id == _scylla_next_token_col) {
                                legacy_in_progress_row.append_cell(_legacy_last_token_col, std::move(c));
                            } else if (id == _scylla_generation_number_col) {
                                legacy_in_progress_row.append_cell(_legacy_generation_number_col, std::move(c));
                            }
                        });
                        mf = clustering_row(
                                adjust_ckey(scylla_in_progress_row.key()),
                                std::move(scylla_in_progress_row.tomb()),
                                std::move(scylla_in_progress_row.marker()),
                                std::move(legacy_in_progress_row));
                    } else if (mf.is_range_tombstone()) {
                        auto scylla_in_progress_rt = std::move(mf).as_range_tombstone();
                        mf = range_tombstone(
                                adjust_ckey(scylla_in_progress_rt.start),
                                scylla_in_progress_rt.start_kind,
                                scylla_in_progress_rt.end,
                                scylla_in_progress_rt.end_kind,
                                scylla_in_progress_rt.tomb);
                    }
                    push_mutation_fragment(std::move(mf));
                }
            });
        }

        virtual void next_partition() override {
            _end_of_stream = false;
            clear_buffer_to_next_partition();
            if (is_buffer_empty()) {
                _underlying.next_partition();
            }
        }

        virtual future<> fast_forward_to(const dht::partition_range& pr, db::timeout_clock::time_point timeout) override {
            clear_buffer();
            _end_of_stream = false;
            return _underlying.fast_forward_to(pr, timeout);
        }

        virtual future<> fast_forward_to(position_range range, db::timeout_clock::time_point timeout) override {
            forward_buffer_to(range.start());
            _end_of_stream = false;
            return _underlying.fast_forward_to(std::move(range), timeout);
        }
    };

public:
    build_progress_virtual_reader(database& db)
            : _db(db) {
    }

    flat_mutation_reader operator()(
            schema_ptr s,
            const dht::partition_range& range,
            const query::partition_slice& slice,
            const io_priority_class& pc,
            tracing::trace_state_ptr trace_state,
            streamed_mutation::forwarding fwd,
            mutation_reader::forwarding fwd_mr) {
        return flat_mutation_reader(std::make_unique<build_progress_reader>(
                std::move(s),
                _db.find_column_family(s->ks_name(), system_keyspace::v3::SCYLLA_VIEWS_BUILDS_IN_PROGRESS),
                range,
                slice,
                pc,
                std::move(trace_state),
                fwd,
                fwd_mr));
    }
};

}<commit_msg>db/view/build_progress_virtual_reader: Fix full ck detection<commit_after>/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "database.hh"
#include "db/system_keyspace.hh"
#include "db/timeout_clock.hh"
#include "dht/i_partitioner.hh"
#include "flat_mutation_reader.hh"
#include "mutation_fragment.hh"
#include "mutation_reader.hh"
#include "query-request.hh"
#include "schema.hh"
#include "tracing/tracing.hh"

#include <boost/range/iterator_range.hpp>

#include <iterator>
#include <memory>

namespace db::view {

// Allows a user to query the views_builds_in_progress system table
// in terms of the scylla_views_builds_in_progress one, which is
// a superset of the former. When querying, we don't have to adjust
// the clustering key, but we have to adjust the requested regular
// columns. When reading the results from the scylla_views_builds_in_progress
// table, we adjust the clustering key (we shed the cpu_id column) and map
// back the regular columns.
class build_progress_virtual_reader {
    database& _db;

    struct build_progress_reader : flat_mutation_reader::impl {
        column_id _scylla_next_token_col;
        column_id _scylla_generation_number_col;
        column_id _legacy_last_token_col;
        column_id _legacy_generation_number_col;
        const query::partition_slice& _legacy_slice;
        query::partition_slice _slice;
        flat_mutation_reader _underlying;

        build_progress_reader(
                schema_ptr legacy_schema,
                column_family& scylla_views_build_progress,
                const dht::partition_range& range,
                const query::partition_slice& slice,
                const io_priority_class& pc,
                tracing::trace_state_ptr trace_state,
                streamed_mutation::forwarding fwd,
                mutation_reader::forwarding fwd_mr)
                : flat_mutation_reader::impl(std::move(legacy_schema))
                , _scylla_next_token_col(scylla_views_build_progress.schema()->get_column_definition("next_token")->id)
                , _scylla_generation_number_col(scylla_views_build_progress.schema()->get_column_definition("generation_number")->id)
                , _legacy_last_token_col(_schema->get_column_definition("last_token")->id)
                , _legacy_generation_number_col(_schema->get_column_definition("generation_number")->id)
                , _legacy_slice(slice)
                , _slice(adjust_partition_slice())
                , _underlying(scylla_views_build_progress.make_reader(
                        scylla_views_build_progress.schema(),
                        range,
                        slice,
                        pc,
                        std::move(trace_state),
                        fwd,
                        fwd_mr)) {
        }

        const schema& underlying_schema() const {
            return *_underlying.schema();
        }

        query::partition_slice adjust_partition_slice() {
            auto slice = _legacy_slice;
            std::vector<column_id> adjusted_columns;
            for (auto col_id : slice.regular_columns) {
                if (col_id == _legacy_last_token_col) {
                    adjusted_columns.push_back(_scylla_next_token_col);
                } else if (col_id == _legacy_generation_number_col) {
                    adjusted_columns.push_back(_scylla_generation_number_col);
                }
            }
            slice.regular_columns = std::move(adjusted_columns);
            return slice;
        }

        clustering_key adjust_ckey(clustering_key& underlying_ck) {
            if (!underlying_ck.is_full(underlying_schema())) {
                return std::move(underlying_ck);
            }
            // Drop the cpu_id from the clustering key
            auto end = underlying_ck.begin(underlying_schema());
            std::advance(end, underlying_schema().clustering_key_size() - 1);
            auto r = boost::make_iterator_range(underlying_ck.begin(underlying_schema()), std::move(end));
            return clustering_key_prefix::from_exploded(r);
        }

        virtual future<> fill_buffer(db::timeout_clock::time_point timeout) override {
            return _underlying.fill_buffer(timeout).then([this] {
                _end_of_stream = _underlying.is_end_of_stream();
                while (!_underlying.is_buffer_empty()) {
                    auto mf = _underlying.pop_mutation_fragment();
                    if (mf.is_clustering_row()) {
                        auto scylla_in_progress_row = std::move(mf).as_clustering_row();
                        auto legacy_in_progress_row = row();
                        // Drop the first_token from the regular columns
                        scylla_in_progress_row.cells().for_each_cell([&, this] (column_id id, atomic_cell_or_collection& c) {
                            if (id == _scylla_next_token_col) {
                                legacy_in_progress_row.append_cell(_legacy_last_token_col, std::move(c));
                            } else if (id == _scylla_generation_number_col) {
                                legacy_in_progress_row.append_cell(_legacy_generation_number_col, std::move(c));
                            }
                        });
                        mf = clustering_row(
                                adjust_ckey(scylla_in_progress_row.key()),
                                std::move(scylla_in_progress_row.tomb()),
                                std::move(scylla_in_progress_row.marker()),
                                std::move(legacy_in_progress_row));
                    } else if (mf.is_range_tombstone()) {
                        auto scylla_in_progress_rt = std::move(mf).as_range_tombstone();
                        mf = range_tombstone(
                                adjust_ckey(scylla_in_progress_rt.start),
                                scylla_in_progress_rt.start_kind,
                                scylla_in_progress_rt.end,
                                scylla_in_progress_rt.end_kind,
                                scylla_in_progress_rt.tomb);
                    }
                    push_mutation_fragment(std::move(mf));
                }
            });
        }

        virtual void next_partition() override {
            _end_of_stream = false;
            clear_buffer_to_next_partition();
            if (is_buffer_empty()) {
                _underlying.next_partition();
            }
        }

        virtual future<> fast_forward_to(const dht::partition_range& pr, db::timeout_clock::time_point timeout) override {
            clear_buffer();
            _end_of_stream = false;
            return _underlying.fast_forward_to(pr, timeout);
        }

        virtual future<> fast_forward_to(position_range range, db::timeout_clock::time_point timeout) override {
            forward_buffer_to(range.start());
            _end_of_stream = false;
            return _underlying.fast_forward_to(std::move(range), timeout);
        }
    };

public:
    build_progress_virtual_reader(database& db)
            : _db(db) {
    }

    flat_mutation_reader operator()(
            schema_ptr s,
            const dht::partition_range& range,
            const query::partition_slice& slice,
            const io_priority_class& pc,
            tracing::trace_state_ptr trace_state,
            streamed_mutation::forwarding fwd,
            mutation_reader::forwarding fwd_mr) {
        return flat_mutation_reader(std::make_unique<build_progress_reader>(
                std::move(s),
                _db.find_column_family(s->ks_name(), system_keyspace::v3::SCYLLA_VIEWS_BUILDS_IN_PROGRESS),
                range,
                slice,
                pc,
                std::move(trace_state),
                fwd,
                fwd_mr));
    }
};

}<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: XMLIndexAlphabeticalSourceContext.hxx,v $
 *
 *  $Revision: 1.4 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-09 15:02:54 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _XMLOFF_XMLINDEXALPHABETICALSOURCECONTEXT_HXX_
#define _XMLOFF_XMLINDEXALPHABETICALSOURCECONTEXT_HXX_

#ifndef _XMLOFF_XMLINDEXSOURCEBASECONTEXT_HXX_
#include "XMLIndexSourceBaseContext.hxx"
#endif

#ifndef _COM_SUN_STAR_UNO_REFERENCE_H_
#include <com/sun/star/uno/Reference.h>
#endif

#ifndef _COM_SUN_STAR_LANG_LOCALE_HPP_
#include <com/sun/star/lang/Locale.hpp>
#endif

namespace com { namespace sun { namespace star {
    namespace xml { namespace sax { class XAttributeList; } }
    namespace beans { class XPropertySet; }
} } }
namespace rtl { class OUString; }

/**
 * Import alphabetical (keyword) index source element
 */
class XMLIndexAlphabeticalSourceContext : public XMLIndexSourceBaseContext
{
    const ::rtl::OUString sMainEntryCharacterStyleName;
    const ::rtl::OUString sUseAlphabeticalSeparators;
    const ::rtl::OUString sUseCombinedEntries;
    const ::rtl::OUString sIsCaseSensitive;
    const ::rtl::OUString sUseKeyAsEntry;
    const ::rtl::OUString sUseUpperCase;
    const ::rtl::OUString sUseDash;
    const ::rtl::OUString sUsePP;
    const ::rtl::OUString sIsCommaSeparated;
    const ::rtl::OUString sSortAlgorithm;
    const ::rtl::OUString sLocale;

    ::com::sun::star::lang::Locale aLocale;
    ::rtl::OUString sAlgorithm;

    ::rtl::OUString sMainEntryStyleName;
    sal_Bool bMainEntryStyleNameOK;

    sal_Bool bSeparators;
    sal_Bool bCombineEntries;
    sal_Bool bCaseSensitive;
    sal_Bool bEntry;
    sal_Bool bUpperCase;
    sal_Bool bCombineDash;
    sal_Bool bCombinePP;
    sal_Bool bCommaSeparated;

public:

    TYPEINFO();

    XMLIndexAlphabeticalSourceContext(
        SvXMLImport& rImport,
        sal_uInt16 nPrfx,
        const ::rtl::OUString& rLocalName,
        ::com::sun::star::uno::Reference<
            ::com::sun::star::beans::XPropertySet> & rPropSet);

    ~XMLIndexAlphabeticalSourceContext();

protected:

    virtual void ProcessAttribute(
        enum IndexSourceParamEnum eParam,
        const ::rtl::OUString& rValue);

    virtual void EndElement();

    virtual SvXMLImportContext* CreateChildContext(
        sal_uInt16 nPrefix,
        const ::rtl::OUString& rLocalName,
        const ::com::sun::star::uno::Reference<
            ::com::sun::star::xml::sax::XAttributeList> & xAttrList );
};

#endif
<commit_msg>INTEGRATION: CWS changefileheader (1.4.448); FILE MERGED 2008/04/01 16:10:05 thb 1.4.448.3: #i85898# Stripping all external header guards 2008/04/01 13:05:23 thb 1.4.448.2: #i85898# Stripping all external header guards 2008/03/31 16:28:33 rt 1.4.448.1: #i87441# Change license header to LPGL v3.<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: XMLIndexAlphabeticalSourceContext.hxx,v $
 * $Revision: 1.5 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

#ifndef _XMLOFF_XMLINDEXALPHABETICALSOURCECONTEXT_HXX_
#define _XMLOFF_XMLINDEXALPHABETICALSOURCECONTEXT_HXX_

#include "XMLIndexSourceBaseContext.hxx"
#include <com/sun/star/uno/Reference.h>
#include <com/sun/star/lang/Locale.hpp>

namespace com { namespace sun { namespace star {
    namespace xml { namespace sax { class XAttributeList; } }
    namespace beans { class XPropertySet; }
} } }
namespace rtl { class OUString; }

/**
 * Import alphabetical (keyword) index source element
 */
class XMLIndexAlphabeticalSourceContext : public XMLIndexSourceBaseContext
{
    const ::rtl::OUString sMainEntryCharacterStyleName;
    const ::rtl::OUString sUseAlphabeticalSeparators;
    const ::rtl::OUString sUseCombinedEntries;
    const ::rtl::OUString sIsCaseSensitive;
    const ::rtl::OUString sUseKeyAsEntry;
    const ::rtl::OUString sUseUpperCase;
    const ::rtl::OUString sUseDash;
    const ::rtl::OUString sUsePP;
    const ::rtl::OUString sIsCommaSeparated;
    const ::rtl::OUString sSortAlgorithm;
    const ::rtl::OUString sLocale;

    ::com::sun::star::lang::Locale aLocale;
    ::rtl::OUString sAlgorithm;

    ::rtl::OUString sMainEntryStyleName;
    sal_Bool bMainEntryStyleNameOK;

    sal_Bool bSeparators;
    sal_Bool bCombineEntries;
    sal_Bool bCaseSensitive;
    sal_Bool bEntry;
    sal_Bool bUpperCase;
    sal_Bool bCombineDash;
    sal_Bool bCombinePP;
    sal_Bool bCommaSeparated;

public:

    TYPEINFO();

    XMLIndexAlphabeticalSourceContext(
        SvXMLImport& rImport,
        sal_uInt16 nPrfx,
        const ::rtl::OUString& rLocalName,
        ::com::sun::star::uno::Reference<
            ::com::sun::star::beans::XPropertySet> & rPropSet);

    ~XMLIndexAlphabeticalSourceContext();

protected:

    virtual void ProcessAttribute(
        enum IndexSourceParamEnum eParam,
        const ::rtl::OUString& rValue);

    virtual void EndElement();

    virtual SvXMLImportContext* CreateChildContext(
        sal_uInt16 nPrefix,
        const ::rtl::OUString& rLocalName,
        const ::com::sun::star::uno::Reference<
            ::com::sun::star::xml::sax::XAttributeList> & xAttrList );
};

#endif
<|endoftext|>


<commit_before><commit_msg>generator: disable warning C4503: decorated name length exceeded (Windows)<commit_after><|endoftext|>


<commit_before>#include "ffi.h"

ForeignCaller::ForeignCaller() {
}

ForeignCaller::~ForeignCaller() {
}

Persistent<FunctionTemplate> ForeignCaller::foreign_caller_template;

Handle<FunctionTemplate> ForeignCaller::MakeTemplate()
{
    HandleScope scope;
    Handle<FunctionTemplate> t = FunctionTemplate::New(New);

    Local<ObjectTemplate> inst = t->InstanceTemplate();
    inst->SetInternalFieldCount(1);

    return scope.Close(t);
}

void ForeignCaller::Initialize(Handle<Object> target)
{
    HandleScope scope;

    if (foreign_caller_template.IsEmpty()) {
        foreign_caller_template = Persistent<FunctionTemplate>::New(MakeTemplate());
    }

    Handle<FunctionTemplate> t = foreign_caller_template;

    NODE_SET_PROTOTYPE_METHOD(t, "exec", Exec);

    target->Set(String::NewSymbol("ForeignCaller"), t->GetFunction());
}

Handle<Value> ForeignCaller::New(const Arguments& args)
{
    HandleScope     scope;
    ForeignCaller   *self = new ForeignCaller();

    if (args.Length() == 5) {
        Pointer *cif    = ObjectWrap::Unwrap<Pointer>(args[0]->ToObject());
        Pointer *fn     = ObjectWrap::Unwrap<Pointer>(args[1]->ToObject());
        Pointer *fnargs = ObjectWrap::Unwrap<Pointer>(args[2]->ToObject());
        Pointer *res    = ObjectWrap::Unwrap<Pointer>(args[3]->ToObject());

        self->m_cif     = (ffi_cif *)cif->GetPointer();
        self->m_fn      = (void (*)(void))fn->GetPointer();
        self->m_res     = (void *)res->GetPointer();
        self->m_fnargs  = (void **)fnargs->GetPointer();

        self->m_async   = args[4]->BooleanValue();
    } else {
      return THROW_ERROR_EXCEPTION("Not enough arguments");
    }

    self->Wrap(args.This());
    return scope.Close(args.This());
}

Handle<Value> ForeignCaller::Exec(const Arguments& args)
{
    HandleScope     scope;
    ForeignCaller   *self = ObjectWrap::Unwrap<ForeignCaller>(args.This());

    if (self->m_async) {
        AsyncCallParams *p = new AsyncCallParams();

        // cuter way of doing this?
        p->cif  = self->m_cif;
        p->ptr  = self->m_fn;
        p->res  = self->m_res;
        p->args = self->m_fnargs;

        // get the events.EventEmitter constructor
        Local<Object> global = Context::GetCurrent()->Global();
        Local<Object> events = global->Get(String::NewSymbol("process"))->ToObject();
        Local<Function> emitterConstructor = Local<Function>::Cast(events->Get(String::NewSymbol("EventEmitter")));

        // construct a new EventEmitter object
        p->emitter = Persistent<Object>::New(emitterConstructor->NewInstance());

        BEGIN_ASYNC(p, ForeignCaller::AsyncFFICall, ForeignCaller::FinishAsyncFFICall);
        return scope.Close(p->emitter);
    } else {
#if __OBJC__ || __OBJC2__
      @try {
#endif
        ffi_call(
            self->m_cif,
            self->m_fn,
            self->m_res,
            self->m_fnargs
        );
#if __OBJC__ || __OBJC2__
      } @catch (id ex) {
        return ThrowException(Pointer::WrapPointer((unsigned char *)ex));
      }
#endif
    }

    return Undefined();
}

async_rtn ForeignCaller::AsyncFFICall(uv_work_t *req) {
    AsyncCallParams *p = (AsyncCallParams *)req->data;
    ffi_call(p->cif, p->ptr, p->res, p->args);
    RETURN_ASYNC;
}

async_rtn ForeignCaller::FinishAsyncFFICall(uv_work_t *req) {
    HandleScope scope;

    AsyncCallParams *p = (AsyncCallParams *)req->data;
    Local<Value> argv[1];

    argv[0] = Local<Value>::New(String::New("success"));

    // get a reference to the 'emit' function
    Local<Value> emitVal = p->emitter->Get(String::NewSymbol("emit"));
    Local<Function> emit = Local<Function>::Cast(emitVal);

    TryCatch try_catch;

    // emit a success event
    emit->Call(p->emitter, 1, argv);

    // dispose of our persistent handle to the EventEmitter object
    p->emitter.Dispose();

    // free up our memory (allocated in FFICall)
    delete p;

    if (try_catch.HasCaught()) {
       FatalException(try_catch);
    }

    RETURN_ASYNC_AFTER;
}
<commit_msg>Whitespace.<commit_after>#include "ffi.h"

ForeignCaller::ForeignCaller() {
}

ForeignCaller::~ForeignCaller() {
}

Persistent<FunctionTemplate> ForeignCaller::foreign_caller_template;

Handle<FunctionTemplate> ForeignCaller::MakeTemplate() {
  HandleScope scope;
  Handle<FunctionTemplate> t = FunctionTemplate::New(New);

  Local<ObjectTemplate> inst = t->InstanceTemplate();
  inst->SetInternalFieldCount(1);

  return scope.Close(t);
}

void ForeignCaller::Initialize(Handle<Object> target) {
  HandleScope scope;

  if (foreign_caller_template.IsEmpty()) {
    foreign_caller_template = Persistent<FunctionTemplate>::New(MakeTemplate());
  }

  Handle<FunctionTemplate> t = foreign_caller_template;

  NODE_SET_PROTOTYPE_METHOD(t, "exec", Exec);

  target->Set(String::NewSymbol("ForeignCaller"), t->GetFunction());
}

Handle<Value> ForeignCaller::New(const Arguments& args) {
  HandleScope     scope;
  ForeignCaller   *self = new ForeignCaller();

  if (args.Length() == 5) {
    Pointer *cif    = ObjectWrap::Unwrap<Pointer>(args[0]->ToObject());
    Pointer *fn     = ObjectWrap::Unwrap<Pointer>(args[1]->ToObject());
    Pointer *fnargs = ObjectWrap::Unwrap<Pointer>(args[2]->ToObject());
    Pointer *res    = ObjectWrap::Unwrap<Pointer>(args[3]->ToObject());

    self->m_cif     = (ffi_cif *)cif->GetPointer();
    self->m_fn      = (void (*)(void))fn->GetPointer();
    self->m_res     = (void *)res->GetPointer();
    self->m_fnargs  = (void **)fnargs->GetPointer();

    self->m_async   = args[4]->BooleanValue();
  } else {
    return THROW_ERROR_EXCEPTION("Not enough arguments");
  }

  self->Wrap(args.This());
  return scope.Close(args.This());
}

Handle<Value> ForeignCaller::Exec(const Arguments& args) {
  HandleScope     scope;
  ForeignCaller   *self = ObjectWrap::Unwrap<ForeignCaller>(args.This());

  if (self->m_async) {
    AsyncCallParams *p = new AsyncCallParams();

    // cuter way of doing this?
    p->cif  = self->m_cif;
    p->ptr  = self->m_fn;
    p->res  = self->m_res;
    p->args = self->m_fnargs;

    // get the events.EventEmitter constructor
    Local<Object> global = Context::GetCurrent()->Global();
    Local<Object> events = global->Get(String::NewSymbol("process"))->ToObject();
    Local<Function> emitterConstructor = Local<Function>::Cast(events->Get(String::NewSymbol("EventEmitter")));

    // construct a new EventEmitter object
    p->emitter = Persistent<Object>::New(emitterConstructor->NewInstance());

    BEGIN_ASYNC(p, ForeignCaller::AsyncFFICall, ForeignCaller::FinishAsyncFFICall);
    return scope.Close(p->emitter);
  } else {
#if __OBJC__ || __OBJC2__
    @try {
#endif
      ffi_call(
          self->m_cif,
          self->m_fn,
          self->m_res,
          self->m_fnargs
          );
#if __OBJC__ || __OBJC2__
    } @catch (id ex) {
      return ThrowException(Pointer::WrapPointer((unsigned char *)ex));
    }
#endif
  }

  return Undefined();
}

async_rtn ForeignCaller::AsyncFFICall(uv_work_t *req) {
  AsyncCallParams *p = (AsyncCallParams *)req->data;
  ffi_call(p->cif, p->ptr, p->res, p->args);
  RETURN_ASYNC;
}

async_rtn ForeignCaller::FinishAsyncFFICall(uv_work_t *req) {
  HandleScope scope;

  AsyncCallParams *p = (AsyncCallParams *)req->data;
  Local<Value> argv[1];

  argv[0] = Local<Value>::New(String::New("success"));

  // get a reference to the 'emit' function
  Local<Value> emitVal = p->emitter->Get(String::NewSymbol("emit"));
  Local<Function> emit = Local<Function>::Cast(emitVal);

  TryCatch try_catch;

  // emit a success event
  emit->Call(p->emitter, 1, argv);

  // dispose of our persistent handle to the EventEmitter object
  p->emitter.Dispose();

  // free up our memory (allocated in FFICall)
  delete p;

  if (try_catch.HasCaught()) {
    FatalException(try_catch);
  }

  RETURN_ASYNC_AFTER;
}
<|endoftext|>


<commit_before>// Copyright (C) 2011 The Libphonenumber Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Author: Philippe Liard

#include <iostream>

#include <gtest/gtest.h>

#include "phonenumbers/unicodestring.h"

using std::ostream;

// Used by GTest to print the expected and actual results in case of failure.
ostream& operator<<(ostream& out, const i18n::phonenumbers::UnicodeString& s) {
  string utf8;
  s.toUTF8String(utf8);
  out << utf8;
  return out;
}

namespace i18n {
namespace phonenumbers {

TEST(UnicodeString, ToUTF8StringWithEmptyString) {
  UnicodeString s;
  string utf8;
  s.toUTF8String(utf8);
  EXPECT_EQ("", utf8);
}

TEST(UnicodeString, ToUTF8String) {
  UnicodeString s("hello");
  string utf8;
  s.toUTF8String(utf8);
  EXPECT_EQ("hello", utf8);
}

TEST(UnicodeString, ToUTF8StringWithNonAscii) {
  UnicodeString s("\xEF\xBC\x95\xEF\xBC\x93" /* "５３" */);
  string utf8;
  s.toUTF8String(utf8);
  EXPECT_EQ("\xEF\xBC\x95\xEF\xBC\x93", utf8);
}

TEST(UnicodeString, AppendCodepoint) {
  UnicodeString s;
  s.append('h');
  ASSERT_EQ(UnicodeString("h"), s);
  s.append('e');
  EXPECT_EQ(UnicodeString("he"), s);
}

TEST(UnicodeString, AppendCodepointWithNonAscii) {
  UnicodeString s;
  s.append(0xFF15 /* ５ */);
  ASSERT_EQ(UnicodeString("\xEF\xBC\x95" /* ５ */), s);
  s.append(0xFF13 /* ３ */);
  EXPECT_EQ(UnicodeString("\xEF\xBC\x95\xEF\xBC\x93" /* ５３ */), s);
}

TEST(UnicodeString, AppendUnicodeString) {
  UnicodeString s;
  s.append(UnicodeString("he"));
  ASSERT_EQ(UnicodeString("he"), s);
  s.append(UnicodeString("llo"));
  EXPECT_EQ(UnicodeString("hello"), s);
}

TEST(UnicodeString, AppendUnicodeStringWithNonAscii) {
  UnicodeString s;
  s.append(UnicodeString("\xEF\xBC\x95" /* ５ */));
  ASSERT_EQ(UnicodeString("\xEF\xBC\x95"), s);
  s.append(UnicodeString("\xEF\xBC\x93" /* ３ */));
  EXPECT_EQ(UnicodeString("\xEF\xBC\x95\xEF\xBC\x93" /* ５３ */), s);
}

TEST(UnicodeString, IndexOf) {
  UnicodeString s("hello");
  EXPECT_EQ(0, s.indexOf('h'));
  EXPECT_EQ(2, s.indexOf('l'));
  EXPECT_EQ(4, s.indexOf('o'));
}

TEST(UnicodeString, IndexOfWithNonAscii) {
  UnicodeString s("\xEF\xBC\x95\xEF\xBC\x93" /* ５３ */);
  EXPECT_EQ(1, s.indexOf(0xFF13 /* ３ */));
}

TEST(UnicodeString, ReplaceWithEmptyInputs) {
  UnicodeString s;
  s.replace(0, 0, UnicodeString(""));
  EXPECT_EQ(UnicodeString(""), s);
}

TEST(UnicodeString, ReplaceWithEmptyReplacement) {
  UnicodeString s("hello");
  s.replace(0, 5, UnicodeString(""));
  EXPECT_EQ(UnicodeString(""), s);
}

TEST(UnicodeString, ReplaceBegining) {
  UnicodeString s("hello world");
  s.replace(0, 5, UnicodeString("HELLO"));
  EXPECT_EQ(UnicodeString("HELLO world"), s);
}

TEST(UnicodeString, ReplaceMiddle) {
  UnicodeString s("hello world");
  s.replace(5, 1, UnicodeString("AB"));
  EXPECT_EQ(UnicodeString("helloABworld"), s);
}

TEST(UnicodeString, ReplaceEnd) {
  UnicodeString s("hello world");
  s.replace(10, 1, UnicodeString("AB"));
  EXPECT_EQ(UnicodeString("hello worlAB"), s);
}

TEST(UnicodeString, ReplaceWithNonAscii) {
  UnicodeString s("hello world");
  s.replace(3, 2, UnicodeString("\xEF\xBC\x91\xEF\xBC\x90" /* １０ */));
  EXPECT_EQ(UnicodeString("hel\xEF\xBC\x91\xEF\xBC\x90 world"), s);
}

TEST(UnicodeString, SetCharBegining) {
  UnicodeString s("hello");
  s.setCharAt(0, 'H');
  EXPECT_EQ(UnicodeString("Hello"), s);
}

TEST(UnicodeString, SetCharMiddle) {
  UnicodeString s("hello");
  s.setCharAt(2, 'L');
  EXPECT_EQ(UnicodeString("heLlo"), s);
}

TEST(UnicodeString, SetCharEnd) {
  UnicodeString s("hello");
  s.setCharAt(4, 'O');
  EXPECT_EQ(UnicodeString("hellO"), s);
}

TEST(UnicodeString, SetCharWithNonAscii) {
  UnicodeString s("hello");
  s.setCharAt(4, 0xFF10 /* ０ */);
  EXPECT_EQ(UnicodeString("hell\xEF\xBC\x90" /* ０ */), s);
}

TEST(UnicodeString, TempSubStringWithEmptyString) {
  EXPECT_EQ(UnicodeString(""), UnicodeString().tempSubString(0, 0));
}

TEST(UnicodeString, TempSubStringWithInvalidInputs) {
  UnicodeString s("hello");
  // tempSubString() returns an empty unicode string if one of the provided
  // paramaters is out of range.
  EXPECT_EQ(UnicodeString(""), s.tempSubString(6));
  EXPECT_EQ(UnicodeString(""), s.tempSubString(2, 6));
}

TEST(UnicodeString, TempSubString) {
  UnicodeString s("hello");
  EXPECT_EQ(UnicodeString(""), s.tempSubString(0, 0));
  EXPECT_EQ(UnicodeString("h"), s.tempSubString(0, 1));
  EXPECT_EQ(UnicodeString("hello"), s.tempSubString(0, 5));
  EXPECT_EQ(UnicodeString("llo"), s.tempSubString(2, 3));
}

TEST(UnicodeString, TempSubStringWithNoLength) {
  UnicodeString s("hello");
  EXPECT_EQ(UnicodeString("hello"), s.tempSubString(0));
  EXPECT_EQ(UnicodeString("llo"), s.tempSubString(2));
}

TEST(UnicodeString, TempSubStringWithNonAscii) {
  UnicodeString s("hel\xEF\xBC\x91\xEF\xBC\x90" /* １０ */);
  EXPECT_EQ(UnicodeString("\xEF\xBC\x91" /* １ */), s.tempSubString(3, 1));
}

TEST(UnicodeString, OperatorEqual) {
  UnicodeString s("hello");
  s = UnicodeString("Hello");
  EXPECT_EQ(UnicodeString("Hello"), s);
}

TEST(UnicodeString, OperatorEqualWithNonAscii) {
  UnicodeString s("hello");
  s = UnicodeString("hel\xEF\xBC\x91\xEF\xBC\x90" /* １０ */);
  EXPECT_EQ(UnicodeString("hel\xEF\xBC\x91\xEF\xBC\x90"), s);
}

TEST(UnicodeString, OperatorBracket) {
  UnicodeString s("hello");
  EXPECT_EQ('h', s[0]);
  EXPECT_EQ('e', s[1]);
  EXPECT_EQ('l', s[2]);
  EXPECT_EQ('l', s[3]);
  EXPECT_EQ('o', s[4]);
}

TEST(UnicodeString, OperatorBracketWithNonAscii) {
  UnicodeString s("hel\xEF\xBC\x91\xEF\xBC\x90" /* １０ */);
  EXPECT_EQ('h', s[0]);
  EXPECT_EQ('e', s[1]);
  EXPECT_EQ('l', s[2]);
  EXPECT_EQ(0xFF11 /* １ */, s[3]);
  EXPECT_EQ(0xFF10 /* ０ */, s[4]);
}

TEST(UnicodeString, OperatorBracketWithIteratorCacheInvalidation) {
  UnicodeString s("hello");
  EXPECT_EQ('h', s[0]);
  EXPECT_EQ('e', s[1]);
  // Modify the string which should invalidate the iterator cache.
  s.setCharAt(1, 'E');
  EXPECT_EQ(UnicodeString("hEllo"), s);
  EXPECT_EQ('E', s[1]);
  // Get the previous character which should invalidate the iterator cache.
  EXPECT_EQ('h', s[0]);
  EXPECT_EQ('o', s[4]);
}

}  // namespace phonenumbers
}  // namespace i18n
<commit_msg>CPP: Fix compilation error on Clang.<commit_after>// Copyright (C) 2011 The Libphonenumber Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Author: Philippe Liard

#include <iostream>

#include <gtest/gtest.h>

#include "phonenumbers/unicodestring.h"

using std::ostream;

namespace i18n {
namespace phonenumbers {

// Used by GTest to print the expected and actual results in case of failure.
ostream& operator<<(ostream& out, const UnicodeString& s) {
  string utf8;
  s.toUTF8String(utf8);
  out << utf8;
  return out;
}

TEST(UnicodeString, ToUTF8StringWithEmptyString) {
  UnicodeString s;
  string utf8;
  s.toUTF8String(utf8);
  EXPECT_EQ("", utf8);
}

TEST(UnicodeString, ToUTF8String) {
  UnicodeString s("hello");
  string utf8;
  s.toUTF8String(utf8);
  EXPECT_EQ("hello", utf8);
}

TEST(UnicodeString, ToUTF8StringWithNonAscii) {
  UnicodeString s("\xEF\xBC\x95\xEF\xBC\x93" /* "５３" */);
  string utf8;
  s.toUTF8String(utf8);
  EXPECT_EQ("\xEF\xBC\x95\xEF\xBC\x93", utf8);
}

TEST(UnicodeString, AppendCodepoint) {
  UnicodeString s;
  s.append('h');
  ASSERT_EQ(UnicodeString("h"), s);
  s.append('e');
  EXPECT_EQ(UnicodeString("he"), s);
}

TEST(UnicodeString, AppendCodepointWithNonAscii) {
  UnicodeString s;
  s.append(0xFF15 /* ５ */);
  ASSERT_EQ(UnicodeString("\xEF\xBC\x95" /* ５ */), s);
  s.append(0xFF13 /* ３ */);
  EXPECT_EQ(UnicodeString("\xEF\xBC\x95\xEF\xBC\x93" /* ５３ */), s);
}

TEST(UnicodeString, AppendUnicodeString) {
  UnicodeString s;
  s.append(UnicodeString("he"));
  ASSERT_EQ(UnicodeString("he"), s);
  s.append(UnicodeString("llo"));
  EXPECT_EQ(UnicodeString("hello"), s);
}

TEST(UnicodeString, AppendUnicodeStringWithNonAscii) {
  UnicodeString s;
  s.append(UnicodeString("\xEF\xBC\x95" /* ５ */));
  ASSERT_EQ(UnicodeString("\xEF\xBC\x95"), s);
  s.append(UnicodeString("\xEF\xBC\x93" /* ３ */));
  EXPECT_EQ(UnicodeString("\xEF\xBC\x95\xEF\xBC\x93" /* ５３ */), s);
}

TEST(UnicodeString, IndexOf) {
  UnicodeString s("hello");
  EXPECT_EQ(0, s.indexOf('h'));
  EXPECT_EQ(2, s.indexOf('l'));
  EXPECT_EQ(4, s.indexOf('o'));
}

TEST(UnicodeString, IndexOfWithNonAscii) {
  UnicodeString s("\xEF\xBC\x95\xEF\xBC\x93" /* ５３ */);
  EXPECT_EQ(1, s.indexOf(0xFF13 /* ３ */));
}

TEST(UnicodeString, ReplaceWithEmptyInputs) {
  UnicodeString s;
  s.replace(0, 0, UnicodeString(""));
  EXPECT_EQ(UnicodeString(""), s);
}

TEST(UnicodeString, ReplaceWithEmptyReplacement) {
  UnicodeString s("hello");
  s.replace(0, 5, UnicodeString(""));
  EXPECT_EQ(UnicodeString(""), s);
}

TEST(UnicodeString, ReplaceBegining) {
  UnicodeString s("hello world");
  s.replace(0, 5, UnicodeString("HELLO"));
  EXPECT_EQ(UnicodeString("HELLO world"), s);
}

TEST(UnicodeString, ReplaceMiddle) {
  UnicodeString s("hello world");
  s.replace(5, 1, UnicodeString("AB"));
  EXPECT_EQ(UnicodeString("helloABworld"), s);
}

TEST(UnicodeString, ReplaceEnd) {
  UnicodeString s("hello world");
  s.replace(10, 1, UnicodeString("AB"));
  EXPECT_EQ(UnicodeString("hello worlAB"), s);
}

TEST(UnicodeString, ReplaceWithNonAscii) {
  UnicodeString s("hello world");
  s.replace(3, 2, UnicodeString("\xEF\xBC\x91\xEF\xBC\x90" /* １０ */));
  EXPECT_EQ(UnicodeString("hel\xEF\xBC\x91\xEF\xBC\x90 world"), s);
}

TEST(UnicodeString, SetCharBegining) {
  UnicodeString s("hello");
  s.setCharAt(0, 'H');
  EXPECT_EQ(UnicodeString("Hello"), s);
}

TEST(UnicodeString, SetCharMiddle) {
  UnicodeString s("hello");
  s.setCharAt(2, 'L');
  EXPECT_EQ(UnicodeString("heLlo"), s);
}

TEST(UnicodeString, SetCharEnd) {
  UnicodeString s("hello");
  s.setCharAt(4, 'O');
  EXPECT_EQ(UnicodeString("hellO"), s);
}

TEST(UnicodeString, SetCharWithNonAscii) {
  UnicodeString s("hello");
  s.setCharAt(4, 0xFF10 /* ０ */);
  EXPECT_EQ(UnicodeString("hell\xEF\xBC\x90" /* ０ */), s);
}

TEST(UnicodeString, TempSubStringWithEmptyString) {
  EXPECT_EQ(UnicodeString(""), UnicodeString().tempSubString(0, 0));
}

TEST(UnicodeString, TempSubStringWithInvalidInputs) {
  UnicodeString s("hello");
  // tempSubString() returns an empty unicode string if one of the provided
  // paramaters is out of range.
  EXPECT_EQ(UnicodeString(""), s.tempSubString(6));
  EXPECT_EQ(UnicodeString(""), s.tempSubString(2, 6));
}

TEST(UnicodeString, TempSubString) {
  UnicodeString s("hello");
  EXPECT_EQ(UnicodeString(""), s.tempSubString(0, 0));
  EXPECT_EQ(UnicodeString("h"), s.tempSubString(0, 1));
  EXPECT_EQ(UnicodeString("hello"), s.tempSubString(0, 5));
  EXPECT_EQ(UnicodeString("llo"), s.tempSubString(2, 3));
}

TEST(UnicodeString, TempSubStringWithNoLength) {
  UnicodeString s("hello");
  EXPECT_EQ(UnicodeString("hello"), s.tempSubString(0));
  EXPECT_EQ(UnicodeString("llo"), s.tempSubString(2));
}

TEST(UnicodeString, TempSubStringWithNonAscii) {
  UnicodeString s("hel\xEF\xBC\x91\xEF\xBC\x90" /* １０ */);
  EXPECT_EQ(UnicodeString("\xEF\xBC\x91" /* １ */), s.tempSubString(3, 1));
}

TEST(UnicodeString, OperatorEqual) {
  UnicodeString s("hello");
  s = UnicodeString("Hello");
  EXPECT_EQ(UnicodeString("Hello"), s);
}

TEST(UnicodeString, OperatorEqualWithNonAscii) {
  UnicodeString s("hello");
  s = UnicodeString("hel\xEF\xBC\x91\xEF\xBC\x90" /* １０ */);
  EXPECT_EQ(UnicodeString("hel\xEF\xBC\x91\xEF\xBC\x90"), s);
}

TEST(UnicodeString, OperatorBracket) {
  UnicodeString s("hello");
  EXPECT_EQ('h', s[0]);
  EXPECT_EQ('e', s[1]);
  EXPECT_EQ('l', s[2]);
  EXPECT_EQ('l', s[3]);
  EXPECT_EQ('o', s[4]);
}

TEST(UnicodeString, OperatorBracketWithNonAscii) {
  UnicodeString s("hel\xEF\xBC\x91\xEF\xBC\x90" /* １０ */);
  EXPECT_EQ('h', s[0]);
  EXPECT_EQ('e', s[1]);
  EXPECT_EQ('l', s[2]);
  EXPECT_EQ(0xFF11 /* １ */, s[3]);
  EXPECT_EQ(0xFF10 /* ０ */, s[4]);
}

TEST(UnicodeString, OperatorBracketWithIteratorCacheInvalidation) {
  UnicodeString s("hello");
  EXPECT_EQ('h', s[0]);
  EXPECT_EQ('e', s[1]);
  // Modify the string which should invalidate the iterator cache.
  s.setCharAt(1, 'E');
  EXPECT_EQ(UnicodeString("hEllo"), s);
  EXPECT_EQ('E', s[1]);
  // Get the previous character which should invalidate the iterator cache.
  EXPECT_EQ('h', s[0]);
  EXPECT_EQ('o', s[4]);
}

}  // namespace phonenumbers
}  // namespace i18n
<|endoftext|>