<commit_before>/****************************************************************************
**
** Copyright (C) 2012 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia.  For licensing terms and
** conditions see http://qt.digia.com/licensing.  For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights.  These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
****************************************************************************/

#include "kit.h"

#include "devicesupport/devicemanager.h"
#include "kitinformation.h"
#include "kitmanager.h"
#include "project.h"
#include "toolchainmanager.h"

#include <utils/qtcassert.h>

#include <QApplication>
#include <QIcon>
#include <QStyle>
#include <QTextStream>
#include <QUuid>

namespace {

const char ID_KEY[] = "PE.Profile.Id";
const char DISPLAYNAME_KEY[] = "PE.Profile.Name";
const char AUTODETECTED_KEY[] = "PE.Profile.AutoDetected";
const char DATA_KEY[] = "PE.Profile.Data";
const char ICON_KEY[] = "PE.Profile.Icon";

} // namespace

namespace ProjectExplorer {

// --------------------------------------------------------------------
// Helper:
// --------------------------------------------------------------------

static QString cleanName(const QString &name)
{
    QString result = name;
    result.replace(QRegExp("\\W"), QLatin1String("_"));
    result.replace(QRegExp("_+"), "_"); // compact _
    result.remove(QRegExp("^_*")); // remove leading _
    result.remove(QRegExp("_+$")); // remove trailing _
    if (result.isEmpty())
        result = QLatin1String("unknown");
    return result;
}

// -------------------------------------------------------------------------
// KitPrivate
// -------------------------------------------------------------------------

namespace Internal {

class KitPrivate
{
public:
    KitPrivate(Core::Id id) :
        m_id(id),
        m_autodetected(false),
        m_isValid(true),
        m_nestedBlockingLevel(0),
        m_mustNotify(false)
    {
        if (!id.isValid())
            m_id = Core::Id(QUuid::createUuid().toString().toLatin1().constData());
    }

    QString m_displayName;
    Core::Id m_id;
    bool m_autodetected;
    bool m_isValid;
    QIcon m_icon;
    QString m_iconPath;
    int m_nestedBlockingLevel;
    bool m_mustNotify;

    QHash<Core::Id, QVariant> m_data;
};

} // namespace Internal

// -------------------------------------------------------------------------
// Kit:
// -------------------------------------------------------------------------

Kit::Kit(Core::Id id) :
    d(new Internal::KitPrivate(id))
{
    KitManager *stm = KitManager::instance();
    KitGuard g(this);
    foreach (KitInformation *sti, stm->kitInformation())
        setValue(sti->dataId(), sti->defaultValue(this));

    setDisplayName(QCoreApplication::translate("ProjectExplorer::Kit", "Unnamed"));
    setIconPath(QLatin1String(":///DESKTOP///"));
}

Kit::~Kit()
{
    delete d;
}

void Kit::blockNotification()
{
    ++d->m_nestedBlockingLevel;
}

void Kit::unblockNotification()
{
    --d->m_nestedBlockingLevel;
    if (d->m_nestedBlockingLevel > 0)
        return;
    if (d->m_mustNotify)
        kitUpdated();
    d->m_mustNotify = false;
}

Kit *Kit::clone(bool keepName) const
{
    Kit *k = new Kit;
    if (keepName)
        k->d->m_displayName = d->m_displayName;
    else
        k->d->m_displayName = QCoreApplication::translate("ProjectExplorer::Kit", "Clone of %1")
                .arg(d->m_displayName);
    k->d->m_autodetected = false;
    k->d->m_data = d->m_data;
    k->d->m_isValid = d->m_isValid;
    k->d->m_icon = d->m_icon;
    k->d->m_iconPath = d->m_iconPath;
    return k;
}

void Kit::copyFrom(const Kit *k)
{
    KitGuard g(this);
    d->m_data = k->d->m_data;
    d->m_iconPath = k->d->m_iconPath;
    d->m_icon = k->d->m_icon;
    d->m_autodetected = k->d->m_autodetected;
    d->m_displayName = k->d->m_displayName;
    d->m_mustNotify = true;
}

bool Kit::isValid() const
{
    return d->m_id.isValid() && d->m_isValid;
}

QList<Task> Kit::validate() const
{
    QList<Task> result;
    QList<KitInformation *> infoList = KitManager::instance()->kitInformation();
    d->m_isValid = true;
    foreach (KitInformation *i, infoList) {
        QList<Task> tmp = i->validate(this);
        foreach (const Task &t, tmp) {
            if (t.type == Task::Error)
                d->m_isValid = false;
        }
        result.append(tmp);
    }
    qSort(result);
    return result;
}

void Kit::fix()
{
    KitGuard g(this);
    foreach (KitInformation *i, KitManager::instance()->kitInformation())
        i->fix(this);
}

QString Kit::displayName() const
{
    return d->m_displayName;
}

static QString candidateName(const QString &name, const QString &postfix)
{
    if (name.contains(postfix))
        return QString();
    QString candidate = name;
    if (!candidate.isEmpty())
        candidate.append(QLatin1Char('-'));
    candidate.append(postfix);
    return candidate;
}

void Kit::setDisplayName(const QString &name)
{
    KitManager *km = KitManager::instance();
    QList<KitInformation *> kitInfo = km->kitInformation();

    QStringList nameList;
    foreach (Kit *k, km->kits()) {
        if (k == this)
            continue;
        nameList << k->displayName();
        foreach (KitInformation *ki, kitInfo) {
            const QString postfix = ki->displayNamePostfix(k);
            if (!postfix.isEmpty())
                nameList << candidateName(k->displayName(), postfix);
        }
    }

    QStringList candidateNames;
    candidateNames << name;

    foreach (KitInformation *ki, kitInfo) {
        const QString postfix = ki->displayNamePostfix(this);
        if (!postfix.isEmpty())
            candidateNames << candidateName(name, postfix);
    }

    QString uniqueName = Project::makeUnique(name, nameList);
    if (uniqueName != name) {
        foreach (const QString &candidate, candidateNames) {
            const QString tmp = Project::makeUnique(candidate, nameList);
            if (tmp == candidate) {
                uniqueName = tmp;
                break;
            }
        }
    }

    if (d->m_displayName == uniqueName)
        return;
    d->m_displayName = uniqueName;
    kitUpdated();
}

QString Kit::fileSystemFriendlyName() const
{
    QString name = cleanName(displayName());
    foreach (Kit *i, KitManager::instance()->kits()) {
        if (i == this)
            continue;
        if (name == cleanName(i->displayName())) {
            // append part of the kit id: That should be unique enough;-)
            // Leading { will be turned into _ which should be fine.
            name = cleanName(name + QLatin1Char('_') + (id().toString().left(7)));
            break;
        }
    }
    return name;
}

bool Kit::isAutoDetected() const
{
    return d->m_autodetected;
}

Core::Id Kit::id() const
{
    return d->m_id;
}

QIcon Kit::icon() const
{
    return d->m_icon;
}

QString Kit::iconPath() const
{
    return d->m_iconPath;
}

void Kit::setIconPath(const QString &path)
{
    if (d->m_iconPath == path)
        return;
    d->m_iconPath = path;
    if (path.isNull())
        d->m_icon = QIcon();
    else if (path == QLatin1String(":///DESKTOP///"))
        d->m_icon = qApp->style()->standardIcon(QStyle::SP_ComputerIcon);
    else
        d->m_icon = QIcon(path);
    kitUpdated();
}

QVariant Kit::value(const Core::Id &key, const QVariant &unset) const
{
    return d->m_data.value(key, unset);
}

bool Kit::hasValue(const Core::Id &key) const
{
    return d->m_data.contains(key);
}

void Kit::setValue(const Core::Id &key, const QVariant &value)
{
    if (d->m_data.value(key) == value)
        return;
    d->m_data.insert(key, value);
    kitUpdated();
}

void Kit::removeKey(const Core::Id &key)
{
    if (!d->m_data.contains(key))
        return;
    d->m_data.remove(key);
    kitUpdated();
}

bool Kit::isDataEqual(const Kit *other) const
{
    return d->m_data == other->d->m_data;
}

bool Kit::isEqual(const Kit *other) const
{
    return isDataEqual(other)
            && d->m_iconPath == other->d->m_iconPath
            && d->m_displayName == other->d->m_displayName;
}

QVariantMap Kit::toMap() const
{
    QVariantMap data;
    data.insert(QLatin1String(ID_KEY), QString::fromLatin1(d->m_id.name()));
    data.insert(QLatin1String(DISPLAYNAME_KEY), d->m_displayName);
    data.insert(QLatin1String(AUTODETECTED_KEY), d->m_autodetected);
    data.insert(QLatin1String(ICON_KEY), d->m_iconPath);

    QVariantMap extra;
    foreach (const Core::Id &key, d->m_data.keys())
        extra.insert(QString::fromLatin1(key.name().constData()), d->m_data.value(key));
    data.insert(QLatin1String(DATA_KEY), extra);

    return data;
}

void Kit::addToEnvironment(Utils::Environment &env) const
{
    QList<KitInformation *> infoList = KitManager::instance()->kitInformation();
    foreach (KitInformation *ki, infoList)
        ki->addToEnvironment(this, env);
}

QString Kit::toHtml()
{
    QString rc;
    QTextStream str(&rc);
    str << "";
    str << "
" << displayName() << "
";
    str << "
";

    if (!isValid()) {
        QList<Task> issues = validate();
        str << "
";
        foreach (const Task &t, issues) {
            str << "";
            switch (t.type) {
            case Task::Error:
                str << QCoreApplication::translate("ProjectExplorer::Kit", "Error:");
                break;
            case Task::Warning:
                str << QCoreApplication::translate("ProjectExplorer::Kit", "Warning:");
                break;
            case Task::Unknown:
            default:
                break;
            }
            str << "" << t.description << "
";
        }
        str << "
";
    }

    QList<KitInformation *> infoList = KitManager::instance()->kitInformation();
    foreach (KitInformation *ki, infoList) {
        KitInformation::ItemList list = ki->toUserOutput(this);
        foreach (const KitInformation::Item &j, list)
            str << "
" << j.first << ":	" << j.second << "
";
    }
    str << "
";
    return rc;
}

bool Kit::fromMap(const QVariantMap &data)
{
    KitGuard g(this);
    const QString id = data.value(QLatin1String(ID_KEY)).toString();
    if (id.isEmpty())
        return false;
    d->m_id = Core::Id(id);
    d->m_autodetected = data.value(QLatin1String(AUTODETECTED_KEY)).toBool();
    setDisplayName(data.value(QLatin1String(DISPLAYNAME_KEY)).toString());
    setIconPath(data.value(QLatin1String(ICON_KEY)).toString());

    QVariantMap extra = data.value(QLatin1String(DATA_KEY)).toMap();
    foreach (const QString &key, extra.keys())
        setValue(Core::Id(key), extra.value(key));

    return true;
}

void Kit::setAutoDetected(bool detected)
{
    d->m_autodetected = detected;
}

void Kit::kitUpdated()
{
    if (d->m_nestedBlockingLevel > 0) {
        d->m_mustNotify = true;
        return;
    }
    validate();
    KitManager::instance()->notifyAboutUpdate(this);
}

} // namespace ProjectExplorer
<commit_msg>Kits: Do not allow kits with empty names<commit_after>/****************************************************************************
**
** Copyright (C) 2012 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia.  For licensing terms and
** conditions see http://qt.digia.com/licensing.  For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights.  These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
****************************************************************************/

#include "kit.h"

#include "devicesupport/devicemanager.h"
#include "kitinformation.h"
#include "kitmanager.h"
#include "project.h"
#include "toolchainmanager.h"

#include <utils/qtcassert.h>

#include <QApplication>
#include <QIcon>
#include <QStyle>
#include <QTextStream>
#include <QUuid>

namespace {

const char ID_KEY[] = "PE.Profile.Id";
const char DISPLAYNAME_KEY[] = "PE.Profile.Name";
const char AUTODETECTED_KEY[] = "PE.Profile.AutoDetected";
const char DATA_KEY[] = "PE.Profile.Data";
const char ICON_KEY[] = "PE.Profile.Icon";

} // namespace

namespace ProjectExplorer {

// --------------------------------------------------------------------
// Helper:
// --------------------------------------------------------------------

static QString cleanName(const QString &name)
{
    QString result = name;
    result.replace(QRegExp("\\W"), QLatin1String("_"));
    result.replace(QRegExp("_+"), "_"); // compact _
    result.remove(QRegExp("^_*")); // remove leading _
    result.remove(QRegExp("_+$")); // remove trailing _
    if (result.isEmpty())
        result = QLatin1String("unknown");
    return result;
}

// -------------------------------------------------------------------------
// KitPrivate
// -------------------------------------------------------------------------

namespace Internal {

class KitPrivate
{
public:
    KitPrivate(Core::Id id) :
        m_id(id),
        m_autodetected(false),
        m_isValid(true),
        m_nestedBlockingLevel(0),
        m_mustNotify(false)
    {
        if (!id.isValid())
            m_id = Core::Id(QUuid::createUuid().toString().toLatin1().constData());
    }

    QString m_displayName;
    Core::Id m_id;
    bool m_autodetected;
    bool m_isValid;
    QIcon m_icon;
    QString m_iconPath;
    int m_nestedBlockingLevel;
    bool m_mustNotify;

    QHash<Core::Id, QVariant> m_data;
};

} // namespace Internal

// -------------------------------------------------------------------------
// Kit:
// -------------------------------------------------------------------------

Kit::Kit(Core::Id id) :
    d(new Internal::KitPrivate(id))
{
    KitManager *stm = KitManager::instance();
    KitGuard g(this);
    foreach (KitInformation *sti, stm->kitInformation())
        setValue(sti->dataId(), sti->defaultValue(this));

    setDisplayName(QCoreApplication::translate("ProjectExplorer::Kit", "Unnamed"));
    setIconPath(QLatin1String(":///DESKTOP///"));
}

Kit::~Kit()
{
    delete d;
}

void Kit::blockNotification()
{
    ++d->m_nestedBlockingLevel;
}

void Kit::unblockNotification()
{
    --d->m_nestedBlockingLevel;
    if (d->m_nestedBlockingLevel > 0)
        return;
    if (d->m_mustNotify)
        kitUpdated();
    d->m_mustNotify = false;
}

Kit *Kit::clone(bool keepName) const
{
    Kit *k = new Kit;
    if (keepName)
        k->d->m_displayName = d->m_displayName;
    else
        k->d->m_displayName = QCoreApplication::translate("ProjectExplorer::Kit", "Clone of %1")
                .arg(d->m_displayName);
    k->d->m_autodetected = false;
    k->d->m_data = d->m_data;
    k->d->m_isValid = d->m_isValid;
    k->d->m_icon = d->m_icon;
    k->d->m_iconPath = d->m_iconPath;
    return k;
}

void Kit::copyFrom(const Kit *k)
{
    KitGuard g(this);
    d->m_data = k->d->m_data;
    d->m_iconPath = k->d->m_iconPath;
    d->m_icon = k->d->m_icon;
    d->m_autodetected = k->d->m_autodetected;
    d->m_displayName = k->d->m_displayName;
    d->m_mustNotify = true;
}

bool Kit::isValid() const
{
    return d->m_id.isValid() && d->m_isValid;
}

QList<Task> Kit::validate() const
{
    QList<Task> result;
    QList<KitInformation *> infoList = KitManager::instance()->kitInformation();
    d->m_isValid = true;
    foreach (KitInformation *i, infoList) {
        QList<Task> tmp = i->validate(this);
        foreach (const Task &t, tmp) {
            if (t.type == Task::Error)
                d->m_isValid = false;
        }
        result.append(tmp);
    }
    qSort(result);
    return result;
}

void Kit::fix()
{
    KitGuard g(this);
    foreach (KitInformation *i, KitManager::instance()->kitInformation())
        i->fix(this);
}

QString Kit::displayName() const
{
    return d->m_displayName;
}

static QString candidateName(const QString &name, const QString &postfix)
{
    if (name.contains(postfix))
        return QString();
    QString candidate = name;
    if (!candidate.isEmpty())
        candidate.append(QLatin1Char('-'));
    candidate.append(postfix);
    return candidate;
}

void Kit::setDisplayName(const QString &name)
{
    KitManager *km = KitManager::instance();
    QList<KitInformation *> kitInfo = km->kitInformation();

    QStringList nameList;
    nameList << QString(); // Disallow empty kit names!
    foreach (Kit *k, km->kits()) {
        if (k == this)
            continue;
        nameList << k->displayName();
        foreach (KitInformation *ki, kitInfo) {
            const QString postfix = ki->displayNamePostfix(k);
            if (!postfix.isEmpty())
                nameList << candidateName(k->displayName(), postfix);
        }
    }

    QStringList candidateNames;
    candidateNames << name;

    foreach (KitInformation *ki, kitInfo) {
        const QString postfix = ki->displayNamePostfix(this);
        if (!postfix.isEmpty())
            candidateNames << candidateName(name, postfix);
    }

    QString uniqueName = Project::makeUnique(name, nameList);
    if (uniqueName != name) {
        foreach (const QString &candidate, candidateNames) {
            const QString tmp = Project::makeUnique(candidate, nameList);
            if (tmp == candidate) {
                uniqueName = tmp;
                break;
            }
        }
    }

    if (d->m_displayName == uniqueName)
        return;
    d->m_displayName = uniqueName;
    kitUpdated();
}

QString Kit::fileSystemFriendlyName() const
{
    QString name = cleanName(displayName());
    foreach (Kit *i, KitManager::instance()->kits()) {
        if (i == this)
            continue;
        if (name == cleanName(i->displayName())) {
            // append part of the kit id: That should be unique enough;-)
            // Leading { will be turned into _ which should be fine.
            name = cleanName(name + QLatin1Char('_') + (id().toString().left(7)));
            break;
        }
    }
    return name;
}

bool Kit::isAutoDetected() const
{
    return d->m_autodetected;
}

Core::Id Kit::id() const
{
    return d->m_id;
}

QIcon Kit::icon() const
{
    return d->m_icon;
}

QString Kit::iconPath() const
{
    return d->m_iconPath;
}

void Kit::setIconPath(const QString &path)
{
    if (d->m_iconPath == path)
        return;
    d->m_iconPath = path;
    if (path.isNull())
        d->m_icon = QIcon();
    else if (path == QLatin1String(":///DESKTOP///"))
        d->m_icon = qApp->style()->standardIcon(QStyle::SP_ComputerIcon);
    else
        d->m_icon = QIcon(path);
    kitUpdated();
}

QVariant Kit::value(const Core::Id &key, const QVariant &unset) const
{
    return d->m_data.value(key, unset);
}

bool Kit::hasValue(const Core::Id &key) const
{
    return d->m_data.contains(key);
}

void Kit::setValue(const Core::Id &key, const QVariant &value)
{
    if (d->m_data.value(key) == value)
        return;
    d->m_data.insert(key, value);
    kitUpdated();
}

void Kit::removeKey(const Core::Id &key)
{
    if (!d->m_data.contains(key))
        return;
    d->m_data.remove(key);
    kitUpdated();
}

bool Kit::isDataEqual(const Kit *other) const
{
    return d->m_data == other->d->m_data;
}

bool Kit::isEqual(const Kit *other) const
{
    return isDataEqual(other)
            && d->m_iconPath == other->d->m_iconPath
            && d->m_displayName == other->d->m_displayName;
}

QVariantMap Kit::toMap() const
{
    QVariantMap data;
    data.insert(QLatin1String(ID_KEY), QString::fromLatin1(d->m_id.name()));
    data.insert(QLatin1String(DISPLAYNAME_KEY), d->m_displayName);
    data.insert(QLatin1String(AUTODETECTED_KEY), d->m_autodetected);
    data.insert(QLatin1String(ICON_KEY), d->m_iconPath);

    QVariantMap extra;
    foreach (const Core::Id &key, d->m_data.keys())
        extra.insert(QString::fromLatin1(key.name().constData()), d->m_data.value(key));
    data.insert(QLatin1String(DATA_KEY), extra);

    return data;
}

void Kit::addToEnvironment(Utils::Environment &env) const
{
    QList<KitInformation *> infoList = KitManager::instance()->kitInformation();
    foreach (KitInformation *ki, infoList)
        ki->addToEnvironment(this, env);
}

QString Kit::toHtml()
{
    QString rc;
    QTextStream str(&rc);
    str << "";
    str << "
" << displayName() << "
";
    str << "
";

    if (!isValid()) {
        QList<Task> issues = validate();
        str << "
";
        foreach (const Task &t, issues) {
            str << "";
            switch (t.type) {
            case Task::Error:
                str << QCoreApplication::translate("ProjectExplorer::Kit", "Error:");
                break;
            case Task::Warning:
                str << QCoreApplication::translate("ProjectExplorer::Kit", "Warning:");
                break;
            case Task::Unknown:
            default:
                break;
            }
            str << "" << t.description << "
";
        }
        str << "
";
    }

    QList<KitInformation *> infoList = KitManager::instance()->kitInformation();
    foreach (KitInformation *ki, infoList) {
        KitInformation::ItemList list = ki->toUserOutput(this);
        foreach (const KitInformation::Item &j, list)
            str << "
" << j.first << ":	" << j.second << "
";
    }
    str << "
";
    return rc;
}

bool Kit::fromMap(const QVariantMap &data)
{
    KitGuard g(this);
    const QString id = data.value(QLatin1String(ID_KEY)).toString();
    if (id.isEmpty())
        return false;
    d->m_id = Core::Id(id);
    d->m_autodetected = data.value(QLatin1String(AUTODETECTED_KEY)).toBool();
    setDisplayName(data.value(QLatin1String(DISPLAYNAME_KEY)).toString());
    setIconPath(data.value(QLatin1String(ICON_KEY)).toString());

    QVariantMap extra = data.value(QLatin1String(DATA_KEY)).toMap();
    foreach (const QString &key, extra.keys())
        setValue(Core::Id(key), extra.value(key));

    return true;
}

void Kit::setAutoDetected(bool detected)
{
    d->m_autodetected = detected;
}

void Kit::kitUpdated()
{
    if (d->m_nestedBlockingLevel > 0) {
        d->m_mustNotify = true;
        return;
    }
    validate();
    KitManager::instance()->notifyAboutUpdate(this);
}

} // namespace ProjectExplorer
<|endoftext|>


<commit_before>/* Copyright 2019 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "ml_metadata/metadata_store/mysql_metadata_source.h"

#include <string>
#include <utility>

#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "ml_metadata/metadata_store/constants.h"
#include "ml_metadata/metadata_store/types.h"
#include "ml_metadata/proto/metadata_source.pb.h"
#include "mysql.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"

namespace ml_metadata {

namespace {

namespace errors = tensorflow::errors;

using ::tensorflow::Status;

constexpr char kBeginTransaction[] = "START TRANSACTION";
constexpr char kCommitTransaction[] = "COMMIT";
constexpr char kRollbackTransaction[] = "ROLLBACK";

// A class that invokes mysql_thread_init() when constructed, and
// mysql_thread_end() when destructed.  It can be used as a
// thread_local to ensure that this happens exactly once per thread
// (see ThreadInitAccess).
class ThreadInitializer {
 public:
  ThreadInitializer() : initialized_(!mysql_thread_init()) {
    // Note: mysql_thread_init returns 0 on success
  }

  // Disallow copy-and-assign.
  ThreadInitializer(const ThreadInitializer&) = delete;
  ThreadInitializer& operator=(const ThreadInitializer&) = delete;

  ~ThreadInitializer() {
    if (initialized_) mysql_thread_end();
  }

  bool initialized() const { return initialized_; }

 private:
  // Indicates whether the call to mysql_thread_init was successfull.
  const bool initialized_;
};

// Initializes a ThreadInitializer for the current thread (if not already done).
// Returns an error if the MYSQL thread initialization fails.
//
// Note: All top-level methods that are invoked by the MetadataSource base class
// need to invoke ThreadInitAccess.
Status ThreadInitAccess() {
  // Cause mysql_thread_init() to be invoked exactly once per thread
  // the first time ThreadInitAccess() is called, and
  // mysql_thread_end() exactly once when the thread dies.
  thread_local ThreadInitializer initializer;
  if (!initializer.initialized()) {
    return errors::Internal("mysql thread initialization not done");
  }
  return Status::OK();
}

// Checks if config is valid.
Status CheckConfig(const MySQLDatabaseConfig& config) {
  std::vector<std::string> config_errors;
  if (config.host().empty() == config.socket().empty()) {
    config_errors.push_back("exactly one of host or socket must be specified");
  }
  if (config.database().empty()) {
    config_errors.push_back("database must not be empty");
  }

  if (!config_errors.empty()) {
    return errors::InvalidArgument(absl::StrJoin(config_errors, ";"));
  }
  return Status::OK();
}

}  // namespace

MySqlMetadataSource::MySqlMetadataSource(const MySQLDatabaseConfig& config)
    : MetadataSource(), config_(config) {
  TF_CHECK_OK(CheckConfig(config));
}

MySqlMetadataSource::~MySqlMetadataSource() { TF_CHECK_OK(CloseImpl()); }

Status MySqlMetadataSource::ConnectImpl() {
  // Initialize the MYSQL object.
  db_ = mysql_init(nullptr);
  if (!db_) {
    LOG(ERROR) << "MySQL error: " << mysql_errno(db_) << ": "
               << mysql_error(db_);
    return errors::Internal("mysql_init failed: errno: ",
                            mysql_errno(db_), ", error: ", mysql_error(db_));
  }

  // Explicitly setup the thread-local initializer.
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ThreadInitAccess(),
                                  "MySql thread init failed at ConnectImpl");

  // Set connection options
  if (config_.has_ssl_options()) {
    const MySQLDatabaseConfig::SSLOptions& ssl = config_.ssl_options();
    // The method set mysql_options, and always return 0. The connection options
    // are used in the `mysql_real_connect`.
    mysql_ssl_set(db_, ssl.key().empty() ? nullptr : ssl.key().c_str(),
                  ssl.cert().empty() ? nullptr : ssl.cert().c_str(),
                  ssl.ca().empty() ? nullptr : ssl.ca().c_str(),
                  ssl.capath().empty() ? nullptr : ssl.capath().c_str(),
                  ssl.cipher().empty() ? nullptr : ssl.cipher().c_str());
    my_bool verify_server_cert = ssl.verify_server_cert() ? 1 : 0;
    mysql_options(db_, MYSQL_OPT_SSL_VERIFY_SERVER_CERT, &verify_server_cert);
  }

  // Connect to the MYSQL server.
  if (!mysql_real_connect(
          db_, config_.host().empty() ? nullptr : config_.host().c_str(),
          config_.user().empty() ? nullptr : config_.user().c_str(),
          config_.password().empty() ? nullptr : config_.password().c_str(),
          /*db=*/nullptr, config_.port(),
          config_.socket().empty() ? nullptr : config_.socket().c_str(),
          /*clientflag=*/0UL)) {
    return errors::Internal("mysql_real_connect failed: errno: ",
                            mysql_errno(db_), ", error: ", mysql_error(db_));
  }

  // Return an error if the default storage engine doesn't support transactions.
  TF_RETURN_WITH_CONTEXT_IF_ERROR(
      CheckTransactionSupport(),
      "checking transaction support of default storage engine");

  // Create the database if not already present and switch to it.
  const std::string create_database_cmd =
      absl::StrCat("CREATE DATABASE IF NOT EXISTS ", config_.database());
  TF_RETURN_WITH_CONTEXT_IF_ERROR(RunQuery(create_database_cmd),
                                  "Creating database ", config_.database(),
                                  " in ConnectImpl");
  const std::string use_database_cmd = absl::StrCat("USE ", config_.database());
  TF_RETURN_WITH_CONTEXT_IF_ERROR(RunQuery(use_database_cmd),
                                  "Changing to database ", config_.database(),
                                  " in ConnectImpl");

  return Status::OK();
}

Status MySqlMetadataSource::CloseImpl() {
  if (db_ != nullptr) {
    TF_RETURN_IF_ERROR(ThreadInitAccess());
    DiscardResultSet();
    mysql_close(db_);
    db_ = nullptr;
  }
  return Status::OK();
}

Status MySqlMetadataSource::ExecuteQueryImpl(const std::string& query,
                                             RecordSet* results) {
  TF_RETURN_WITH_CONTEXT_IF_ERROR(
      ThreadInitAccess(), "MySql thread init failed at ExecuteQueryImpl");

  // Run the query.
  Status status = RunQuery(query);

  // Return on failure.
  TF_RETURN_IF_ERROR(status);

  // If query is successfull, convert the results.
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ConvertMySqlRowSetToRecordSet(results),
                                  "ConvertMySqlRowSetToRecordSet for query ",
                                  query);
  return Status::OK();
}

Status MySqlMetadataSource::CommitImpl() {
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ThreadInitAccess(),
                                  "MySql thread init failed at CommitImpl");
  return RunQuery(kCommitTransaction);
}

Status MySqlMetadataSource::RollbackImpl() {
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ThreadInitAccess(),
                                  "MySql thread init failed at RollbackImpl");
  return RunQuery(kRollbackTransaction);
}

Status MySqlMetadataSource::BeginImpl() {
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ThreadInitAccess(),
                                  "MySql thread init failed at BeginImpl");
  return RunQuery(kBeginTransaction);
}

Status MySqlMetadataSource::CheckTransactionSupport() {
  constexpr char kCheckTransactionSupport[] =
      "SELECT ENGINE, TRANSACTIONS FROM INFORMATION_SCHEMA.ENGINES WHERE "
      "ENGINE=(SELECT @@default_storage_engine)";

  TF_RETURN_IF_ERROR(RunQuery(kCheckTransactionSupport));

  RecordSet record_set;
  TF_RETURN_IF_ERROR(ConvertMySqlRowSetToRecordSet(&record_set));
  if (record_set.records_size() != 1 ||
      record_set.records(0).values_size() != 2) {
    return errors::Internal(
        "Expected query ", kCheckTransactionSupport,
        " to generate exactly single row with 2 columns, but got ",
        record_set.DebugString());
  }
  const RecordSet::Record& record = record_set.records(0);
  if (record.values(1) != "YES") {
    return errors::Internal(
        "no transaction support for default_storage_engine ", record.values(0));
  }

  return Status::OK();
}

Status MySqlMetadataSource::RunQuery(const std::string& query) {
  DiscardResultSet();

  int query_status = mysql_query(db_, query.c_str());
  if (query_status) {
    int64 error_number = mysql_errno(db_);
    // 2006: sever closes the connection due to inactive client;
    // client reports server has gone away, we reconnect the server for the
    // client if the query is begin transaction.
    if (error_number == 2006 && query == kBeginTransaction) {
      TF_RETURN_IF_ERROR(CloseImpl());
      TF_RETURN_IF_ERROR(ConnectImpl());
      return RunQuery(query);
    }
    // 1213: inno db aborts deadlock when running concurrent transactions.
    // returns Aborted for client side to retry.
    if (error_number == 1213 || error_number == 1205) {
      return errors::Aborted("mysql_query aborted: errno: ", error_number,
                             ", error: ", mysql_error(db_));
    }
    return errors::Internal("mysql_query failed: errno: ", error_number,
                            ", error: ", mysql_error(db_));
  }

  result_set_ = mysql_store_result(db_);
  if (!result_set_ && mysql_field_count(db_) != 0) {
    return errors::Internal("mysql_query ", query,
                            " returned an unexpected NULL result_set: Errno: ",
                            mysql_errno(db_), ", Error: ", mysql_error(db_));
  }

  return Status::OK();
}

void MySqlMetadataSource::DiscardResultSet() {
  if (result_set_ != nullptr) {
    // Fetch any leftover rows (MySQL requires this).
    // Note: Since we always store the query result locally in RunQuery(),
    // we might not need to explicitly fetch the row here. But fetching rows
    // here won't hurt and will be a bit more robust in case we decide to lazily
    // fetch the rows from the server.
    while (mysql_fetch_row(result_set_)) {
    }
    mysql_free_result(result_set_);
    result_set_ = nullptr;
  }
}

Status MySqlMetadataSource::ConvertMySqlRowSetToRecordSet(
    RecordSet* record_set_out) {
  RecordSet record_set;

  if (result_set_ == nullptr) {
    return Status::OK();
  }

  MYSQL_ROW row;
  while ((row = mysql_fetch_row(result_set_)) != nullptr) {
    RecordSet::Record record;
    std::vector<std::string> col_names;

    uint32 num_cols = mysql_num_fields(result_set_);
    for (uint32 col = 0; col < num_cols; ++col) {
      MYSQL_FIELD* field = mysql_fetch_field_direct(result_set_, col);
      if (field == nullptr) {
        return errors::Internal(
            "Error in retrieving column description for index ", col);
      }
      const std::string col_name(field->org_name);
      if (record_set.column_names().empty()) {
        col_names.push_back(col_name);
      }

      if (row[col] == nullptr && !(field->flags & NOT_NULL_FLAG)) {
        record.add_values(kMetadataSourceNull);
      } else {
        record.add_values(absl::StrCat(row[col]));
      }
    }
    *record_set.mutable_records()->Add() = record;

    if (record_set.column_names().empty()) {
      *record_set.mutable_column_names() = {col_names.begin(), col_names.end()};
    }
  }

  if (record_set_out != nullptr) {
    *record_set_out = std::move(record_set);
  }
  return Status::OK();
}

std::string MySqlMetadataSource::EscapeString(absl::string_view value) const {
  CHECK(db_ != nullptr);
  // in the worst case, each character needs to be escaped by backslash, and the
  // string is appended an additional terminating null character.
  char* buffer = new char[value.length() * 2 + 1];
  CHECK(mysql_real_escape_string(db_, buffer, value.data(), value.length()) !=
        -1UL)
      << "NO_BACKSLASH_ESCAPES SQL mode should not be enabled.";
  std::string result(buffer);
  delete[] buffer;
  return result;
}

}  // namespace ml_metadata
<commit_msg>Internal change.<commit_after>/* Copyright 2019 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "ml_metadata/metadata_store/mysql_metadata_source.h"

#include <string>
#include <utility>

#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "ml_metadata/metadata_store/constants.h"
#include "ml_metadata/metadata_store/types.h"
#include "ml_metadata/proto/metadata_source.pb.h"
#include "mysql.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"

namespace ml_metadata {

namespace {

namespace errors = tensorflow::errors;

using ::tensorflow::Status;

constexpr char kBeginTransaction[] = "START TRANSACTION";
constexpr char kCommitTransaction[] = "COMMIT";
constexpr char kRollbackTransaction[] = "ROLLBACK";

// A class that invokes mysql_thread_init() when constructed, and
// mysql_thread_end() when destructed.  It can be used as a
// thread_local to ensure that this happens exactly once per thread
// (see ThreadInitAccess).
class ThreadInitializer {
 public:
  ThreadInitializer() : initialized_(!mysql_thread_init()) {
    // Note: mysql_thread_init returns 0 on success
  }

  // Disallow copy-and-assign.
  ThreadInitializer(const ThreadInitializer&) = delete;
  ThreadInitializer& operator=(const ThreadInitializer&) = delete;

  ~ThreadInitializer() {
    if (initialized_) mysql_thread_end();
  }

  bool initialized() const { return initialized_; }

 private:
  // Indicates whether the call to mysql_thread_init was successfull.
  const bool initialized_;
};

// Initializes a ThreadInitializer for the current thread (if not already done).
// Returns an error if the MYSQL thread initialization fails.
//
// Note: All top-level methods that are invoked by the MetadataSource base class
// need to invoke ThreadInitAccess.
Status ThreadInitAccess() {
  // Cause mysql_thread_init() to be invoked exactly once per thread
  // the first time ThreadInitAccess() is called, and
  // mysql_thread_end() exactly once when the thread dies.
  thread_local ThreadInitializer initializer;
  if (!initializer.initialized()) {
    return errors::Internal("mysql thread initialization not done");
  }
  return Status::OK();
}

// Checks if config is valid.
Status CheckConfig(const MySQLDatabaseConfig& config) {
  std::vector<std::string> config_errors;
  if (config.host().empty() == config.socket().empty()) {
    config_errors.push_back("exactly one of host or socket must be specified");
  }
  if (config.database().empty()) {
    config_errors.push_back("database must not be empty");
  }

  if (!config_errors.empty()) {
    return errors::InvalidArgument(absl::StrJoin(config_errors, ";"));
  }
  return Status::OK();
}

}  // namespace

MySqlMetadataSource::MySqlMetadataSource(const MySQLDatabaseConfig& config)
    : MetadataSource(), config_(config) {
  TF_CHECK_OK(CheckConfig(config));
}

MySqlMetadataSource::~MySqlMetadataSource() { TF_CHECK_OK(CloseImpl()); }

Status MySqlMetadataSource::ConnectImpl() {
  // Initialize the MYSQL object.
  db_ = mysql_init(nullptr);
  if (!db_) {
    LOG(ERROR) << "MySQL error: " << mysql_errno(db_) << ": "
               << mysql_error(db_);
    return errors::Internal("mysql_init failed: errno: ",
                            mysql_errno(db_), ", error: ", mysql_error(db_));
  }

  // Explicitly setup the thread-local initializer.
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ThreadInitAccess(),
                                  "MySql thread init failed at ConnectImpl");

  // Set connection options
  if (config_.has_ssl_options()) {
    const MySQLDatabaseConfig::SSLOptions& ssl = config_.ssl_options();
    // The method set mysql_options, and always return 0. The connection options
    // are used in the `mysql_real_connect`.
    mysql_ssl_set(db_, ssl.key().empty() ? nullptr : ssl.key().c_str(),
                  ssl.cert().empty() ? nullptr : ssl.cert().c_str(),
                  ssl.ca().empty() ? nullptr : ssl.ca().c_str(),
                  ssl.capath().empty() ? nullptr : ssl.capath().c_str(),
                  ssl.cipher().empty() ? nullptr : ssl.cipher().c_str());
    my_bool verify_server_cert = ssl.verify_server_cert() ? 1 : 0;
    mysql_options(db_, MYSQL_OPT_SSL_VERIFY_SERVER_CERT, &verify_server_cert);
  }

  // Connect to the MYSQL server.
  db_ = mysql_real_connect(
          db_, config_.host().empty() ? nullptr : config_.host().c_str(),
          config_.user().empty() ? nullptr : config_.user().c_str(),
          config_.password().empty() ? nullptr : config_.password().c_str(),
          /*db=*/nullptr, config_.port(),
          config_.socket().empty() ? nullptr : config_.socket().c_str(),
          /*clientflag=*/0UL);

  if (!db_) {
    return errors::Internal("mysql_real_connect failed: errno: ",
                            mysql_errno(db_), ", error: ", mysql_error(db_));
  }

  // Return an error if the default storage engine doesn't support transactions.
  TF_RETURN_WITH_CONTEXT_IF_ERROR(
      CheckTransactionSupport(),
      "checking transaction support of default storage engine");

  // Create the database if not already present and switch to it.
  const std::string create_database_cmd =
      absl::StrCat("CREATE DATABASE IF NOT EXISTS ", config_.database());
  TF_RETURN_WITH_CONTEXT_IF_ERROR(RunQuery(create_database_cmd),
                                  "Creating database ", config_.database(),
                                  " in ConnectImpl");
  const std::string use_database_cmd = absl::StrCat("USE ", config_.database());
  TF_RETURN_WITH_CONTEXT_IF_ERROR(RunQuery(use_database_cmd),
                                  "Changing to database ", config_.database(),
                                  " in ConnectImpl");

  return Status::OK();
}

Status MySqlMetadataSource::CloseImpl() {
  if (db_ != nullptr) {
    TF_RETURN_IF_ERROR(ThreadInitAccess());
    DiscardResultSet();
    mysql_close(db_);
    db_ = nullptr;
  }
  return Status::OK();
}

Status MySqlMetadataSource::ExecuteQueryImpl(const std::string& query,
                                             RecordSet* results) {
  TF_RETURN_WITH_CONTEXT_IF_ERROR(
      ThreadInitAccess(), "MySql thread init failed at ExecuteQueryImpl");

  // Run the query.
  Status status = RunQuery(query);

  // Return on failure.
  TF_RETURN_IF_ERROR(status);

  // If query is successfull, convert the results.
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ConvertMySqlRowSetToRecordSet(results),
                                  "ConvertMySqlRowSetToRecordSet for query ",
                                  query);
  return Status::OK();
}

Status MySqlMetadataSource::CommitImpl() {
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ThreadInitAccess(),
                                  "MySql thread init failed at CommitImpl");
  return RunQuery(kCommitTransaction);
}

Status MySqlMetadataSource::RollbackImpl() {
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ThreadInitAccess(),
                                  "MySql thread init failed at RollbackImpl");
  return RunQuery(kRollbackTransaction);
}

Status MySqlMetadataSource::BeginImpl() {
  TF_RETURN_WITH_CONTEXT_IF_ERROR(ThreadInitAccess(),
                                  "MySql thread init failed at BeginImpl");
  return RunQuery(kBeginTransaction);
}

Status MySqlMetadataSource::CheckTransactionSupport() {
  constexpr char kCheckTransactionSupport[] =
      "SELECT ENGINE, TRANSACTIONS FROM INFORMATION_SCHEMA.ENGINES WHERE "
      "ENGINE=(SELECT @@default_storage_engine)";

  TF_RETURN_IF_ERROR(RunQuery(kCheckTransactionSupport));

  RecordSet record_set;
  TF_RETURN_IF_ERROR(ConvertMySqlRowSetToRecordSet(&record_set));
  if (record_set.records_size() != 1 ||
      record_set.records(0).values_size() != 2) {
    return errors::Internal(
        "Expected query ", kCheckTransactionSupport,
        " to generate exactly single row with 2 columns, but got ",
        record_set.DebugString());
  }
  const RecordSet::Record& record = record_set.records(0);
  if (record.values(1) != "YES") {
    return errors::Internal(
        "no transaction support for default_storage_engine ", record.values(0));
  }

  return Status::OK();
}

Status MySqlMetadataSource::RunQuery(const std::string& query) {
  DiscardResultSet();

  int query_status = mysql_query(db_, query.c_str());
  if (query_status) {
    int64 error_number = mysql_errno(db_);
    // 2006: sever closes the connection due to inactive client;
    // client reports server has gone away, we reconnect the server for the
    // client if the query is begin transaction.
    if (error_number == 2006 && query == kBeginTransaction) {
      TF_RETURN_IF_ERROR(CloseImpl());
      TF_RETURN_IF_ERROR(ConnectImpl());
      return RunQuery(query);
    }
    // 1213: inno db aborts deadlock when running concurrent transactions.
    // returns Aborted for client side to retry.
    if (error_number == 1213 || error_number == 1205) {
      return errors::Aborted("mysql_query aborted: errno: ", error_number,
                             ", error: ", mysql_error(db_));
    }
    return errors::Internal("mysql_query failed: errno: ", error_number,
                            ", error: ", mysql_error(db_));
  }

  result_set_ = mysql_store_result(db_);
  if (!result_set_ && mysql_field_count(db_) != 0) {
    return errors::Internal("mysql_query ", query,
                            " returned an unexpected NULL result_set: Errno: ",
                            mysql_errno(db_), ", Error: ", mysql_error(db_));
  }

  return Status::OK();
}

void MySqlMetadataSource::DiscardResultSet() {
  if (result_set_ != nullptr) {
    // Fetch any leftover rows (MySQL requires this).
    // Note: Since we always store the query result locally in RunQuery(),
    // we might not need to explicitly fetch the row here. But fetching rows
    // here won't hurt and will be a bit more robust in case we decide to lazily
    // fetch the rows from the server.
    while (mysql_fetch_row(result_set_)) {
    }
    mysql_free_result(result_set_);
    result_set_ = nullptr;
  }
}

Status MySqlMetadataSource::ConvertMySqlRowSetToRecordSet(
    RecordSet* record_set_out) {
  RecordSet record_set;

  if (result_set_ == nullptr) {
    return Status::OK();
  }

  MYSQL_ROW row;
  while ((row = mysql_fetch_row(result_set_)) != nullptr) {
    RecordSet::Record record;
    std::vector<std::string> col_names;

    uint32 num_cols = mysql_num_fields(result_set_);
    for (uint32 col = 0; col < num_cols; ++col) {
      MYSQL_FIELD* field = mysql_fetch_field_direct(result_set_, col);
      if (field == nullptr) {
        return errors::Internal(
            "Error in retrieving column description for index ", col);
      }
      const std::string col_name(field->org_name);
      if (record_set.column_names().empty()) {
        col_names.push_back(col_name);
      }

      if (row[col] == nullptr && !(field->flags & NOT_NULL_FLAG)) {
        record.add_values(kMetadataSourceNull);
      } else {
        record.add_values(absl::StrCat(row[col]));
      }
    }
    *record_set.mutable_records()->Add() = record;

    if (record_set.column_names().empty()) {
      *record_set.mutable_column_names() = {col_names.begin(), col_names.end()};
    }
  }

  if (record_set_out != nullptr) {
    *record_set_out = std::move(record_set);
  }
  return Status::OK();
}

std::string MySqlMetadataSource::EscapeString(absl::string_view value) const {
  CHECK(db_ != nullptr);
  // in the worst case, each character needs to be escaped by backslash, and the
  // string is appended an additional terminating null character.
  char* buffer = new char[value.length() * 2 + 1];
  CHECK(mysql_real_escape_string(db_, buffer, value.data(), value.length()) !=
        -1UL)
      << "NO_BACKSLASH_ESCAPES SQL mode should not be enabled.";
  std::string result(buffer);
  delete[] buffer;
  return result;
}

}  // namespace ml_metadata
<|endoftext|>


<commit_before>#include "block.hpp"
#include <algorithm>
#include <turbo/algorithm/recovery.hpp>
#include <turbo/algorithm/recovery.hxx>
#include <turbo/memory/alignment.hpp>
#include <turbo/container/mpmc_ring_queue.hxx>
#include <turbo/toolset/extension.hpp>

namespace turbo {
namespace memory {

out_of_memory_error::out_of_memory_error(const std::string& what)
    :
	runtime_error(what)
{ }

out_of_memory_error::out_of_memory_error(const char* what)
    :
	runtime_error(what)
{ }

invalid_size_error::invalid_size_error(const std::string& what)
    :
	invalid_argument(what)
{ }

invalid_size_error::invalid_size_error(const char* what)
    :
	invalid_argument(what)
{ }

invalid_alignment_error::invalid_alignment_error(const std::string& what)
    :
	invalid_argument(what)
{ }

invalid_alignment_error::invalid_alignment_error(const char* what)
    :
	invalid_argument(what)
{ }

invalid_pointer_error::invalid_pointer_error(const std::string& what)
    :
	invalid_argument(what)
{ }

invalid_pointer_error::invalid_pointer_error(const char* what)
    :
	invalid_argument(what)
{ }

block::block(std::size_t value_size, capacity_type capacity)
    :
	block(value_size, capacity, alignof(void*))
{ }

block::block(std::size_t value_size, capacity_type capacity, std::size_t alignment)
    :
	value_size_(value_size),
	capacity_(capacity),
	usable_size_(capacity == 0U ? 0U : value_size_ * (capacity_ + 1)), // need extra in case of bad alignment
	storage_(capacity == 0U ? nullptr : new std::uint8_t[usable_size_]),
	base_(capacity == 0U ? nullptr : &(storage_[0])),
	free_list_(capacity)
{
    if (TURBO_UNLIKELY(value_size_ == 0))
    {
	throw invalid_size_error("value size cannot be 0");
    }
    if (capacity_ == 0U)
    {
	return;
    }
    if (TURBO_UNLIKELY(!storage_))
    {
	throw out_of_memory_error("insufficient space in heap");
    }
    std::fill_n(base_, usable_size_, 0U);
    void* tmp = base_;
    if (TURBO_UNLIKELY(!turbo::memory::align(alignment, value_size_, tmp, usable_size_)))
    {
	throw invalid_alignment_error("alignment exceeds requested total size");
    }
    base_ = static_cast<std::uint8_t*>(tmp);
    capacity_ = usable_size_ / value_size;
    if (TURBO_UNLIKELY(capacity_ < capacity))
    {
	throw out_of_memory_error("available space in heap is insufficient for required alignment");
    }
    else
    {
	capacity_ = capacity;
	namespace tar = turbo::algorithm::recovery;
	for (capacity_type index = 0; index < capacity_; ++index)
	{
	    tar::retry_with_random_backoff([&] () -> tar::try_state
	    {
		if (free_list_.try_enqueue_copy(index) == free_list_type::producer::result::success)
		{
		    return tar::try_state::done;
		}
		else
		{
		    return tar::try_state::retry;
		}
	    });
	}
    }
}

void* block::allocate()
{
    namespace tar = turbo::algorithm::recovery;
    capacity_type reservation = 0U;
    void* result = nullptr;
    if (is_empty())
    {
	return nullptr;
    }
    tar::retry_with_random_backoff([&] () -> tar::try_state
    {
	switch (free_list_.try_dequeue_copy(reservation))
	{
	    case free_list_type::consumer::result::queue_empty:
	    {
		// no free blocks available
		result = nullptr;
		return tar::try_state::done;
	    }
	    case free_list_type::consumer::result::success:
	    {
		result = &(base_[reservation * value_size_]);
		return tar::try_state::done;
	    }
	    default:
	    {
		return tar::try_state::retry;
	    }
	}
    });
    return result;
}

void block::free(void* pointer)
{
    namespace tar = turbo::algorithm::recovery;
    if (is_empty() || pointer == nullptr)
    {
	return;
    }
    std::size_t diff = static_cast<std::uint8_t*>(pointer) - base_;
    if (diff % value_size_ != 0)
    {
	throw invalid_pointer_error("address points to the middle of a value");
    }
    std::size_t offset = diff / value_size_;
    if (offset < (usable_size_ / value_size_))
    {
	tar::retry_with_random_backoff([&] () -> tar::try_state
	{
	    switch (free_list_.try_enqueue_copy(offset))
	    {
		case free_list_type::producer::result::queue_full:
		{
		    // log a warning?
		    return tar::try_state::done;
		}
		case free_list_type::producer::result::success:
		{
		    return tar::try_state::done;
		}
		default:
		{
		    return tar::try_state::retry;
		}
	    }
	});
    }
    else
    {
	throw invalid_pointer_error("given address does not come from this block");
    }
}

} // namespace memory
} // namespace turbo
<commit_msg>Teere's no point in using retry_with_random_backoff inside the block constructor since other threads can't access it yet; it also greatly improves the constructor's performance<commit_after>#include "block.hpp"
#include <algorithm>
#include <turbo/algorithm/recovery.hpp>
#include <turbo/algorithm/recovery.hxx>
#include <turbo/memory/alignment.hpp>
#include <turbo/container/mpmc_ring_queue.hxx>
#include <turbo/toolset/extension.hpp>

namespace turbo {
namespace memory {

out_of_memory_error::out_of_memory_error(const std::string& what)
    :
	runtime_error(what)
{ }

out_of_memory_error::out_of_memory_error(const char* what)
    :
	runtime_error(what)
{ }

invalid_size_error::invalid_size_error(const std::string& what)
    :
	invalid_argument(what)
{ }

invalid_size_error::invalid_size_error(const char* what)
    :
	invalid_argument(what)
{ }

invalid_alignment_error::invalid_alignment_error(const std::string& what)
    :
	invalid_argument(what)
{ }

invalid_alignment_error::invalid_alignment_error(const char* what)
    :
	invalid_argument(what)
{ }

invalid_pointer_error::invalid_pointer_error(const std::string& what)
    :
	invalid_argument(what)
{ }

invalid_pointer_error::invalid_pointer_error(const char* what)
    :
	invalid_argument(what)
{ }

block::block(std::size_t value_size, capacity_type capacity)
    :
	block(value_size, capacity, alignof(void*))
{ }

block::block(std::size_t value_size, capacity_type capacity, std::size_t alignment)
    :
	value_size_(value_size),
	capacity_(capacity),
	usable_size_(capacity == 0U ? 0U : value_size_ * (capacity_ + 1)), // need extra in case of bad alignment
	storage_(capacity == 0U ? nullptr : new std::uint8_t[usable_size_]),
	base_(capacity == 0U ? nullptr : &(storage_[0])),
	free_list_(capacity)
{
    if (TURBO_UNLIKELY(value_size_ == 0))
    {
	throw invalid_size_error("value size cannot be 0");
    }
    if (capacity_ == 0U)
    {
	return;
    }
    if (TURBO_UNLIKELY(!storage_))
    {
	throw out_of_memory_error("insufficient space in heap");
    }
    std::fill_n(base_, usable_size_, 0U);
    void* tmp = base_;
    if (TURBO_UNLIKELY(!turbo::memory::align(alignment, value_size_, tmp, usable_size_)))
    {
	throw invalid_alignment_error("alignment exceeds requested total size");
    }
    base_ = static_cast<std::uint8_t*>(tmp);
    capacity_ = usable_size_ / value_size;
    if (TURBO_UNLIKELY(capacity_ < capacity))
    {
	throw out_of_memory_error("available space in heap is insufficient for required alignment");
    }
    else
    {
	capacity_ = capacity;
	for (capacity_type index = 0; index < capacity_; ++index)
	{
	    free_list_.try_enqueue_copy(index);
	}
    }
}

void* block::allocate()
{
    namespace tar = turbo::algorithm::recovery;
    capacity_type reservation = 0U;
    void* result = nullptr;
    if (is_empty())
    {
	return nullptr;
    }
    tar::retry_with_random_backoff([&] () -> tar::try_state
    {
	switch (free_list_.try_dequeue_copy(reservation))
	{
	    case free_list_type::consumer::result::queue_empty:
	    {
		// no free blocks available
		result = nullptr;
		return tar::try_state::done;
	    }
	    case free_list_type::consumer::result::success:
	    {
		result = &(base_[reservation * value_size_]);
		return tar::try_state::done;
	    }
	    default:
	    {
		return tar::try_state::retry;
	    }
	}
    });
    return result;
}

void block::free(void* pointer)
{
    namespace tar = turbo::algorithm::recovery;
    if (is_empty() || pointer == nullptr)
    {
	return;
    }
    std::size_t diff = static_cast<std::uint8_t*>(pointer) - base_;
    if (diff % value_size_ != 0)
    {
	throw invalid_pointer_error("address points to the middle of a value");
    }
    std::size_t offset = diff / value_size_;
    if (offset < (usable_size_ / value_size_))
    {
	tar::retry_with_random_backoff([&] () -> tar::try_state
	{
	    switch (free_list_.try_enqueue_copy(offset))
	    {
		case free_list_type::producer::result::queue_full:
		{
		    // log a warning?
		    return tar::try_state::done;
		}
		case free_list_type::producer::result::success:
		{
		    return tar::try_state::done;
		}
		default:
		{
		    return tar::try_state::retry;
		}
	    }
	});
    }
    else
    {
	throw invalid_pointer_error("given address does not come from this block");
    }
}

} // namespace memory
} // namespace turbo
<|endoftext|>


<commit_before>/* Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "ml_metadata/tools/mlmd_bench/fill_types_workload.h"

#include <random>

#include "absl/memory/memory.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/substitute.h"
#include "ml_metadata/metadata_store/metadata_store.h"
#include "ml_metadata/metadata_store/types.h"
#include "ml_metadata/proto/metadata_store.pb.h"
#include "ml_metadata/proto/metadata_store_service.pb.h"
#include "ml_metadata/tools/mlmd_bench/proto/mlmd_bench.pb.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"

namespace ml_metadata {
namespace {

// A template function where the Type can be ArtifactType / ExecutionType /
// ContextType. It takes a `type_name` to generate a type and generates
// `number_properties` properties. It also increase the stored bytes for the
// `type` in `curr_bytes`.
template <typename Type>
void GenerateType(const std::string& type_name, const int64 num_properties,
                  Type& type, int64& curr_bytes) {
  // The random type name will be a random number.
  type.set_name(type_name);
  // The curr_bytes records the total transferred bytes for executing each work
  // item.
  curr_bytes += type.name().size();
  for (int64 i = 0; i < num_properties; i++) {
    (*type.mutable_properties())[absl::StrCat("p-", i)] = STRING;
    curr_bytes += absl::StrCat("p-", i).size();
  }
}

// Gets the number of current types(num_curr_type) and total
// types(num_total_type) for later insert or update. Also updates the
// get_response for later update. Returns detailed error if query executions
// failed.
tensorflow::Status GetNumberOfTypes(const FillTypesConfig& fill_types_config,
                                    MetadataStore*& store, int64& num_curr_type,
                                    int64& num_total_type,
                                    GetTypeResponseType& get_response) {
  int64 num_artifact_type = 0, num_execution_type = 0, num_context_type = 0;
  switch (fill_types_config.specification()) {
    case FillTypesConfig::ARTIFACT_TYPE: {
      get_response.emplace<GetArtifactTypesResponse>();
      TF_RETURN_IF_ERROR(store->GetArtifactTypes(
          /*request=*/{}, &absl::get<GetArtifactTypesResponse>(get_response)));
      num_artifact_type = absl::get<GetArtifactTypesResponse>(get_response)
                              .artifact_types_size();
      num_curr_type = num_artifact_type;
      break;
    }
    case FillTypesConfig::EXECUTION_TYPE: {
      get_response.emplace<GetExecutionTypesResponse>();
      TF_RETURN_IF_ERROR(store->GetExecutionTypes(
          /*request=*/{}, &absl::get<GetExecutionTypesResponse>(get_response)));
      num_execution_type = absl::get<GetExecutionTypesResponse>(get_response)
                               .execution_types_size();
      num_curr_type = num_execution_type;
      break;
    }
    case FillTypesConfig::CONTEXT_TYPE: {
      get_response.emplace<GetContextTypesResponse>();
      TF_RETURN_IF_ERROR(store->GetContextTypes(
          /*request=*/{}, &absl::get<GetContextTypesResponse>(get_response)));
      num_context_type =
          absl::get<GetContextTypesResponse>(get_response).context_types_size();
      num_curr_type = num_context_type;
      break;
    }
    default:
      LOG(FATAL) << "Wrong specification for FillTypes!";
  }
  num_total_type = num_artifact_type + num_execution_type + num_context_type;
  return tensorflow::Status::OK();
}

// Inserts new types into the db if the current types inside db is not enough
// for update. Returns detailed error if query executions failed.
tensorflow::Status MakeUpTypesForUpdate(
    const FillTypesConfig& fill_types_config, MetadataStore*& store,
    int64 num_type_to_make_up) {
  FillTypesConfig make_up_config = fill_types_config;
  make_up_config.set_update(false);
  std::unique_ptr<FillTypes> make_up_fill_types;
  make_up_fill_types = absl::make_unique<FillTypes>(
      FillTypes(make_up_config, num_type_to_make_up));
  TF_RETURN_IF_ERROR(make_up_fill_types->SetUp(store));
  for (int64 i = 0; i < num_type_to_make_up; ++i) {
    OpStats op_stats;
    TF_RETURN_IF_ERROR(make_up_fill_types->RunOp(i, store, op_stats));
  }
  return tensorflow::Status::OK();
}

// A template function where the Type can be ArtifactType / ExecutionType /
// ContextType.
// Takes an existed type and generates a new type for later update accordingly.
// The updated type will have some new fields added and the number of new added
// fields will be generated w.r.t. the uniform distribution.
template <typename Type>
void UpdateType(const int64 num_properties, const Type& existed_type,
                Type& updated_type, int64& curr_bytes) {
  // Except the new added fields, update_type will the same as existed_type.
  updated_type = existed_type;
  curr_bytes += existed_type.name().size();
  for (auto& pair : existed_type.properties()) {
    // pair.first is the property of existed_type.
    curr_bytes += pair.first.size();
  }
  for (int64 i = 0; i < num_properties; i++) {
    (*updated_type.mutable_properties())[absl::StrCat("add_p-", i)] = STRING;
    curr_bytes += absl::StrCat("add_p-", i).size();
  }
}

}  // namespace

FillTypes::FillTypes(const FillTypesConfig& fill_types_config,
                     int64 num_operations)
    : fill_types_config_(fill_types_config), num_operations_(num_operations) {
  switch (fill_types_config_.specification()) {
    case FillTypesConfig::ARTIFACT_TYPE: {
      name_ = "fill_artifact_type";
      break;
    }
    case FillTypesConfig::EXECUTION_TYPE: {
      name_ = "fill_execution_type";
      break;
    }
    case FillTypesConfig::CONTEXT_TYPE: {
      name_ = "fill_context_type";
      break;
    }
    default:
      LOG(FATAL) << "Wrong specification for FillTypes!";
  }
}

tensorflow::Status FillTypes::SetUpImpl(MetadataStore* store) {
  LOG(INFO) << "Setting up ...";

  int64 curr_bytes = 0;
  // Uniform distribution that describes the number of properties for each
  // generated types.
  UniformDistribution num_properties = fill_types_config_.num_properties();
  int64 min = num_properties.minimum();
  int64 max = num_properties.maximum();
  std::uniform_int_distribution<int64> uniform_dist{min, max};
  // The seed for the random generator is the time when the FillTypes is
  // created.
  std::minstd_rand0 gen(absl::ToUnixMillis(absl::Now()));

  // Gets the number of current types(num_curr_type) and total
  // types(num_total_type) for later insert or update.
  int64 num_curr_type = 0, num_total_type = 0;
  GetTypeResponseType get_response;
  TF_RETURN_IF_ERROR(GetNumberOfTypes(fill_types_config_, store, num_curr_type,
                                      num_total_type, get_response));

  // If the number of current types is less than the update number of
  // operations, calls MakeUpTypesForUpdate() for inserting new types into the
  // db for later update.
  if (fill_types_config_.update() && num_curr_type < num_operations_) {
    int64 num_type_to_make_up = num_operations_ - num_curr_type;
    TF_RETURN_IF_ERROR(
        MakeUpTypesForUpdate(fill_types_config_, store, num_type_to_make_up));
    // Updates the get_response to contain the up-to-date types inside db for
    // later update.
    TF_RETURN_IF_ERROR(GetNumberOfTypes(fill_types_config_, store,
                                        num_curr_type, num_total_type,
                                        get_response));
  }

  for (int64 i = num_total_type; i < num_total_type + num_operations_; i++) {
    curr_bytes = 0;
    int64 update_type_index = i - num_total_type;
    FillTypeWorkItemType put_request;
    const std::string type_name = absl::StrCat("type_", i);
    const int64 num_properties = uniform_dist(gen);
    switch (fill_types_config_.specification()) {
      case FillTypesConfig::ARTIFACT_TYPE: {
        put_request.emplace<PutArtifactTypeRequest>();
        if (fill_types_config_.update()) {
          // For update purpose, the can_add_fields field should be set to true.
          absl::get<PutArtifactTypeRequest>(put_request)
              .set_can_add_fields(true);
          UpdateType<ArtifactType>(
              num_properties,
              absl::get<GetArtifactTypesResponse>(get_response)
                  .artifact_types()[update_type_index],
              *absl::get<PutArtifactTypeRequest>(put_request)
                   .mutable_artifact_type(),
              curr_bytes);
        } else {
          GenerateType<ArtifactType>(
              type_name, num_properties,
              *absl::get<PutArtifactTypeRequest>(put_request)
                   .mutable_artifact_type(),
              curr_bytes);
        }
        break;
      }
      case FillTypesConfig::EXECUTION_TYPE: {
        put_request.emplace<PutExecutionTypeRequest>();
        if (fill_types_config_.update()) {
          // For update purpose, the can_add_fields field should be set to true.
          absl::get<PutExecutionTypeRequest>(put_request)
              .set_can_add_fields(true);
          UpdateType<ExecutionType>(
              num_properties,
              absl::get<GetExecutionTypesResponse>(get_response)
                  .execution_types()[update_type_index],
              *absl::get<PutExecutionTypeRequest>(put_request)
                   .mutable_execution_type(),
              curr_bytes);
        } else {
          GenerateType<ExecutionType>(
              type_name, num_properties,
              *absl::get<PutExecutionTypeRequest>(put_request)
                   .mutable_execution_type(),
              curr_bytes);
        }
        break;
      }
      case FillTypesConfig::CONTEXT_TYPE: {
        put_request.emplace<PutContextTypeRequest>();
        if (fill_types_config_.update()) {
          // For update purpose, the can_add_fields field should be set to true.
          absl::get<PutContextTypeRequest>(put_request)
              .set_can_add_fields(true);
          UpdateType<ContextType>(
              num_properties,
              absl::get<GetContextTypesResponse>(get_response)
                  .context_types()[update_type_index],
              *absl::get<PutContextTypeRequest>(put_request)
                   .mutable_context_type(),
              curr_bytes);
        } else {
          GenerateType<ContextType>(
              type_name, num_properties,
              *absl::get<PutContextTypeRequest>(put_request)
                   .mutable_context_type(),
              curr_bytes);
        }
        break;
      }
      default:
        return tensorflow::errors::InvalidArgument("Wrong specification!");
    }
    // Updates work_items_.
    work_items_.emplace_back(put_request, curr_bytes);
  }
  return tensorflow::Status::OK();
}

// Executions of work items.
tensorflow::Status FillTypes::RunOpImpl(const int64 work_items_index,
                                        MetadataStore* store) {
  const int64 i = work_items_index;
  switch (fill_types_config_.specification()) {
    case FillTypesConfig::ARTIFACT_TYPE: {
      PutArtifactTypeRequest put_request =
          absl::get<PutArtifactTypeRequest>(work_items_[i].first);
      PutArtifactTypeResponse put_response;
      return store->PutArtifactType(put_request, &put_response);
    }
    case FillTypesConfig::EXECUTION_TYPE: {
      PutExecutionTypeRequest put_request =
          absl::get<PutExecutionTypeRequest>(work_items_[i].first);
      PutExecutionTypeResponse put_response;
      return store->PutExecutionType(put_request, &put_response);
    }
    case FillTypesConfig::CONTEXT_TYPE: {
      PutContextTypeRequest put_request =
          absl::get<PutContextTypeRequest>(work_items_[i].first);
      PutContextTypeResponse put_response;
      return store->PutContextType(put_request, &put_response);
    }
    default:
      return tensorflow::errors::InvalidArgument("Wrong specification!");
  }
  return tensorflow::errors::InvalidArgument(
      "Cannot execute the query due to wrong specification!");
}

tensorflow::Status FillTypes::TearDownImpl() {
  work_items_.clear();
  return tensorflow::Status::OK();
}

std::string FillTypes::GetName() { return name_; }

}  // namespace ml_metadata
<commit_msg>Fixs GetNumberOfTypes() for update mode of FillTypes.<commit_after>/* Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "ml_metadata/tools/mlmd_bench/fill_types_workload.h"

#include <random>

#include "absl/memory/memory.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/substitute.h"
#include "ml_metadata/metadata_store/metadata_store.h"
#include "ml_metadata/metadata_store/types.h"
#include "ml_metadata/proto/metadata_store.pb.h"
#include "ml_metadata/proto/metadata_store_service.pb.h"
#include "ml_metadata/tools/mlmd_bench/proto/mlmd_bench.pb.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"

namespace ml_metadata {
namespace {

// A template function where the Type can be ArtifactType / ExecutionType /
// ContextType. It takes a `type_name` to generate a type and generates
// `number_properties` properties. It also increase the stored bytes for the
// `type` in `curr_bytes`.
template <typename Type>
void GenerateType(const std::string& type_name, const int64 num_properties,
                  Type& type, int64& curr_bytes) {
  // The random type name will be a random number.
  type.set_name(type_name);
  // The curr_bytes records the total transferred bytes for executing each work
  // item.
  curr_bytes += type.name().size();
  for (int64 i = 0; i < num_properties; i++) {
    (*type.mutable_properties())[absl::StrCat("p-", i)] = STRING;
    curr_bytes += absl::StrCat("p-", i).size();
  }
}

// Gets the number of current types(num_curr_type) and total
// types(num_total_type) for later insert or update. Also updates the
// get_response for later update. Returns detailed error if query executions
// failed.
tensorflow::Status GetNumberOfTypes(const FillTypesConfig& fill_types_config,
                                    MetadataStore* store, int64& num_curr_type,
                                    int64& num_total_type,
                                    GetTypeResponseType& get_response) {
  GetArtifactTypesResponse get_artifact_type_response;
  TF_RETURN_IF_ERROR(store->GetArtifactTypes(
      /*request=*/{}, &get_artifact_type_response));
  int64 num_artifact_type = get_artifact_type_response.artifact_types_size();

  GetExecutionTypesResponse get_execution_type_response;
  TF_RETURN_IF_ERROR(store->GetExecutionTypes(
      /*request=*/{}, &get_execution_type_response));
  int64 num_execution_type = get_execution_type_response.execution_types_size();

  GetContextTypesResponse get_context_type_response;
  TF_RETURN_IF_ERROR(store->GetContextTypes(
      /*request=*/{}, &get_context_type_response));
  int64 num_context_type = get_context_type_response.context_types_size();

  switch (fill_types_config.specification()) {
    case FillTypesConfig::ARTIFACT_TYPE: {
      num_curr_type = num_artifact_type;
      get_response.emplace<GetArtifactTypesResponse>(
          get_artifact_type_response);
      break;
    }
    case FillTypesConfig::EXECUTION_TYPE: {
      num_curr_type = num_execution_type;
      get_response.emplace<GetExecutionTypesResponse>(
          get_execution_type_response);
      break;
    }
    case FillTypesConfig::CONTEXT_TYPE: {
      num_curr_type = num_context_type;
      get_response.emplace<GetContextTypesResponse>(get_context_type_response);
      break;
    }
    default:
      LOG(FATAL) << "Wrong specification for FillTypes!";
  }

  num_total_type = num_artifact_type + num_execution_type + num_context_type;
  return tensorflow::Status::OK();
}

// Inserts new types into the db if the current types inside db is not enough
// for update. Returns detailed error if query executions failed.
tensorflow::Status MakeUpTypesForUpdate(
    const FillTypesConfig& fill_types_config, MetadataStore* store,
    int64 num_type_to_make_up) {
  FillTypesConfig make_up_config = fill_types_config;
  make_up_config.set_update(false);
  std::unique_ptr<FillTypes> make_up_fill_types;
  make_up_fill_types = absl::make_unique<FillTypes>(
      FillTypes(make_up_config, num_type_to_make_up));
  TF_RETURN_IF_ERROR(make_up_fill_types->SetUp(store));
  for (int64 i = 0; i < num_type_to_make_up; ++i) {
    OpStats op_stats;
    TF_RETURN_IF_ERROR(make_up_fill_types->RunOp(i, store, op_stats));
  }
  return tensorflow::Status::OK();
}

// A template function where the Type can be ArtifactType / ExecutionType /
// ContextType.
// Takes an existed type and generates a new type for later update accordingly.
// The updated type will have some new fields added and the number of new added
// fields will be generated w.r.t. the uniform distribution.
template <typename Type>
void UpdateType(const int64 num_properties, const Type& existed_type,
                Type& updated_type, int64& curr_bytes) {
  // Except the new added fields, update_type will the same as existed_type.
  updated_type = existed_type;
  curr_bytes += existed_type.name().size();
  for (auto& pair : existed_type.properties()) {
    // pair.first is the property of existed_type.
    curr_bytes += pair.first.size();
  }
  for (int64 i = 0; i < num_properties; i++) {
    (*updated_type.mutable_properties())[absl::StrCat("add_p-", i)] = STRING;
    curr_bytes += absl::StrCat("add_p-", i).size();
  }
}

}  // namespace

FillTypes::FillTypes(const FillTypesConfig& fill_types_config,
                     int64 num_operations)
    : fill_types_config_(fill_types_config), num_operations_(num_operations) {
  switch (fill_types_config_.specification()) {
    case FillTypesConfig::ARTIFACT_TYPE: {
      name_ = "fill_artifact_type";
      break;
    }
    case FillTypesConfig::EXECUTION_TYPE: {
      name_ = "fill_execution_type";
      break;
    }
    case FillTypesConfig::CONTEXT_TYPE: {
      name_ = "fill_context_type";
      break;
    }
    default:
      LOG(FATAL) << "Wrong specification for FillTypes!";
  }
  if (fill_types_config_.update()) {
    name_ += "(update)";
  }
}

tensorflow::Status FillTypes::SetUpImpl(MetadataStore* store) {
  LOG(INFO) << "Setting up ...";

  int64 curr_bytes = 0;
  // Uniform distribution that describes the number of properties for each
  // generated types.
  UniformDistribution num_properties = fill_types_config_.num_properties();
  int64 min = num_properties.minimum();
  int64 max = num_properties.maximum();
  std::uniform_int_distribution<int64> uniform_dist{min, max};
  // The seed for the random generator is the time when the FillTypes is
  // created.
  std::minstd_rand0 gen(absl::ToUnixMillis(absl::Now()));

  // Gets the number of current types(num_curr_type) and total
  // types(num_total_type) for later insert or update.
  int64 num_curr_type = 0, num_total_type = 0;
  GetTypeResponseType get_response;
  TF_RETURN_IF_ERROR(GetNumberOfTypes(fill_types_config_, store, num_curr_type,
                                      num_total_type, get_response));

  // If the number of current types is less than the update number of
  // operations, calls MakeUpTypesForUpdate() for inserting new types into the
  // db for later update.
  if (fill_types_config_.update() && num_curr_type < num_operations_) {
    int64 num_type_to_make_up = num_operations_ - num_curr_type;
    TF_RETURN_IF_ERROR(
        MakeUpTypesForUpdate(fill_types_config_, store, num_type_to_make_up));
    // Updates the get_response to contain the up-to-date types inside db for
    // later update.
    TF_RETURN_IF_ERROR(GetNumberOfTypes(fill_types_config_, store,
                                        num_curr_type, num_total_type,
                                        get_response));
  }

  for (int64 i = num_total_type; i < num_total_type + num_operations_; i++) {
    curr_bytes = 0;
    int64 update_type_index = i - num_total_type;
    FillTypeWorkItemType put_request;
    const std::string type_name = absl::StrCat("type_", i);
    const int64 num_properties = uniform_dist(gen);
    switch (fill_types_config_.specification()) {
      case FillTypesConfig::ARTIFACT_TYPE: {
        put_request.emplace<PutArtifactTypeRequest>();
        if (fill_types_config_.update()) {
          // For update purpose, the can_add_fields field should be set to true.
          absl::get<PutArtifactTypeRequest>(put_request)
              .set_can_add_fields(true);
          UpdateType<ArtifactType>(
              num_properties,
              absl::get<GetArtifactTypesResponse>(get_response)
                  .artifact_types()[update_type_index],
              *absl::get<PutArtifactTypeRequest>(put_request)
                   .mutable_artifact_type(),
              curr_bytes);
        } else {
          GenerateType<ArtifactType>(
              type_name, num_properties,
              *absl::get<PutArtifactTypeRequest>(put_request)
                   .mutable_artifact_type(),
              curr_bytes);
        }
        break;
      }
      case FillTypesConfig::EXECUTION_TYPE: {
        put_request.emplace<PutExecutionTypeRequest>();
        if (fill_types_config_.update()) {
          // For update purpose, the can_add_fields field should be set to true.
          absl::get<PutExecutionTypeRequest>(put_request)
              .set_can_add_fields(true);
          UpdateType<ExecutionType>(
              num_properties,
              absl::get<GetExecutionTypesResponse>(get_response)
                  .execution_types()[update_type_index],
              *absl::get<PutExecutionTypeRequest>(put_request)
                   .mutable_execution_type(),
              curr_bytes);
        } else {
          GenerateType<ExecutionType>(
              type_name, num_properties,
              *absl::get<PutExecutionTypeRequest>(put_request)
                   .mutable_execution_type(),
              curr_bytes);
        }
        break;
      }
      case FillTypesConfig::CONTEXT_TYPE: {
        put_request.emplace<PutContextTypeRequest>();
        if (fill_types_config_.update()) {
          // For update purpose, the can_add_fields field should be set to true.
          absl::get<PutContextTypeRequest>(put_request)
              .set_can_add_fields(true);
          UpdateType<ContextType>(
              num_properties,
              absl::get<GetContextTypesResponse>(get_response)
                  .context_types()[update_type_index],
              *absl::get<PutContextTypeRequest>(put_request)
                   .mutable_context_type(),
              curr_bytes);
        } else {
          GenerateType<ContextType>(
              type_name, num_properties,
              *absl::get<PutContextTypeRequest>(put_request)
                   .mutable_context_type(),
              curr_bytes);
        }
        break;
      }
      default:
        return tensorflow::errors::InvalidArgument("Wrong specification!");
    }
    // Updates work_items_.
    work_items_.emplace_back(put_request, curr_bytes);
  }
  return tensorflow::Status::OK();
}

// Executions of work items.
tensorflow::Status FillTypes::RunOpImpl(const int64 work_items_index,
                                        MetadataStore* store) {
  const int64 i = work_items_index;
  switch (fill_types_config_.specification()) {
    case FillTypesConfig::ARTIFACT_TYPE: {
      PutArtifactTypeRequest put_request =
          absl::get<PutArtifactTypeRequest>(work_items_[i].first);
      PutArtifactTypeResponse put_response;
      return store->PutArtifactType(put_request, &put_response);
    }
    case FillTypesConfig::EXECUTION_TYPE: {
      PutExecutionTypeRequest put_request =
          absl::get<PutExecutionTypeRequest>(work_items_[i].first);
      PutExecutionTypeResponse put_response;
      return store->PutExecutionType(put_request, &put_response);
    }
    case FillTypesConfig::CONTEXT_TYPE: {
      PutContextTypeRequest put_request =
          absl::get<PutContextTypeRequest>(work_items_[i].first);
      PutContextTypeResponse put_response;
      return store->PutContextType(put_request, &put_response);
    }
    default:
      return tensorflow::errors::InvalidArgument("Wrong specification!");
  }
  return tensorflow::errors::InvalidArgument(
      "Cannot execute the query due to wrong specification!");
}

tensorflow::Status FillTypes::TearDownImpl() {
  work_items_.clear();
  return tensorflow::Status::OK();
}

std::string FillTypes::GetName() { return name_; }

}  // namespace ml_metadata
<|endoftext|>


<commit_before>#include "Heart2.h"

Heart2::Heart2(Display &display) : BitmapAnimation(display, heart) {

}

#define purple 0x640000

std::vector<std::vector<uint32_t>> Heart2::heart = {
    {
        0, purple, purple, 0, 0 , purple, purple, 0,
        purple, purple, purple, purple, purple, purple, purple, purple,
        purple, purple, purple, purple, purple, purple, purple, purple,
        purple, purple, purple, purple, purple, purple, purple, purple,
        purple, purple, purple, purple, purple, purple, purple, purple,
        0, purple, purple, purple, purple, purple, purple, 0,
        0, 0, purple, purple, purple, purple, 0, 0,
        0, 0, 0, purple, purple, 0, 0, 0,
    },
    {
        0, 0, 0, 0, 0 , 0, 0, 0,
        0, 0, purple, 0, 0, purple, 0, 0,
        0, purple, purple, purple, purple, purple, purple, 0,
        0, purple, purple, purple, purple, purple, purple, 0,
        0, purple, purple, purple, purple, purple, purple, 0,
        0, 0, purple, purple, purple, purple, 0, 0,
        0, 0, 0, purple, purple, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0,
    },
};
<commit_msg>Match up heart to original<commit_after>#include "Heart2.h"

Heart2::Heart2(Display &display) : BitmapAnimation(display, heart) {

}

#define red 0x640000
#define purple 0x640064

std::vector<std::vector<uint32_t>> Heart2::heart = {
    {
        0, red, red, 0, 0 , red, red, 0,
        red, red, purple, red, red, purple, red, red,
        red, purple, purple, purple, purple, purple, purple, red,
        red, purple, purple, purple, purple, purple, purple, red,
        red, purple, purple, purple, purple, purple, purple, red,
        0, red, purple, purple, purple, purple, red, 0,
        0, 0, red, purple, purple, red, 0, 0,
        0, 0, 0, red, red, 0, 0, 0,
    },
    {
        0, 0, 0, 0, 0 , 0, 0, 0,
        0, 0, red, 0, 0, red, 0, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, 0, red, purple, purple, red, 0, 0,
        0, 0, 0, red, red, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0,
    },
    {
        0, red, red, 0, 0 , red, red, 0,
        red, red, red, red, red, red, red, red,
        red, red, red, red, red, red, red, red,
        red, red, red, red, red, red, red, red,
        red, red, red, red, red, red, red, red,
        0, red, red, red, red, red, red, 0,
        0, 0, red, red, red, red, 0, 0,
        0, 0, 0, red, red, 0, 0, 0,
    },
    {
        0, 0, 0, 0, 0 , 0, 0, 0,
        0, 0, red, 0, 0, red, 0, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, 0, red, purple, purple, red, 0, 0,
        0, 0, 0, red, red, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0,
    },
    {
        0, red, red, 0, 0 , red, red, 0,
        red, red, purple, red, red, purple, red, red,
        red, purple, purple, purple, purple, purple, purple, red,
        red, purple, purple, purple, purple, purple, purple, red,
        red, purple, purple, purple, purple, purple, purple, red,
        0, red, purple, purple, purple, purple, red, 0,
        0, 0, red, purple, purple, red, 0, 0,
        0, 0, 0, red, red, 0, 0, 0,
    },
    {
        0, 0, 0, 0, 0 , 0, 0, 0,
        0, 0, red, 0, 0, red, 0, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, 0, red, purple, purple, red, 0, 0,
        0, 0, 0, red, red, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0,
    },
    {
        0, red, red, 0, 0 , red, red, 0,
        red, red, purple, red, red, purple, red, red,
        red, purple, purple, purple, purple, purple, purple, red,
        red, purple, purple, purple, purple, purple, purple, red,
        red, purple, purple, purple, purple, purple, purple, red,
        0, red, purple, purple, purple, purple, red, 0,
        0, 0, red, purple, purple, red, 0, 0,
        0, 0, 0, red, red, 0, 0, 0,
    },
    {
        0, 0, 0, 0, 0 , 0, 0, 0,
        0, 0, red, 0, 0, red, 0, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, red, purple, purple, purple, purple, red, 0,
        0, 0, red, purple, purple, red, 0, 0,
        0, 0, 0, red, red, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0,
    },
};
<|endoftext|>