<commit_before>/*
 * Copyright 2015 Milian Wolff <mail@milianw.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#include "parser.h"

#include <ThreadWeaver/ThreadWeaver>
#include <KFormat>
#include <KLocalizedString>

#include <QTextStream>
#include <QDebug>

#include "../accumulatedtracedata.h"
#include "flamegraph.h"

#include <vector>

using namespace std;

namespace {

// TODO: use QString directly
struct StringCache
{
    StringCache()
    {
        m_ipAddresses.reserve(16384);
    }

    QString func(const InstructionPointer& ip) const
    {
        if (ip.functionIndex) {
            // TODO: support removal of template arguments
            return stringify(ip.functionIndex);
        } else {
            auto& ipAddr = m_ipAddresses[ip.instructionPointer];
            if (ipAddr.isEmpty()) {
                ipAddr = QLatin1String("0x") + QString::number(ip.instructionPointer, 16);
            }
            return ipAddr;
        }
    }

    QString file(const InstructionPointer& ip) const
    {
        if (ip.fileIndex) {
            return stringify(ip.fileIndex);
        } else {
            return {};
        }
    }

    QString module(const InstructionPointer& ip) const
    {
        return stringify(ip.moduleIndex);
    }

    QString stringify(const StringIndex index) const
    {
        if (!index || index.index > m_strings.size()) {
            return {};
        } else {
            return m_strings.at(index.index - 1);
        }
    }

    LocationData location(const InstructionPointer& ip) const
    {
        return {func(ip), file(ip), module(ip), ip.line};
    }

    void update(const vector<string>& strings)
    {
        transform(strings.begin() + m_strings.size(), strings.end(),
                  back_inserter(m_strings), [] (const string& str) { return QString::fromStdString(str); });
    }

    vector<QString> m_strings;
    mutable QHash<uint64_t, QString> m_ipAddresses;
};

struct ChartMergeData
{
    QString function;
    quint64 consumed;
    quint64 allocations;
    quint64 allocated;
    bool operator<(const QString& rhs) const
    {
        return function < rhs;
    }
};
static_assert(std::is_nothrow_move_assignable<ChartMergeData>::value, "ChartMergeData must be nothrow move assignable for performance reasons");

struct ParserData final : public AccumulatedTraceData
{
    ParserData()
    {
        // start off with null data at the origin
        consumedChartData.data.rows.push_back({});
        allocatedChartData.data.rows.push_back({});
        allocationsChartData.data.rows.push_back({});
        // index 0 indicates the total row
        consumedChartData.labelIds.insert(i18n("total"), 0);
        allocatedChartData.labelIds.insert(i18n("total"), 0);
        allocationsChartData.labelIds.insert(i18n("total"), 0);
    }

    void handleTimeStamp(uint64_t /*oldStamp*/, uint64_t newStamp)
    {
        stringCache.update(strings);
        maxConsumedSinceLastTimeStamp = max(maxConsumedSinceLastTimeStamp, leaked);

        // merge data for top 10 functions in this timestamp
        vector<ChartMergeData> mergedData;
        for (const auto& allocation : allocations) {
            const auto function = stringCache.func(findIp(findTrace(allocation.traceIndex).ipIndex));
            auto it = lower_bound(mergedData.begin(), mergedData.end(), function);
            if (it != mergedData.end() && it->function == function) {
                it->allocated += allocation.allocated;
                it->allocations += allocation.allocations;
                it->consumed += allocation.leaked;
            } else {
                it = mergedData.insert(it, {function, allocation.leaked, allocation.allocations, allocation.allocated});
            }
        }

        auto addChartData = [&] (quint64 ChartMergeData::* member, ChartDataWithLabels* data, quint64 totalCost) {
            ChartRows row;
            row.timeStamp = newStamp;
            row.cost.insert(0, totalCost);
            sort(mergedData.begin(), mergedData.end(), [=] (const ChartMergeData& left, const ChartMergeData& right) {
                return left.*member > right.*member;
            });
            for (size_t i = 0; i < min(size_t(10), mergedData.size()); ++i) {
                const auto& alloc = mergedData[i];
                if (!(alloc.*member)) {
                    break;
                }
                auto& id = data->labelIds[alloc.function];
                if (!id) {
                    id = data->labelIds.size() - 1;
                }
                row.cost.insert(id, alloc.*member);
            }
            data->data.rows.append(row);

            if (newStamp == totalTime) {
                // finalize and convert labels
                data->data.labels.reserve(data->labelIds.size());
                for (auto it = data->labelIds.constBegin(); it != data->labelIds.constEnd(); ++it) {
                    data->data.labels.insert(it.value(), it.key());
                }
            }
        };
        addChartData(&ChartMergeData::consumed, &consumedChartData, maxConsumedSinceLastTimeStamp);
        addChartData(&ChartMergeData::allocated, &allocatedChartData, totalAllocated);
        addChartData(&ChartMergeData::allocations, &allocationsChartData, totalAllocations);
        maxConsumedSinceLastTimeStamp = 0;
    }

    void handleAllocation()
    {
        maxConsumedSinceLastTimeStamp = max(maxConsumedSinceLastTimeStamp, leaked);
    }

    void handleDebuggee(const char* command)
    {
        debuggee = command;
    }

    string debuggee;

    struct ChartDataWithLabels
    {
        ChartData data;
        QHash<QString, int> labelIds;
    };
    ChartDataWithLabels consumedChartData;
    ChartDataWithLabels allocationsChartData;
    ChartDataWithLabels allocatedChartData;
    uint64_t maxConsumedSinceLastTimeStamp = 0;

    StringCache stringCache;
};

QString generateSummary(const ParserData& data)
{
    QString ret;
    KFormat format;
    QTextStream stream(&ret);
    const double totalTimeS = 0.001 * data.totalTime;
    stream << "<qt>"
           << i18n("debuggee: %1", QString::fromStdString(data.debuggee)) << "
"
           // xgettext:no-c-format
           << i18n("total runtime: %1s", totalTimeS) << "
"
           << i18n("bytes allocated in total (ignoring deallocations): %1 (%2/s)",
                   format.formatByteSize(data.totalAllocated, 2), format.formatByteSize(data.totalAllocated / totalTimeS)) << "
"
           << i18n("calls to allocation functions: %1 (%2/s)",
                   data.totalAllocations, quint64(data.totalAllocations / totalTimeS)) << "
"
           << i18n("peak heap memory consumption: %1", format.formatByteSize(data.peak)) << "
"
           << i18n("total memory leaked: %1", format.formatByteSize(data.leaked)) << "
";
    stream << "</qt>";
    return ret;
}

void setParents(QVector<RowData>& children, const RowData* parent)
{
    for (auto& row: children) {
        row.parent = parent;
        setParents(row.children, &row);
    }
}

QVector<RowData> mergeAllocations(const ParserData& data)
{
    QVector<RowData> topRows;
    // merge allocations, leave parent pointers invalid (their location may change)
    for (const auto& allocation : data.allocations) {
        auto traceIndex = allocation.traceIndex;
        auto rows = &topRows;
        while (traceIndex) {
            const auto& trace = data.findTrace(traceIndex);
            const auto& ip = data.findIp(trace.ipIndex);
            // TODO: only store the IpIndex and use that
            auto location = data.stringCache.location(ip);
            auto it = lower_bound(rows->begin(), rows->end(), location);
            if (it != rows->end() && it->location == location) {
                it->allocated += allocation.allocated;
                it->allocations += allocation.allocations;
                it->leaked += allocation.leaked;
                it->peak += allocation.peak;
            } else {
                it = rows->insert(it, {allocation.allocations, allocation.peak, allocation.leaked, allocation.allocated,
                                        location, nullptr, {}});
            }
            if (data.isStopIndex(ip.functionIndex)) {
                break;
            }
            traceIndex = trace.parentIndex;
            rows = &it->children;
        }
    }
    // now set the parents, the data is constant from here on
    setParents(topRows, nullptr);
    return topRows;
}

RowData* findByLocation(const RowData& row, QVector<RowData>* data)
{
    for (int i = 0; i < data->size(); ++i) {
        if (data->at(i).location == row.location) {
            return data->data() + i;
        }
    }
    return nullptr;
}

void buildTopDown(const QVector<RowData>& bottomUpData, QVector<RowData>* topDownData)
{
    foreach (const auto& row, bottomUpData) {
        if (row.children.isEmpty()) {
            // leaf node found, bubble up the parent chain to build a top-down tree
            auto node = &row;
            auto stack = topDownData;
            while (node) {
                auto data = findByLocation(*node, stack);
                if (!data) {
                    // create an empty top-down item for this bottom-up node
                    *stack << RowData{0, 0, 0, 0, node->location, nullptr, {}};
                    data = &stack->back();
                }
                // always use the leaf node's cost and propagate that one up the chain
                // otherwise we'd count the cost of some nodes multiple times
                data->allocations += row.allocations;
                data->peak += row.peak;
                data->leaked += row.leaked;
                data->allocated += row.allocated;
                stack = &data->children;
                node = node->parent;
            }
        } else {
            // recurse to find a leaf
            buildTopDown(row.children, topDownData);
        }
    }
}

QVector<RowData> toTopDownData(const QVector<RowData>& bottomUpData)
{
    QVector<RowData> topRows;
    buildTopDown(bottomUpData, &topRows);
    // now set the parents, the data is constant from here on
    setParents(topRows, nullptr);
    return topRows;
}

}

Parser::Parser(QObject* parent)
    : QObject(parent)
{}

Parser::~Parser() = default;

void Parser::parse(const QString& path)
{
    using namespace ThreadWeaver;
    stream() << make_job([=]() {
        ParserData data;
        data.read(path.toStdString());
        emit summaryAvailable(generateSummary(data));
        const auto mergedAllocations = mergeAllocations(data);
        emit bottomUpDataAvailable(mergedAllocations);
        emit consumedChartDataAvailable(data.consumedChartData.data);
        emit allocationsChartDataAvailable(data.allocationsChartData.data);
        emit allocatedChartDataAvailable(data.allocatedChartData.data);
        const auto topDownData = toTopDownData(mergedAllocations);
        emit topDownDataAvailable(topDownData);
        emit flameGraphDataAvailable(FlameGraph::parseData(topDownData));
        emit finished();
    });
}
<commit_msg>Only create one chart data per second.<commit_after>/*
 * Copyright 2015 Milian Wolff <mail@milianw.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#include "parser.h"

#include <ThreadWeaver/ThreadWeaver>
#include <KFormat>
#include <KLocalizedString>

#include <QTextStream>
#include <QDebug>

#include "../accumulatedtracedata.h"
#include "flamegraph.h"

#include <vector>

using namespace std;

namespace {

// TODO: use QString directly
struct StringCache
{
    StringCache()
    {
        m_ipAddresses.reserve(16384);
    }

    QString func(const InstructionPointer& ip) const
    {
        if (ip.functionIndex) {
            // TODO: support removal of template arguments
            return stringify(ip.functionIndex);
        } else {
            auto& ipAddr = m_ipAddresses[ip.instructionPointer];
            if (ipAddr.isEmpty()) {
                ipAddr = QLatin1String("0x") + QString::number(ip.instructionPointer, 16);
            }
            return ipAddr;
        }
    }

    QString file(const InstructionPointer& ip) const
    {
        if (ip.fileIndex) {
            return stringify(ip.fileIndex);
        } else {
            return {};
        }
    }

    QString module(const InstructionPointer& ip) const
    {
        return stringify(ip.moduleIndex);
    }

    QString stringify(const StringIndex index) const
    {
        if (!index || index.index > m_strings.size()) {
            return {};
        } else {
            return m_strings.at(index.index - 1);
        }
    }

    LocationData location(const InstructionPointer& ip) const
    {
        return {func(ip), file(ip), module(ip), ip.line};
    }

    void update(const vector<string>& strings)
    {
        transform(strings.begin() + m_strings.size(), strings.end(),
                  back_inserter(m_strings), [] (const string& str) { return QString::fromStdString(str); });
    }

    vector<QString> m_strings;
    mutable QHash<uint64_t, QString> m_ipAddresses;
};

struct ChartMergeData
{
    QString function;
    quint64 consumed;
    quint64 allocations;
    quint64 allocated;
    bool operator<(const QString& rhs) const
    {
        return function < rhs;
    }
};
static_assert(std::is_nothrow_move_assignable<ChartMergeData>::value, "ChartMergeData must be nothrow move assignable for performance reasons");

struct ParserData final : public AccumulatedTraceData
{
    ParserData()
    {
        // start off with null data at the origin
        consumedChartData.data.rows.push_back({});
        allocatedChartData.data.rows.push_back({});
        allocationsChartData.data.rows.push_back({});
        // index 0 indicates the total row
        consumedChartData.labelIds.insert(i18n("total"), 0);
        allocatedChartData.labelIds.insert(i18n("total"), 0);
        allocationsChartData.labelIds.insert(i18n("total"), 0);
    }

    void handleTimeStamp(uint64_t /*oldStamp*/, uint64_t newStamp)
    {
        maxConsumedSinceLastTimeStamp = max(maxConsumedSinceLastTimeStamp, leaked);
        // TODO: make this configurable via the GUI
        const uint64_t diffBetweenTimeStamps = 1000; //  1000ms = 1s
        if (newStamp != totalTime && newStamp - lastTimeStamp < diffBetweenTimeStamps) {
            return;
        }
        lastTimeStamp = newStamp;
        stringCache.update(strings);

        // merge data for top 10 functions in this timestamp
        vector<ChartMergeData> mergedData;
        for (const auto& allocation : allocations) {
            const auto function = stringCache.func(findIp(findTrace(allocation.traceIndex).ipIndex));
            auto it = lower_bound(mergedData.begin(), mergedData.end(), function);
            if (it != mergedData.end() && it->function == function) {
                it->allocated += allocation.allocated;
                it->allocations += allocation.allocations;
                it->consumed += allocation.leaked;
            } else {
                it = mergedData.insert(it, {function, allocation.leaked, allocation.allocations, allocation.allocated});
            }
        }

        auto addChartData = [&] (quint64 ChartMergeData::* member, ChartDataWithLabels* data, quint64 totalCost) {
            ChartRows row;
            row.timeStamp = newStamp;
            row.cost.insert(0, totalCost);
            sort(mergedData.begin(), mergedData.end(), [=] (const ChartMergeData& left, const ChartMergeData& right) {
                return left.*member > right.*member;
            });
            for (size_t i = 0; i < min(size_t(10), mergedData.size()); ++i) {
                const auto& alloc = mergedData[i];
                if (!(alloc.*member)) {
                    break;
                }
                auto& id = data->labelIds[alloc.function];
                if (!id) {
                    id = data->labelIds.size() - 1;
                }
                row.cost.insert(id, alloc.*member);
            }
            data->data.rows.append(row);

            if (newStamp == totalTime) {
                // finalize and convert labels
                data->data.labels.reserve(data->labelIds.size());
                for (auto it = data->labelIds.constBegin(); it != data->labelIds.constEnd(); ++it) {
                    data->data.labels.insert(it.value(), it.key());
                }
            }
        };
        addChartData(&ChartMergeData::consumed, &consumedChartData, maxConsumedSinceLastTimeStamp);
        addChartData(&ChartMergeData::allocated, &allocatedChartData, totalAllocated);
        addChartData(&ChartMergeData::allocations, &allocationsChartData, totalAllocations);
        maxConsumedSinceLastTimeStamp = 0;
    }

    void handleAllocation()
    {
        maxConsumedSinceLastTimeStamp = max(maxConsumedSinceLastTimeStamp, leaked);
    }

    void handleDebuggee(const char* command)
    {
        debuggee = command;
    }

    string debuggee;

    struct ChartDataWithLabels
    {
        ChartData data;
        QHash<QString, int> labelIds;
    };
    ChartDataWithLabels consumedChartData;
    ChartDataWithLabels allocationsChartData;
    ChartDataWithLabels allocatedChartData;
    uint64_t maxConsumedSinceLastTimeStamp = 0;
    uint64_t lastTimeStamp = 0;

    StringCache stringCache;
};

QString generateSummary(const ParserData& data)
{
    QString ret;
    KFormat format;
    QTextStream stream(&ret);
    const double totalTimeS = 0.001 * data.totalTime;
    stream << "<qt>"
           << i18n("debuggee: %1", QString::fromStdString(data.debuggee)) << "
"
           // xgettext:no-c-format
           << i18n("total runtime: %1s", totalTimeS) << "
"
           << i18n("bytes allocated in total (ignoring deallocations): %1 (%2/s)",
                   format.formatByteSize(data.totalAllocated, 2), format.formatByteSize(data.totalAllocated / totalTimeS)) << "
"
           << i18n("calls to allocation functions: %1 (%2/s)",
                   data.totalAllocations, quint64(data.totalAllocations / totalTimeS)) << "
"
           << i18n("peak heap memory consumption: %1", format.formatByteSize(data.peak)) << "
"
           << i18n("total memory leaked: %1", format.formatByteSize(data.leaked)) << "
";
    stream << "</qt>";
    return ret;
}

void setParents(QVector<RowData>& children, const RowData* parent)
{
    for (auto& row: children) {
        row.parent = parent;
        setParents(row.children, &row);
    }
}

QVector<RowData> mergeAllocations(const ParserData& data)
{
    QVector<RowData> topRows;
    // merge allocations, leave parent pointers invalid (their location may change)
    for (const auto& allocation : data.allocations) {
        auto traceIndex = allocation.traceIndex;
        auto rows = &topRows;
        while (traceIndex) {
            const auto& trace = data.findTrace(traceIndex);
            const auto& ip = data.findIp(trace.ipIndex);
            // TODO: only store the IpIndex and use that
            auto location = data.stringCache.location(ip);
            auto it = lower_bound(rows->begin(), rows->end(), location);
            if (it != rows->end() && it->location == location) {
                it->allocated += allocation.allocated;
                it->allocations += allocation.allocations;
                it->leaked += allocation.leaked;
                it->peak += allocation.peak;
            } else {
                it = rows->insert(it, {allocation.allocations, allocation.peak, allocation.leaked, allocation.allocated,
                                        location, nullptr, {}});
            }
            if (data.isStopIndex(ip.functionIndex)) {
                break;
            }
            traceIndex = trace.parentIndex;
            rows = &it->children;
        }
    }
    // now set the parents, the data is constant from here on
    setParents(topRows, nullptr);
    return topRows;
}

RowData* findByLocation(const RowData& row, QVector<RowData>* data)
{
    for (int i = 0; i < data->size(); ++i) {
        if (data->at(i).location == row.location) {
            return data->data() + i;
        }
    }
    return nullptr;
}

void buildTopDown(const QVector<RowData>& bottomUpData, QVector<RowData>* topDownData)
{
    foreach (const auto& row, bottomUpData) {
        if (row.children.isEmpty()) {
            // leaf node found, bubble up the parent chain to build a top-down tree
            auto node = &row;
            auto stack = topDownData;
            while (node) {
                auto data = findByLocation(*node, stack);
                if (!data) {
                    // create an empty top-down item for this bottom-up node
                    *stack << RowData{0, 0, 0, 0, node->location, nullptr, {}};
                    data = &stack->back();
                }
                // always use the leaf node's cost and propagate that one up the chain
                // otherwise we'd count the cost of some nodes multiple times
                data->allocations += row.allocations;
                data->peak += row.peak;
                data->leaked += row.leaked;
                data->allocated += row.allocated;
                stack = &data->children;
                node = node->parent;
            }
        } else {
            // recurse to find a leaf
            buildTopDown(row.children, topDownData);
        }
    }
}

QVector<RowData> toTopDownData(const QVector<RowData>& bottomUpData)
{
    QVector<RowData> topRows;
    buildTopDown(bottomUpData, &topRows);
    // now set the parents, the data is constant from here on
    setParents(topRows, nullptr);
    return topRows;
}

}

Parser::Parser(QObject* parent)
    : QObject(parent)
{}

Parser::~Parser() = default;

void Parser::parse(const QString& path)
{
    using namespace ThreadWeaver;
    stream() << make_job([=]() {
        ParserData data;
        data.read(path.toStdString());
        emit summaryAvailable(generateSummary(data));
        const auto mergedAllocations = mergeAllocations(data);
        emit bottomUpDataAvailable(mergedAllocations);
        emit consumedChartDataAvailable(data.consumedChartData.data);
        emit allocationsChartDataAvailable(data.allocationsChartData.data);
        emit allocatedChartDataAvailable(data.allocatedChartData.data);
        const auto topDownData = toTopDownData(mergedAllocations);
        emit topDownDataAvailable(topDownData);
        emit flameGraphDataAvailable(FlameGraph::parseData(topDownData));
        emit finished();
    });
}
<|endoftext|>


<commit_before>#include "pch.h"
#include "axl_io_File.h"
#include "axl_rtl_String.h"

namespace axl {
namespace io {

//.............................................................................

#if (_AXL_ENV == AXL_ENV_WIN)

bool
CFile::Open (
	const char* pFileName,
	uint_t Flags
	)
{
	uint_t AccessMode = (Flags & EFileFlag_ReadOnly) ?
		GENERIC_READ :
		GENERIC_READ | GENERIC_WRITE;

	uint_t ShareMode = (Flags & EFileFlag_Exclusive) ?
		0 :
		(Flags & EFileFlag_ShareWrite) ?
			FILE_SHARE_READ | FILE_SHARE_WRITE :
			FILE_SHARE_READ;

	uint_t CreationDisposition = (Flags & (EFileFlag_ReadOnly | EFileFlag_OpenExisting)) ?
		OPEN_EXISTING :
		OPEN_ALWAYS;

	uint_t FlagsAttributes = (Flags & EFileFlag_DeleteOnClose) ?
		FILE_FLAG_DELETE_ON_CLOSE :
		0;

	if (Flags & EFileFlag_Asynchronous)
		FlagsAttributes |= FILE_FLAG_OVERLAPPED;

	char Buffer [256];
	rtl::CString_w FileName (ref::EBuf_Stack, Buffer, sizeof (Buffer));
	FileName = pFileName;

	return m_File.Create (
		FileName,
		AccessMode,
		ShareMode,
		NULL,
		CreationDisposition,
		FlagsAttributes
		);
}

#elif (_AXL_ENV == AXL_ENV_POSIX)

uint_t
GetPosixOpenFlags (uint_t FileFlags)
{
	uint_t PosixFlags = (FileFlags & EFileFlag_ReadOnly) ? O_RDONLY : O_RDWR;

	if (!(FileFlags & EFileFlag_OpenExisting))
		PosixFlags |= O_CREAT;

	if (FileFlags & EFileFlag_Asynchronous)
		PosixFlags |= O_NONBLOCK;

	return PosixFlags;
}

bool
CFile::Open (
	const char* pFileName,
	uint_t Flags
	)
{
	uint_t PosixFlags = GetPosixOpenFlags (Flags);

	// TODO: handle exclusive and share write flags with fcntl locks

	bool Result = m_File.Open (pFileName, PosixFlags);
	if (!Result)
		return false;

	if (Flags & EFileFlag_DeleteOnClose)
		unlink (pFileName);

	return true;
}

#endif

size_t
CFile::WriteFormat_va (
	const char* pFormat,
	axl_va_list va
	)
{
	char Buffer [256];
	rtl::CString String (ref::EBuf_Stack, Buffer, sizeof (Buffer));
	String.Format_va (pFormat, va);

	return Write (String, String.GetLength ());
}

//.............................................................................

} // namespace io
} // namespace axl
<commit_msg>[axl.io] add FILE_SHARE_DELETE flag to file open flags -- important when using io::EFileFlag_DeleteOnClose<commit_after>#include "pch.h"
#include "axl_io_File.h"
#include "axl_rtl_String.h"

namespace axl {
namespace io {

//.............................................................................

#if (_AXL_ENV == AXL_ENV_WIN)

bool
CFile::Open (
	const char* pFileName,
	uint_t Flags
	)
{
	uint_t AccessMode = (Flags & EFileFlag_ReadOnly) ?
		GENERIC_READ :
		GENERIC_READ | GENERIC_WRITE;

	uint_t ShareMode = (Flags & EFileFlag_Exclusive) ?
		0 :
		(Flags & EFileFlag_ShareWrite) ?
			FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE :
			FILE_SHARE_READ | FILE_SHARE_DELETE;

	uint_t CreationDisposition = (Flags & (EFileFlag_ReadOnly | EFileFlag_OpenExisting)) ?
		OPEN_EXISTING :
		OPEN_ALWAYS;

	uint_t FlagsAttributes = (Flags & EFileFlag_DeleteOnClose) ?
		FILE_FLAG_DELETE_ON_CLOSE :
		0;

	if (Flags & EFileFlag_Asynchronous)
		FlagsAttributes |= FILE_FLAG_OVERLAPPED;

	char Buffer [256];
	rtl::CString_w FileName (ref::EBuf_Stack, Buffer, sizeof (Buffer));
	FileName = pFileName;

	return m_File.Create (
		FileName,
		AccessMode,
		ShareMode,
		NULL,
		CreationDisposition,
		FlagsAttributes
		);
}

#elif (_AXL_ENV == AXL_ENV_POSIX)

uint_t
GetPosixOpenFlags (uint_t FileFlags)
{
	uint_t PosixFlags = (FileFlags & EFileFlag_ReadOnly) ? O_RDONLY : O_RDWR;

	if (!(FileFlags & EFileFlag_OpenExisting))
		PosixFlags |= O_CREAT;

	if (FileFlags & EFileFlag_Asynchronous)
		PosixFlags |= O_NONBLOCK;

	return PosixFlags;
}

bool
CFile::Open (
	const char* pFileName,
	uint_t Flags
	)
{
	uint_t PosixFlags = GetPosixOpenFlags (Flags);

	// TODO: handle exclusive and share write flags with fcntl locks

	bool Result = m_File.Open (pFileName, PosixFlags);
	if (!Result)
		return false;

	if (Flags & EFileFlag_DeleteOnClose)
		unlink (pFileName);

	return true;
}

#endif

size_t
CFile::WriteFormat_va (
	const char* pFormat,
	axl_va_list va
	)
{
	char Buffer [256];
	rtl::CString String (ref::EBuf_Stack, Buffer, sizeof (Buffer));
	String.Format_va (pFormat, va);

	return Write (String, String.GetLength ());
}

//.............................................................................

} // namespace io
} // namespace axl
<|endoftext|>


<commit_before>// ModelRegistry.hxx -- interface to the OSG model registry
//
// Copyright (C) 2005-2007 Mathias Froehlich
// Copyright (C) 2007  Tim Moore <timoore@redhat.com>
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License as
// published by the Free Software Foundation; either version 2 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#ifndef _SG_MODELREGISTRY_HXX
#define _SG_MODELREGISTRY_HXX 1

#include <osg/ref_ptr>
#include <osg/Node>
#include <osgDB/FileUtils>
#include <osgDB/FileNameUtils>
#include <osgDB/ReaderWriter>
#include <osgDB/Registry>

#include <simgear/compiler.h>
#include <simgear/scene/util/OsgSingleton.hxx>

#include <string>
#include <map>

// Class to register per file extension read callbacks with the OSG
// registry, mostly to control caching and post load optimization /
// copying that happens above the level of the ReaderWriter.
namespace simgear
{

// Different caching and optimization strategies are needed for
// different file types. Most loaded files should be optimized and the
// optimized version should be cached. When an .osg file is
// substituted for another, it is assumed to be optimized already but
// it should be cached too (under the name of the original?). .stg
// files should not be cached (that's the pager's job) but the files
// it causes to be loaded should be. .btg files are already optimized
// and shouldn't be cached.
//
// Complicating this is the effect that removing CACHE_NODES has from
// the ReaderWriter options: it switches the object cache with an
// empty one, so that's not an option for the files that could be
// loaded from a .stg file. So, we'll let
// Registry::readNodeImplementation cache a loaded file and then add
// the optimized version to the cache ourselves, replacing the
// original subgraph.
//
// To support all these options with a minimum of duplication, the
// readNode function is specified as a template with a bunch of
// pluggable (and predefined) policies.
template <typename ProcessPolicy, typename CachePolicy, typename OptimizePolicy,
          typename SubstitutePolicy, typename BVHPolicy>
class ModelRegistryCallback : public osgDB::Registry::ReadFileCallback {
public:
    ModelRegistryCallback(const std::string& extension) :
        _processPolicy(extension), _cachePolicy(extension),
        _optimizePolicy(extension),
        _substitutePolicy(extension), _bvhPolicy(extension)
    {
    }
    virtual osgDB::ReaderWriter::ReadResult
    readNode(const std::string& fileName,
             const osgDB::Options* opt)
    {
        using namespace osg;
        using namespace osgDB;
        using osgDB::ReaderWriter;
//        Registry* registry = Registry::instance();
        ref_ptr<osg::Node> optimizedNode = _cachePolicy.find(fileName, opt);
        if (!optimizedNode.valid()) {
            std::string otherFileName = _substitutePolicy.substitute(fileName,
                                                                     opt);
            ReaderWriter::ReadResult res;
            if (!otherFileName.empty()) {
                res = loadUsingReaderWriter(otherFileName, opt);
                if (res.validNode())
                    optimizedNode = res.getNode();
            }
            if (!optimizedNode.valid()) {
                res = loadUsingReaderWriter(fileName, opt);
                if (!res.validNode())
                    return res;
                ref_ptr<osg::Node> processedNode
                    = _processPolicy.process(res.getNode(), fileName, opt);
                optimizedNode = _optimizePolicy.optimize(processedNode.get(),
                                                         fileName, opt);
            }
            _bvhPolicy.buildBVH(fileName, optimizedNode.get());
            _cachePolicy.addToCache(fileName, optimizedNode.get());
        }
        return ReaderWriter::ReadResult(optimizedNode.get());
    }
protected:
    static osgDB::ReaderWriter::ReadResult
    loadUsingReaderWriter(const std::string& fileName,
                          const osgDB::Options* opt)
    {
        using namespace osgDB;
        ReaderWriter* rw = Registry::instance()
            ->getReaderWriterForExtension(osgDB::getFileExtension(fileName));
        if (!rw)
            return ReaderWriter::ReadResult(); // FILE_NOT_HANDLED
        return rw->readNode(fileName, opt);
    }

    ProcessPolicy _processPolicy;
    CachePolicy _cachePolicy;
    OptimizePolicy _optimizePolicy;
    SubstitutePolicy _substitutePolicy;
    BVHPolicy _bvhPolicy;
    virtual ~ModelRegistryCallback() {}
};

// Predefined policies

struct DefaultProcessPolicy {
    DefaultProcessPolicy(const std::string& extension) {}
    osg::Node* process(osg::Node* node, const std::string& filename,
                       const osgDB::Options* opt);
};

struct DefaultCachePolicy {
    DefaultCachePolicy(const std::string& extension) {}
    osg::Node* find(const std::string& fileName,
                    const osgDB::Options* opt);
    void addToCache(const std::string& filename, osg::Node* node);
};

struct NoCachePolicy {
    NoCachePolicy(const std::string& extension) {}
    osg::Node* find(const std::string& fileName,
                    const osgDB::Options* opt)
    {
        return 0;
    }
    void addToCache(const std::string& filename, osg::Node* node) {}
};

class OptimizeModelPolicy {
public:
    OptimizeModelPolicy(const std::string& extension);
    osg::Node* optimize(osg::Node* node, const std::string& fileName,
                        const osgDB::Options* opt);
protected:
    unsigned _osgOptions;
};

struct NoOptimizePolicy {
    NoOptimizePolicy(const std::string& extension) {}
    osg::Node* optimize(osg::Node* node, const std::string& fileName,
                        const osgDB::Options* opt)
    {
        return node;
    }
};

struct OSGSubstitutePolicy {
    OSGSubstitutePolicy(const std::string& extension) {}
    std::string substitute(const std::string& name,
                           const osgDB::Options* opt);
};

struct NoSubstitutePolicy {
    NoSubstitutePolicy(const std::string& extension) {}
    std::string substitute(const std::string& name,
                           const osgDB::Options* opt)
    {
        return std::string();
    }
};

struct BuildLeafBVHPolicy {
    BuildLeafBVHPolicy(const std::string& extension) {}
    void buildBVH(const std::string& fileName, osg::Node* node);
};

struct BuildGroupBVHPolicy {
    BuildGroupBVHPolicy(const std::string& extension) {}
    void buildBVH(const std::string& fileName, osg::Node* node);
};

struct NoBuildBVHPolicy {
    NoBuildBVHPolicy(const std::string& extension) {}
    void buildBVH(const std::string& fileName, osg::Node* node);
};

typedef ModelRegistryCallback<DefaultProcessPolicy, DefaultCachePolicy,
                              OptimizeModelPolicy,
                              OSGSubstitutePolicy, BuildLeafBVHPolicy>
DefaultCallback;

// The manager for the callbacks
class ModelRegistry : public osgDB::Registry::ReadFileCallback,
                      public ReferencedSingleton<ModelRegistry> {
public:
    ModelRegistry();
    virtual osgDB::ReaderWriter::ReadResult
    readImage(const std::string& fileName,
              const osgDB::Options* opt);
    virtual osgDB::ReaderWriter::ReadResult
    readNode(const std::string& fileName,
             const osgDB::Options* opt);
    void addImageCallbackForExtension(const std::string& extension,
                                      osgDB::Registry::ReadFileCallback*
                                      callback);
    void addNodeCallbackForExtension(const std::string& extension,
                                     osgDB::Registry::ReadFileCallback*
                                     callback);
    virtual ~ModelRegistry() {}
protected:
    typedef std::map<std::string, osg::ref_ptr<osgDB::Registry::ReadFileCallback> >
    CallbackMap;
    CallbackMap imageCallbackMap;
    CallbackMap nodeCallbackMap;
    osg::ref_ptr<DefaultCallback> _defaultCallback;
};

// Callback that only loads the file without any caching or
// postprocessing.
typedef ModelRegistryCallback<DefaultProcessPolicy, NoCachePolicy,
                              NoOptimizePolicy,
                              NoSubstitutePolicy, BuildLeafBVHPolicy>
LoadOnlyCallback;

// Proxy for registering extension-based callbacks

template<typename T>
class ModelRegistryCallbackProxy
{
public:
    ModelRegistryCallbackProxy(std::string extension)
    {
        ModelRegistry::instance()
            ->addNodeCallbackForExtension(extension, new T(extension));
    }
};
}
#endif // _SG_MODELREGISTRY_HXX
<commit_msg>scenery: Allow switching off bvh generation.<commit_after>// ModelRegistry.hxx -- interface to the OSG model registry
//
// Copyright (C) 2005-2007 Mathias Froehlich
// Copyright (C) 2007  Tim Moore <timoore@redhat.com>
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License as
// published by the Free Software Foundation; either version 2 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#ifndef _SG_MODELREGISTRY_HXX
#define _SG_MODELREGISTRY_HXX 1

#include <osg/ref_ptr>
#include <osg/Node>
#include <osgDB/FileUtils>
#include <osgDB/FileNameUtils>
#include <osgDB/ReaderWriter>
#include <osgDB/Registry>

#include <simgear/compiler.h>
#include <simgear/scene/util/OsgSingleton.hxx>

#include <string>
#include <map>

// Class to register per file extension read callbacks with the OSG
// registry, mostly to control caching and post load optimization /
// copying that happens above the level of the ReaderWriter.
namespace simgear
{

// Different caching and optimization strategies are needed for
// different file types. Most loaded files should be optimized and the
// optimized version should be cached. When an .osg file is
// substituted for another, it is assumed to be optimized already but
// it should be cached too (under the name of the original?). .stg
// files should not be cached (that's the pager's job) but the files
// it causes to be loaded should be. .btg files are already optimized
// and shouldn't be cached.
//
// Complicating this is the effect that removing CACHE_NODES has from
// the ReaderWriter options: it switches the object cache with an
// empty one, so that's not an option for the files that could be
// loaded from a .stg file. So, we'll let
// Registry::readNodeImplementation cache a loaded file and then add
// the optimized version to the cache ourselves, replacing the
// original subgraph.
//
// To support all these options with a minimum of duplication, the
// readNode function is specified as a template with a bunch of
// pluggable (and predefined) policies.
template <typename ProcessPolicy, typename CachePolicy, typename OptimizePolicy,
          typename SubstitutePolicy, typename BVHPolicy>
class ModelRegistryCallback : public osgDB::Registry::ReadFileCallback {
public:
    ModelRegistryCallback(const std::string& extension) :
        _processPolicy(extension), _cachePolicy(extension),
        _optimizePolicy(extension),
        _substitutePolicy(extension), _bvhPolicy(extension)
    {
    }
    virtual osgDB::ReaderWriter::ReadResult
    readNode(const std::string& fileName,
             const osgDB::Options* opt)
    {
        using namespace osg;
        using namespace osgDB;
        using osgDB::ReaderWriter;
//        Registry* registry = Registry::instance();
        ref_ptr<osg::Node> optimizedNode = _cachePolicy.find(fileName, opt);
        if (!optimizedNode.valid()) {
            std::string otherFileName = _substitutePolicy.substitute(fileName,
                                                                     opt);
            ReaderWriter::ReadResult res;
            if (!otherFileName.empty()) {
                res = loadUsingReaderWriter(otherFileName, opt);
                if (res.validNode())
                    optimizedNode = res.getNode();
            }
            if (!optimizedNode.valid()) {
                res = loadUsingReaderWriter(fileName, opt);
                if (!res.validNode())
                    return res;
                ref_ptr<osg::Node> processedNode
                    = _processPolicy.process(res.getNode(), fileName, opt);
                optimizedNode = _optimizePolicy.optimize(processedNode.get(),
                                                         fileName, opt);
            }
            if (opt->getPluginStringData("SimGear::BOUNDINGVOLUMES") != "OFF")
                _bvhPolicy.buildBVH(fileName, optimizedNode.get());
            _cachePolicy.addToCache(fileName, optimizedNode.get());
        }
        return ReaderWriter::ReadResult(optimizedNode.get());
    }
protected:
    static osgDB::ReaderWriter::ReadResult
    loadUsingReaderWriter(const std::string& fileName,
                          const osgDB::Options* opt)
    {
        using namespace osgDB;
        ReaderWriter* rw = Registry::instance()
            ->getReaderWriterForExtension(osgDB::getFileExtension(fileName));
        if (!rw)
            return ReaderWriter::ReadResult(); // FILE_NOT_HANDLED
        return rw->readNode(fileName, opt);
    }

    ProcessPolicy _processPolicy;
    CachePolicy _cachePolicy;
    OptimizePolicy _optimizePolicy;
    SubstitutePolicy _substitutePolicy;
    BVHPolicy _bvhPolicy;
    virtual ~ModelRegistryCallback() {}
};

// Predefined policies

struct DefaultProcessPolicy {
    DefaultProcessPolicy(const std::string& extension) {}
    osg::Node* process(osg::Node* node, const std::string& filename,
                       const osgDB::Options* opt);
};

struct DefaultCachePolicy {
    DefaultCachePolicy(const std::string& extension) {}
    osg::Node* find(const std::string& fileName,
                    const osgDB::Options* opt);
    void addToCache(const std::string& filename, osg::Node* node);
};

struct NoCachePolicy {
    NoCachePolicy(const std::string& extension) {}
    osg::Node* find(const std::string& fileName,
                    const osgDB::Options* opt)
    {
        return 0;
    }
    void addToCache(const std::string& filename, osg::Node* node) {}
};

class OptimizeModelPolicy {
public:
    OptimizeModelPolicy(const std::string& extension);
    osg::Node* optimize(osg::Node* node, const std::string& fileName,
                        const osgDB::Options* opt);
protected:
    unsigned _osgOptions;
};

struct NoOptimizePolicy {
    NoOptimizePolicy(const std::string& extension) {}
    osg::Node* optimize(osg::Node* node, const std::string& fileName,
                        const osgDB::Options* opt)
    {
        return node;
    }
};

struct OSGSubstitutePolicy {
    OSGSubstitutePolicy(const std::string& extension) {}
    std::string substitute(const std::string& name,
                           const osgDB::Options* opt);
};

struct NoSubstitutePolicy {
    NoSubstitutePolicy(const std::string& extension) {}
    std::string substitute(const std::string& name,
                           const osgDB::Options* opt)
    {
        return std::string();
    }
};

struct BuildLeafBVHPolicy {
    BuildLeafBVHPolicy(const std::string& extension) {}
    void buildBVH(const std::string& fileName, osg::Node* node);
};

struct BuildGroupBVHPolicy {
    BuildGroupBVHPolicy(const std::string& extension) {}
    void buildBVH(const std::string& fileName, osg::Node* node);
};

struct NoBuildBVHPolicy {
    NoBuildBVHPolicy(const std::string& extension) {}
    void buildBVH(const std::string& fileName, osg::Node* node);
};

typedef ModelRegistryCallback<DefaultProcessPolicy, DefaultCachePolicy,
                              OptimizeModelPolicy,
                              OSGSubstitutePolicy, BuildLeafBVHPolicy>
DefaultCallback;

// The manager for the callbacks
class ModelRegistry : public osgDB::Registry::ReadFileCallback,
                      public ReferencedSingleton<ModelRegistry> {
public:
    ModelRegistry();
    virtual osgDB::ReaderWriter::ReadResult
    readImage(const std::string& fileName,
              const osgDB::Options* opt);
    virtual osgDB::ReaderWriter::ReadResult
    readNode(const std::string& fileName,
             const osgDB::Options* opt);
    void addImageCallbackForExtension(const std::string& extension,
                                      osgDB::Registry::ReadFileCallback*
                                      callback);
    void addNodeCallbackForExtension(const std::string& extension,
                                     osgDB::Registry::ReadFileCallback*
                                     callback);
    virtual ~ModelRegistry() {}
protected:
    typedef std::map<std::string, osg::ref_ptr<osgDB::Registry::ReadFileCallback> >
    CallbackMap;
    CallbackMap imageCallbackMap;
    CallbackMap nodeCallbackMap;
    osg::ref_ptr<DefaultCallback> _defaultCallback;
};

// Callback that only loads the file without any caching or
// postprocessing.
typedef ModelRegistryCallback<DefaultProcessPolicy, NoCachePolicy,
                              NoOptimizePolicy,
                              NoSubstitutePolicy, BuildLeafBVHPolicy>
LoadOnlyCallback;

// Proxy for registering extension-based callbacks

template<typename T>
class ModelRegistryCallbackProxy
{
public:
    ModelRegistryCallbackProxy(std::string extension)
    {
        ModelRegistry::instance()
            ->addNodeCallbackForExtension(extension, new T(extension));
    }
};
}
#endif // _SG_MODELREGISTRY_HXX
<|endoftext|>


<commit_before>#include "stdafx.h"
#include "Wind_API.h"

#include "util.h"
#include "WindEvent.h"
#include "SocketPair.h"

#include "kdb+.util/K_ptr.h"
#include "kdb+.util/type_convert.h"
#include <iostream>
#include <mutex>	//C++11
#include <map>

namespace Wind {
	namespace pubsub {

		bool prepare() {
			return SockPair::prepare();
		}

		bool finalize() {
			return SockPair::finalize();
		}

		// This is essentially a concurrent map...
		//NOTE: All std::map<> interface are _not_ protected! Use only the member functions wrapped below.
		class CallbackRegistry : public std::map<::WQID, std::string> {
		public:
			std::string& operator[](::WQID qid) {
				lock_guard guard(lock_);
				return parent::operator[](qid);
			}
			size_type erase(::WQID qid) {
				lock_guard guard(lock_);
				return parent::erase(qid);
			}
		private:
			typedef std::map<::WQID, std::string> parent;
			typedef std::lock_guard<std::mutex> lock_guard;
			mutable std::mutex lock_;
		};

		static CallbackRegistry REGISTRY;
		std::size_t const CLIENT = 0;
		std::size_t const SERVER = 1;

		// Data processor (executed within q's main thread)
		K invokeCallback(I socket) {
			static_assert(sizeof(::SOCKET) == sizeof(I), "SOCKET vs I: type mismatch!");
			::SOCKET sock = socket;
			assert(sock != INVALID_SOCKET);

			// Receive (WQID, len, serialized_K) from subscription thread
#			define RECV_CHECK(expectedSize, errorMsg)	\
			if (recvd != (expectedSize)) {	\
				std::cerr << "<recv> " << (errorMsg) << ": " << recvd << " < " << (expectedSize) << std::endl;	\
				return K_NIL;	\
			}
			::WQID qid = 0;
			int recvd = ::recv(sock, reinterpret_cast<char*>(&qid), sizeof(::WQID), 0);
			RECV_CHECK(sizeof(::WQID), "WQID incomplete");
			J len = 0;
			recvd = ::recv(sock, reinterpret_cast<char*>(&len), sizeof(J), 0);
			RECV_CHECK(sizeof(J), "size incomplete");
			std::vector<char> buffer(static_cast<std::size_t>(len), 0);
			recvd = ::recv(sock, &buffer[0], buffer.size(), 0);
			RECV_CHECK(len, "data incomplete");
#			undef RECV_CHECK

			// Deserialize K object
			q::K_ptr serialized(ktn(KB, len));
			std::memcpy(kG(serialized.get()), &buffer[0], buffer.size());
#			if 0	//TODO: okx(K) will only be available in q.lib 3.2... to check back...
			if (!okx(serialized.get())) {
				std::cerr << "<recv> invalid data: ";
				for (auto p = buffer.begin(); p != buffer.end(); ++p) {
					std::cerr << std::setiosflags(std::ios::uppercase)
						<< std::setfill('0') << std::setw(2) << std::hex << static_cast<int>(*p);
				}
				std::cerr << std::endl;
				return K_NIL;
			}
#			endif
			q::K_ptr result(d9(serialized.get()));

			// Identify the origial query and callback
			std::string const callback = REGISTRY[qid];
			if (callback.empty()) {
				std::cerr << "unknown WQID: " << qid << std::endl;
				return K_NIL;
			}

			static_assert(std::is_same<::WQID, J>::value, "WQID data type mismatch");
			q::K_ptr output(k(0, const_cast<S>(callback.c_str()), kj(qid), result.release(), K_NIL));
			if (output->t == -128) {
				std::cerr << "<q> '" << output->s << std::endl;
			}
			return output.release();
		}

		// Result publisher (executed within subscription thread)
		bool publishResult(Event const& event, SockPair::SOCKET_ptr const* socks) {
			// Convert Wind event into K object
			std::string error;
			q::K_ptr data;
			if (event.ErrCode != WQERR_OK) {
				std::cerr << "<WQ> subscription error: " << util::error2Text(event.ErrCode) << std::endl;
				return false;
			}
			try {
				data.reset(event.parse());
			}
			catch (std::string const& ex) {
				std::cerr << "<WQ> response format error: " << ex << std::endl;
				return false;
			}

			// Serialize K object into bytes
			static_assert(std::is_same<::WQID, J>::value, "WQID data type mismatch");
			q::K_ptr serialized(b9(-1, data.get()));
			assert((serialized->t == KG) && (serialized->n > 0));

			// Send tuple (WQID, len, serialized_K) over to the main thread
#			define SEND_CHECK(expectedSize, errorMsg)	\
			if (sent != (expectedSize)) {	\
				std::cerr << "<send> " << (errorMsg) << ": " << sent << " < " << (expectedSize) << std::endl;	\
				return false;	\
			}
			int sent = ::send(*socks[SERVER], reinterpret_cast<char const*>(&event.RequestID), sizeof(::WQID), 0);
			SEND_CHECK(sizeof(::WQID), "WQID incomplete");
			std::vector<char> buffer(kG(serialized.get()), kG(serialized.get()) + serialized->n);
			sent = ::send(*socks[SERVER], reinterpret_cast<char const*>(&(serialized->n)), sizeof(serialized->n), 0);
			SEND_CHECK(sizeof(serialized->n), "size incomplete");
			sent = ::send(*socks[SERVER], &buffer[0], buffer.size(), 0);
			SEND_CHECK(serialized->n, "data incomplete");
#			undef SEND_CHECK

			return true;
		}

		int WINAPI subscribe(::WQEvent* pEvent, LPVOID lpUserParam) {
			SockPair::SOCKET_ptr* socks = static_cast<SockPair::SOCKET_ptr*>(lpUserParam);
			assert(socks != NULL);

			assert(pEvent != NULL);
			switch (pEvent->EventType) {
			case eWQPartialResponse:
				return publishResult(static_cast<Event&>(*pEvent), socks);
			case eWQErrorReport:
				std::cerr << "<WQ> error report: " << util::error2Text(pEvent->ErrCode) << std::endl;
				return false;
			case eWQOthers:
				// Detect unsubscribe and free up socket pair
				if (pEvent->ErrCode == WQERR_USER_CANCEL) {
					sd0(*socks[CLIENT]);
					REGISTRY.erase(pEvent->RequestID);
					delete[] socks;
					return true;
				}
				else {
					// fall through
				}
			default:
				std::cerr << "<WQ> unsupported subscription response: " << *pEvent << std::endl;
				return false;
			}
		}

	}//namespace Wind::pubsub
}//namespace Wind

WIND_API K K_DECL Wind_wsq(K windCodes, K indicators, K params, K callback) {
	std::wstring codes, indis, paras;
	std::string cb;
	try {
		codes = Wind::util::qList2WStringJoin(windCodes, L',');
		indis = Wind::util::qList2WStringJoin(indicators, L',');
		paras = Wind::util::qDict2WStringMapJoin(params, L';', L'=');
		cb = q::q2String(callback);
	}
	catch (std::string const& error) {
		return q::error2q(error);
	}

	// Create async socket pair
	std::unique_ptr<SockPair::SOCKET_ptr[]> socks(new ::SockPair::SOCKET_ptr[2]);
	::SOCKET server, client;
	int result = SockPair::make_socket_pair(server, client);
	if (result != 0) {
		std::ostringstream buffer;
		buffer << "<SockPair> " << SockPair::getError(result);
		return q::error2q(buffer.str());
	}
	using Wind::pubsub::SERVER;
	using Wind::pubsub::CLIENT;
	socks[SERVER].reset(server);
	socks[CLIENT].reset(client);

	// Setup subscription
	static_assert(sizeof(::SOCKET) == sizeof(I), "SOCKET vs I: type mismatch!");
	::WQID const qid = ::WSQ(codes.c_str(), indis.c_str(), paras.c_str(),
		&Wind::pubsub::subscribe, socks.get());
	if (qid <= 0) {
		std::ostringstream buffer;
		buffer << "<WQ> subscription error: " << Wind::util::error2Text(static_cast<::WQErr>(qid));
		return q::error2q(buffer.str());
	}
	else {
		sd1(*socks[CLIENT], &Wind::pubsub::invokeCallback);
		Wind::pubsub::REGISTRY[qid] = cb;
		socks.release();
		static_assert(std::is_same<::WQID, J>::value, "WQID data type mismatch");
		return kj(qid);
	}
}

WIND_API K K_DECL Wind_cr(K qid) {
	::WQID id = -1;
	static_assert(std::is_same<::WQID, J>::value, "WQID data type mismatch");
	try {
		id = q::q2Dec(qid);
	}
	catch (std::string const& error) {
		return q::error2q(error);
	}

	::WQErr const error = ::CancelRequest(id);
	if (error != WQERR_OK) {
		std::ostringstream buffer;
		buffer << "<WQ> cancellation error: " << Wind::util::error2Text(error);
		return q::error2q(buffer.str());
	}
	else {
		return K_NIL;
	}
}

WIND_API K K_DECL Wind_car(K _) {
	::WQErr const error = ::CancelAllRequest();
	if (error != WQERR_OK) {
		std::ostringstream buffer;
		buffer << "<WQ> cancellation error: " << Wind::util::error2Text(error);
		return q::error2q(buffer.str());
	}
	else {
		return K_NIL;
	}
}
<commit_msg>+ Prevent accidental misuse of Wind::pubsub::REGISTRY + Reduce data copying on data serialization/deserialization<commit_after>#include "stdafx.h"
#include "Wind_API.h"

#include "util.h"
#include "WindEvent.h"
#include "SocketPair.h"

#include "kdb+.util/K_ptr.h"
#include "kdb+.util/type_convert.h"
#include <iostream>
#include <mutex>	//C++11
#include <map>

namespace Wind {
	namespace pubsub {

		bool prepare() {
			return SockPair::prepare();
		}

		bool finalize() {
			return SockPair::finalize();
		}

		// This is essentially a concurrent map...
		//NOTE: All std::map<> interface are _not_ protected! Use only the member functions wrapped below.
		class CallbackRegistry : private std::map<::WQID, std::string> {
		public:
			std::string& operator[](::WQID qid) {
				lock_guard guard(lock_);
				return parent::operator[](qid);
			}
			size_type erase(::WQID qid) {
				lock_guard guard(lock_);
				return parent::erase(qid);
			}
		private:
			typedef std::map<::WQID, std::string> parent;
			typedef std::lock_guard<std::mutex> lock_guard;
			mutable std::mutex lock_;
		};

		static CallbackRegistry REGISTRY;
		std::size_t const CLIENT = 0;
		std::size_t const SERVER = 1;

		// Data processor (executed within q's main thread)
		K invokeCallback(I socket) {
			static_assert(sizeof(::SOCKET) == sizeof(I), "SOCKET vs I: type mismatch!");
			::SOCKET sock = socket;
			assert(sock != INVALID_SOCKET);

			// Receive (WQID, len, serialized_K) from subscription thread
#			define RECV_CHECK(expectedSize, errorMsg)	\
			if (recvd != (expectedSize)) {	\
				std::cerr << "<recv> " << (errorMsg) << ": " << recvd << " < " << (expectedSize) << std::endl;	\
				return K_NIL;	\
			}
			::WQID qid = 0;
			int recvd = ::recv(sock, reinterpret_cast<char*>(&qid), sizeof(::WQID), 0);
			RECV_CHECK(sizeof(::WQID), "WQID incomplete");
			std::size_t len = 0;
			recvd = ::recv(sock, reinterpret_cast<char*>(&len), sizeof(len), 0);
			RECV_CHECK(sizeof(len), "size incomplete");
			if (len > static_cast<std::size_t>(std::numeric_limits<int>::max())) {
				std::cerr << "<recv> serialized data (" << len << ") > 2G" << std::endl;
				return K_NIL;
			}
			q::K_ptr serialized(ktn(KB, len));
			std::memset(kG(serialized.get()), 0, len);
			recvd = ::recv(sock, reinterpret_cast<char*>(kG(serialized.get())), len, 0);
			RECV_CHECK(len, "data incomplete");
#			undef RECV_CHECK

			// Deserialize K object
#			if 0	//TODO: okx(K) will only be available in q.lib 3.2... to check back...
			if (!okx(serialized.get())) {
				std::cerr << "<recv> invalid data: ";
				for (auto p = buffer.begin(); p != buffer.end(); ++p) {
					std::cerr << std::setiosflags(std::ios::uppercase)
						<< std::setfill('0') << std::setw(2) << std::hex << static_cast<int>(*p);
				}
				std::cerr << std::endl;
				return K_NIL;
			}
#			endif
			q::K_ptr result(d9(serialized.get()));

			// Identify the origial query and callback
			std::string const callback = REGISTRY[qid];
			if (callback.empty()) {
				std::cerr << "unknown WQID: " << qid << std::endl;
				return K_NIL;
			}

			static_assert(std::is_same<::WQID, J>::value, "WQID data type mismatch");
			q::K_ptr output(k(0, const_cast<S>(callback.c_str()), kj(qid), result.release(), K_NIL));
			if (output->t == -128) {
				std::cerr << "<q> '" << output->s << std::endl;
			}
			return output.release();
		}

		// Result publisher (executed within subscription thread)
		bool publishResult(Event const& event, SockPair::SOCKET_ptr const* socks) {
			// Convert Wind event into K object
			std::string error;
			q::K_ptr data;
			if (event.ErrCode != WQERR_OK) {
				std::cerr << "<WQ> subscription error: " << util::error2Text(event.ErrCode) << std::endl;
				return false;
			}
			try {
				data.reset(event.parse());
			}
			catch (std::string const& ex) {
				std::cerr << "<WQ> response format error: " << ex << std::endl;
				return false;
			}

			// Serialize K object into bytes
			static_assert(std::is_same<::WQID, J>::value, "WQID data type mismatch");
			q::K_ptr serialized(b9(-1, data.get()));
			assert((serialized->t == KG) && (serialized->n > 0));

			// Send tuple (WQID, len, serialized_K) over to the main thread
#			define SEND_CHECK(expectedSize, errorMsg)	\
			if (sent != (expectedSize)) {	\
				std::cerr << "<send> " << (errorMsg) << ": " << sent << " < " << (expectedSize) << std::endl;	\
				return false;	\
			}
			int sent = ::send(*socks[SERVER], reinterpret_cast<char const*>(&event.RequestID), sizeof(::WQID), 0);
			SEND_CHECK(sizeof(::WQID), "WQID incomplete");
			std::size_t const len = static_cast<std::size_t>(serialized->n);
			if (len > static_cast<std::size_t>(std::numeric_limits<int>::max())) {
				std::cerr << "<send> serialized data (" << len << ") > 2G" << std::endl;
				return false;
			}
			sent = ::send(*socks[SERVER], reinterpret_cast<char const*>(&len), sizeof(len), 0);
			SEND_CHECK(sizeof(len), "size incomplete");
			sent = ::send(*socks[SERVER], reinterpret_cast<char const*>(kG(serialized.get())), len, 0);
			SEND_CHECK(serialized->n, "data incomplete");
#			undef SEND_CHECK

			return true;
		}

		int WINAPI subscribe(::WQEvent* pEvent, LPVOID lpUserParam) {
			SockPair::SOCKET_ptr* socks = static_cast<SockPair::SOCKET_ptr*>(lpUserParam);
			assert(socks != NULL);

			assert(pEvent != NULL);
			switch (pEvent->EventType) {
			case eWQPartialResponse:
				return publishResult(static_cast<Event&>(*pEvent), socks);
			case eWQErrorReport:
				std::cerr << "<WQ> error report: " << util::error2Text(pEvent->ErrCode) << std::endl;
				return false;
			case eWQOthers:
				// Detect unsubscribe and free up socket pair
				if (pEvent->ErrCode == WQERR_USER_CANCEL) {
					sd0(*socks[CLIENT]);
					REGISTRY.erase(pEvent->RequestID);
					delete[] socks;
					return true;
				}
				else {
					// fall through
				}
			default:
				std::cerr << "<WQ> unsupported subscription response: " << *pEvent << std::endl;
				return false;
			}
		}

	}//namespace Wind::pubsub
}//namespace Wind

WIND_API K K_DECL Wind_wsq(K windCodes, K indicators, K params, K callback) {
	std::wstring codes, indis, paras;
	std::string cb;
	try {
		codes = Wind::util::qList2WStringJoin(windCodes, L',');
		indis = Wind::util::qList2WStringJoin(indicators, L',');
		paras = Wind::util::qDict2WStringMapJoin(params, L';', L'=');
		cb = q::q2String(callback);
	}
	catch (std::string const& error) {
		return q::error2q(error);
	}

	// Create async socket pair
	std::unique_ptr<SockPair::SOCKET_ptr[]> socks(new ::SockPair::SOCKET_ptr[2]);
	::SOCKET server, client;
	int result = SockPair::make_socket_pair(server, client);
	if (result != 0) {
		std::ostringstream buffer;
		buffer << "<SockPair> " << SockPair::getError(result);
		return q::error2q(buffer.str());
	}
	using Wind::pubsub::SERVER;
	using Wind::pubsub::CLIENT;
	socks[SERVER].reset(server);
	socks[CLIENT].reset(client);

	// Setup subscription
	static_assert(sizeof(::SOCKET) == sizeof(I), "SOCKET vs I: type mismatch!");
	::WQID const qid = ::WSQ(codes.c_str(), indis.c_str(), paras.c_str(),
		&Wind::pubsub::subscribe, socks.get());
	if (qid <= 0) {
		std::ostringstream buffer;
		buffer << "<WQ> subscription error: " << Wind::util::error2Text(static_cast<::WQErr>(qid));
		return q::error2q(buffer.str());
	}
	else {
		sd1(*socks[CLIENT], &Wind::pubsub::invokeCallback);
		Wind::pubsub::REGISTRY[qid] = cb;
		socks.release();
		static_assert(std::is_same<::WQID, J>::value, "WQID data type mismatch");
		return kj(qid);
	}
}

WIND_API K K_DECL Wind_cr(K qid) {
	::WQID id = -1;
	static_assert(std::is_same<::WQID, J>::value, "WQID data type mismatch");
	try {
		id = q::q2Dec(qid);
	}
	catch (std::string const& error) {
		return q::error2q(error);
	}

	::WQErr const error = ::CancelRequest(id);
	if (error != WQERR_OK) {
		std::ostringstream buffer;
		buffer << "<WQ> cancellation error: " << Wind::util::error2Text(error);
		return q::error2q(buffer.str());
	}
	else {
		return K_NIL;
	}
}

WIND_API K K_DECL Wind_car(K _) {
	::WQErr const error = ::CancelAllRequest();
	if (error != WQERR_OK) {
		std::ostringstream buffer;
		buffer << "<WQ> cancellation error: " << Wind::util::error2Text(error);
		return q::error2q(buffer.str());
	}
	else {
		return K_NIL;
	}
}
<|endoftext|>


<commit_before>///
/// @file  phi_mpi.cpp
/// @brief The PhiCache class calculates the partial sieve function
///        (a.k.a. Legendre-sum) using the recursive formula:
///        phi(x, a) = phi(x, a - 1) - phi(x / primes[a], a - 1).
///        phi(x, a) counts the numbers <= x that are not divisible by
///        any of the first a primes. The algorithm used is an
///        optimized version of the algorithm described in Tomás
///        Oliveira e Silva's paper [1]. I have added 5 optimizations
///        to my implementation which significantly speed up the
///        calculation:
///
///        * Cache results of phi(x, a)
///        * Calculate phi(x, a) using formula [2] if a <= 6
///        * Calculate phi(x, a) using pi(x) lookup table
///        * Calculate all phi(x, a) = 1 upfront
///        * Stop recursion at c instead of 1
///
///       [1] Tomás Oliveira e Silva, Computing pi(x): the combinatorial
///           method, Revista do DETUA, vol. 4, no. 6, March 2006, p. 761.
///           http://sweet.ua.pt/tos/bib/5.4.pdf
///       [2] phi(x, a) = (x / pp) * φ(pp) + phi(x % pp, a)
///           with pp = 2 * 3 * ... * prime[a]
///
/// Copyright (C) 2016 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <PiTable.hpp>
#include <primecount-internal.hpp>
#include <primecount.hpp>
#include <pmath.hpp>
#include <PhiTiny.hpp>
#include <min_max.hpp>

#include <stdint.h>
#include <algorithm>
#include <vector>
#include <cassert>
#include <limits>

#ifdef _OPENMP
  #include <omp.h>
#endif

using namespace std;
using namespace primecount;

namespace {

class PhiCache
{
public:
  PhiCache(const vector<int64_t>& primes,
           const PiTable& pi) :
    primes_(primes),
    pi_(pi),
    bytes_(0)
  {
    // primecount uses 1-indexing i.e. primes[1] = 2
    assert(primes_[0] == 0);
    size_t max_size = CACHE_A_LIMIT + 1;
    cache_.resize(min(primes.size(), max_size));
  }

  /// Calculate phi(x, a) using the recursive formula:
  /// phi(x, a) = phi(x, a - 1) - phi(x / primes_[a], a - 1)
  ///
  template <int SIGN>
  int64_t phi(int64_t x, int64_t a)
  {
    int64_t sum = 0;

    if (x <= primes_[a])
      sum = SIGN;
    else if (is_phi_tiny(a))
      sum = phi_tiny(x, a) * SIGN;
    else if (is_phi_by_pix(x, a))
      sum = (pi_[x] - a + 1) * SIGN;
    else
    {
      int64_t sqrtx = isqrt(x);
      int64_t pi_sqrtx = a;

      if (sqrtx < pi_.size() && sqrtx < primes_[a])
        pi_sqrtx = pi_[sqrtx];

      // Move out of the loop the calculations where phi(x2, a2) = 1
      // phi(x, a) = 1 if primes_[a] >= x
      // x2 = x / primes_[a2 + 1]
      // phi(x2, a2) = 1 if primes_[a2] >= x / primes_[a2 + 1]
      // phi(x2, a2) = 1 if primes_[a2] >= sqrt(x)
      // phi(x2, a2) = 1 if a2 >= pi(sqrt(x))
      // \sum_{a2 = pi(sqrt(x))}^{a - 1} phi(x2, a2) = a - pi(sqrt(x))
      //
      sum = (a - pi_sqrtx) * -SIGN;

      // phi(x, c) = phi(x, 1) - \sum_{a2 = 1}^{c - 1} phi(x / primes_[a2 + 1], a2)
      int64_t c = min(PhiTiny::max_a(), pi_sqrtx);
      sum += phi_tiny(x, c) * SIGN;
      int64_t a2 = c;

      // 64-bit integer division, slow
      for (; a2 < pi_sqrtx && x > numeric_limits<uint32_t>::max(); a2++)
      {
        int64_t x2 = x / primes_[a2 + 1];
        if (is_cached(x2, a2))
          sum += cache_[a2][x2] * -SIGN;
        else
          sum += phi<-SIGN>(x2, a2);
      }

      // 32-bit integer division, faster
      for (; a2 < pi_sqrtx; a2++)
      {
        int64_t x2 = (uint32_t) x / (uint32_t) primes_[a2 + 1];
        if (is_cached(x2, a2))
          sum += cache_[a2][x2] * -SIGN;
        else
          sum += phi<-SIGN>(x2, a2);
      }
    }

    if (write_to_cache(x, a))
      cache_[a][x] = (uint16_t) (sum * SIGN);

    return sum;
  }

private:
  enum
  {
    /// Cache phi(x, a) results if a <= CACHE_A_LIMIT
    CACHE_A_LIMIT = 500,
    /// Keep the cache size below CACHE_BYTES_LIMIT per thread
    CACHE_BYTES_LIMIT = 16 << 20
  };

  /// Cache of phi(x, a) results
  vector<vector<uint16_t> > cache_;
  const vector<int64_t>& primes_;
  const PiTable& pi_;
  int64_t bytes_;

  void operator=(const PhiCache&);

  bool is_phi_by_pix(int64_t x, int64_t a) const
  {
    return x < pi_.size() &&
           x < isquare(primes_[a + 1]);
  }

  int64_t cache_size(int64_t a) const
  {
    return (int64_t) cache_[a].size();
  }

  bool is_cached(int64_t x, int64_t a) const
  {
    return a <= CACHE_A_LIMIT &&
           x < cache_size(a) &&
           cache_[a][x] != 0;
  }

  bool write_to_cache(int64_t x, int64_t a)
  {
    if (a > CACHE_A_LIMIT ||
        x > numeric_limits<uint16_t>::max())
      return false;

    // check if we need to increase cache size
    if (x >= cache_size(a))
    {
      if (bytes_ > CACHE_BYTES_LIMIT)
        return false;
      bytes_ += (x + 1 - cache_size(a)) * 2;
      cache_[a].resize(x + 1, 0);
    }

    return true;
  }
};

} // namespace

namespace primecount {

/// Returns a vector with phi(x, i) values for 0 <= i <= a.
/// phi(x, a) counts the numbers <= x that are not divisible by any
/// of the first a primes.
///
vector<int64_t> phi_vector(int64_t x,
                           int64_t a,
                           const vector<int64_t>& primes,
                           const PiTable& pi,
                           int threads)
{
  vector<int64_t> phi(a + 2, 0);

  // phi(x, 0) = x
  // primecount uses 1-indexing
  phi[1] = x;

  if (x > 0 && a > 0)
  {
    PhiCache cache(primes, pi);

    int64_t p14 = ipow((int64_t) 10, 14);
    int64_t thread_threshold = p14 / primes[a];
    threads = ideal_num_threads(threads, x, thread_threshold);

    // this loop scales only up to about 8 CPU cores
    // because the cache requires too much memory bandwidth
    threads = min(8, threads);

    #pragma omp parallel for num_threads(threads) schedule(dynamic, 16) firstprivate(cache)
    for (int64_t a2 = 2; a2 <= a; a2++)
      phi[a2] = cache.phi<-1>(x / primes[a2 - 1], a2 - 2);

    // calculate phi(x, i) using partial results
    for (int64_t i = 2; i <= a; i++)
      phi[i] += phi[i - 1];
  }

  return phi;
}

} // namespace
<commit_msg>Refactor<commit_after>///
/// @file  phi_mpi.cpp
/// @brief The PhiCache class calculates the partial sieve function
///        (a.k.a. Legendre-sum) using the recursive formula:
///        phi(x, a) = phi(x, a - 1) - phi(x / primes[a], a - 1).
///        phi(x, a) counts the numbers <= x that are not divisible by
///        any of the first a primes. The algorithm used is an
///        optimized version of the algorithm described in Tomás
///        Oliveira e Silva's paper [1]. I have added 5 optimizations
///        to my implementation which significantly speed up the
///        calculation:
///
///        * Cache results of phi(x, a)
///        * Calculate phi(x, a) using formula [2] if a <= 6
///        * Calculate phi(x, a) using pi(x) lookup table
///        * Calculate all phi(x, a) = 1 upfront
///        * Stop recursion at c instead of 1
///
///       [1] Tomás Oliveira e Silva, Computing pi(x): the combinatorial
///           method, Revista do DETUA, vol. 4, no. 6, March 2006, p. 761.
///           http://sweet.ua.pt/tos/bib/5.4.pdf
///       [2] phi(x, a) = (x / pp) * φ(pp) + phi(x % pp, a)
///           with pp = 2 * 3 * ... * prime[a]
///
/// Copyright (C) 2016 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <PiTable.hpp>
#include <primecount-internal.hpp>
#include <primecount.hpp>
#include <pmath.hpp>
#include <PhiTiny.hpp>
#include <min_max.hpp>

#include <stdint.h>
#include <algorithm>
#include <vector>
#include <limits>

#ifdef _OPENMP
  #include <omp.h>
#endif

using namespace std;
using namespace primecount;

namespace {

class PhiCache
{
public:
  PhiCache(const vector<int64_t>& primes,
           const PiTable& pi) :
    primes_(primes),
    pi_(pi),
    bytes_(0)
  {
    size_t max_size = MAX_A + 1;
    size_t size = min(primes.size(), max_size);
    cache_.resize(size);
  }

  /// Calculate phi(x, a) using the recursive formula:
  /// phi(x, a) = phi(x, a - 1) - phi(x / primes_[a], a - 1)
  ///
  template <int SIGN>
  int64_t phi(int64_t x, int64_t a)
  {
    int64_t sum = 0;

    if (x <= primes_[a])
      sum = SIGN;
    else if (is_phi_tiny(a))
      sum = phi_tiny(x, a) * SIGN;
    else if (is_phi_by_pix(x, a))
      sum = (pi_[x] - a + 1) * SIGN;
    else
    {
      int64_t sqrtx = isqrt(x);
      int64_t pi_sqrtx = a;

      if (sqrtx < pi_.size() && sqrtx < primes_[a])
        pi_sqrtx = pi_[sqrtx];

      // Move out of the loop the calculations where phi(x2, a2) = 1
      // phi(x, a) = 1 if primes_[a] >= x
      // x2 = x / primes_[a2 + 1]
      // phi(x2, a2) = 1 if primes_[a2] >= x / primes_[a2 + 1]
      // phi(x2, a2) = 1 if primes_[a2] >= sqrt(x)
      // phi(x2, a2) = 1 if a2 >= pi(sqrt(x))
      // \sum_{a2 = pi(sqrt(x))}^{a - 1} phi(x2, a2) = a - pi(sqrt(x))
      //
      sum = (a - pi_sqrtx) * -SIGN;

      // phi(x, c) = phi(x, 1) - \sum_{a2 = 1}^{c - 1} phi(x / primes_[a2 + 1], a2)
      int64_t c = min(PhiTiny::max_a(), pi_sqrtx);
      sum += phi_tiny(x, c) * SIGN;
      int64_t a2 = c;

      // 64-bit integer division, slow
      for (; a2 < pi_sqrtx &&
             x > numeric_limits<uint32_t>::max(); a2++)
      {
        int64_t x2 = x / primes_[a2 + 1];
        if (is_cached(x2, a2))
          sum += cache_[a2][x2] * -SIGN;
        else
          sum += phi<-SIGN>(x2, a2);
      }

      // 32-bit integer division, fast
      for (; a2 < pi_sqrtx; a2++)
      {
        int64_t x2 = (uint32_t) x / (uint32_t) primes_[a2 + 1];
        if (is_cached(x2, a2))
          sum += cache_[a2][x2] * -SIGN;
        else
          sum += phi<-SIGN>(x2, a2);
      }
    }

    if (write_to_cache(x, a))
      cache_[a][x] = (uint16_t) (sum * SIGN);

    return sum;
  }

private:
  enum
  {
    /// Cache phi(x, a) results if a <= MAX_A
    MAX_A = 500,
    /// Keep the cache size below MAX_BYTES per thread
    MAX_BYTES = 16 << 20
  };

  /// Cache of phi(x, a) results
  vector<vector<uint16_t> > cache_;
  const vector<int64_t>& primes_;
  const PiTable& pi_;
  int64_t bytes_;

  void operator=(const PhiCache&);

  bool is_phi_by_pix(int64_t x, int64_t a) const
  {
    return x < pi_.size() &&
           x < isquare(primes_[a + 1]);
  }

  int64_t cache_size(int64_t a) const
  {
    return (int64_t) cache_[a].size();
  }

  bool is_cached(int64_t x, int64_t a) const
  {
    return a <= MAX_A &&
           x < cache_size(a) &&
           cache_[a][x] != 0;
  }

  bool write_to_cache(int64_t x, int64_t a)
  {
    if (a > MAX_A ||
        x > numeric_limits<uint16_t>::max())
      return false;

    //  we need to increase cache size
    if (x >= cache_size(a))
    {
      if (bytes_ > MAX_BYTES)
        return false;
      bytes_ += (x + 1 - cache_size(a)) * 2;
      cache_[a].resize(x + 1, 0);
    }

    return true;
  }
};

} // namespace

namespace primecount {

/// Returns a vector with phi(x, i) values for 0 <= i <= a.
/// phi(x, a) counts the numbers <= x that are not divisible by any
/// of the first a primes.
///
vector<int64_t> phi_vector(int64_t x,
                           int64_t a,
                           const vector<int64_t>& primes,
                           const PiTable& pi,
                           int threads)
{
  vector<int64_t> phi(a + 2, 0);
  phi[1] = x;

  if (x > 0 && a > 0)
  {
    PhiCache cache(primes, pi);

    int64_t p14 = ipow((int64_t) 10, 14);
    int64_t thread_threshold = p14 / primes[a];
    threads = ideal_num_threads(threads, x, thread_threshold);

    // this loop scales only up to about 8 CPU cores
    // because the cache requires too much memory bandwidth
    threads = min(8, threads);

    #pragma omp parallel for num_threads(threads) schedule(dynamic, 16) firstprivate(cache)
    for (int64_t a2 = 2; a2 <= a; a2++)
      phi[a2] = cache.phi<-1>(x / primes[a2 - 1], a2 - 2);

    // calculate phi(x, i) using partial results
    for (int64_t i = 2; i <= a; i++)
      phi[i] += phi[i - 1];
  }

  return phi;
}

} // namespace
<|endoftext|>


<commit_before>//=======================================================================
// Copyright Baptiste Wicht 2011.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================

#include <memory>
#include <thread>

#include "VisitorUtils.hpp"
#include "Options.hpp"
#include "PerfsTimer.hpp"
#include "iterators.hpp"
#include "likely.hpp"

#include "mtac/Pass.hpp"
#include "mtac/Optimizer.hpp"
#include "mtac/Program.hpp"
#include "mtac/Printer.hpp"

//The custom optimizations
#include "mtac/VariableCleaner.hpp"
#include "mtac/FunctionOptimizations.hpp"
#include "mtac/DeadCodeElimination.hpp"
#include "mtac/BasicBlockOptimizations.hpp"
#include "mtac/BranchOptimizations.hpp"
#include "mtac/ConcatReduction.hpp"
#include "mtac/inlining.hpp"

//The optimization visitors
#include "mtac/ArithmeticIdentities.hpp"
#include "mtac/ReduceInStrength.hpp"
#include "mtac/ConstantFolding.hpp"
#include "mtac/RemoveAssign.hpp"
#include "mtac/RemoveMultipleAssign.hpp"
#include "mtac/MathPropagation.hpp"

//The data-flow problems
#include "mtac/GlobalOptimizations.hpp"
#include "mtac/ConstantPropagationProblem.hpp"
#include "mtac/OffsetConstantPropagationProblem.hpp"
#include "mtac/CommonSubexpressionElimination.hpp"

using namespace eddic;

namespace {

static const unsigned int MAX_THREADS = 2;

template<typename Visitor>
bool apply_to_all(std::shared_ptr<mtac::Function> function){
    Visitor visitor;

    for(auto& block : function->getBasicBlocks()){
        visit_each(visitor, block->statements);
    }

    return visitor.optimized;
}

template<typename Visitor>
bool apply_to_basic_blocks_two_pass(std::shared_ptr<mtac::Function> function){
    bool optimized = false;

    for(auto& block : function->getBasicBlocks()){
        Visitor visitor;
        visitor.pass = mtac::Pass::DATA_MINING;

        visit_each(visitor, block->statements);

        visitor.pass = mtac::Pass::OPTIMIZE;

        visit_each(visitor, block->statements);

        optimized |= visitor.optimized;
    }

    return optimized;
}

template<typename Problem>
bool data_flow_optimization(std::shared_ptr<mtac::Function> function){
    bool optimized = false;

    Problem problem;

    auto results = mtac::data_flow(function, problem);

    //Once the data-flow problem is fixed, statements can be optimized
    for(auto& block : function->getBasicBlocks()){
        for(auto& statement : block->statements){
            optimized |= problem.optimize(statement, results);
        }
    }

    return optimized;
}

bool debug(const std::string& name, bool b, std::shared_ptr<mtac::Function> function){
    if(option_defined("dev")){
        if(b){
            std::cout << "optimization " << name << " returned true" << std::endl;

            //Print the function
            print(function);
        } else {
            std::cout << "optimization " << name << " returned false" << std::endl;
        }
    }

    return b;
}

template<typename Functor>
bool debug(const std::string& name, Functor functor, std::shared_ptr<mtac::Function> function){
    bool b;
    {
        PerfsTimer timer(name);

        b = functor(function);
    }

    return debug(name, b, function);
}

template<typename Functor, typename Arg2>
bool debug(const std::string& name, Functor functor, std::shared_ptr<mtac::Function> function, Arg2& arg2){
    bool b;
    {
        PerfsTimer timer(name);

        b = functor(function, arg2);
    }

    return debug(name, b, function);
}

void remove_nop(std::shared_ptr<mtac::Function> function){
    for(auto& block : function->getBasicBlocks()){
        auto it = iterate(block->statements);

        while(it.has_next()){
            if(unlikely(boost::get<std::shared_ptr<mtac::NoOp>>(&*it))){
                it.erase();
                continue;
            } else if(auto* ptr = boost::get<std::shared_ptr<mtac::Quadruple>>(&*it)){
                if((*ptr)->op == mtac::Operator::NOP){
                    it.erase();
                    continue;
                }
            }

            ++it;
        }
    }
}

}

void optimize_function(std::shared_ptr<mtac::Function> function, std::shared_ptr<StringPool> pool){
    if(option_defined("dev")){
        std::cout << "Start optimizations on " << function->getName() << std::endl;

        print(function);
    }

    bool optimized;
    do {
        optimized = false;

        optimized |= debug("Aritmetic Identities", &apply_to_all<mtac::ArithmeticIdentities>, function);
        optimized |= debug("Reduce in Strength", &apply_to_all<mtac::ReduceInStrength>, function);
        optimized |= debug("Constant folding", &apply_to_all<mtac::ConstantFolding>, function);

        optimized |= debug("Constant propagation", &data_flow_optimization<mtac::ConstantPropagationProblem>, function);
        optimized |= debug("Offset Constant Propagation", &data_flow_optimization<mtac::OffsetConstantPropagationProblem>, function);

        //If there was optimizations here, better to try again before perfoming common subexpression
        if(optimized){
            continue;
        }

        optimized |= debug("Common Subexpression Elimination", &data_flow_optimization<mtac::CommonSubexpressionElimination>, function);

        optimized |= debug("Math Propagation", &apply_to_basic_blocks_two_pass<mtac::MathPropagation>, function);

        optimized |= debug("Optimize Branches", &mtac::optimize_branches, function);
        optimized |= debug("Optimize Concat", &mtac::optimize_concat, function, pool);
        optimized |= debug("Remove dead basic block", &mtac::remove_dead_basic_blocks, function);
        optimized |= debug("Merge basic block", &mtac::merge_basic_blocks, function);

        optimized |= debug("Dead-Code Elimination", &mtac::dead_code_elimination, function);

        remove_nop(function);
    } while (optimized);

    //Remove variables that are not used after optimizations
    clean_variables(function);
}

void basic_optimize_function(std::shared_ptr<mtac::Function> function){
    if(option_defined("dev")){
        std::cout << "Start basic optimizations on " << function->getName() << std::endl;

        print(function);
    }

    debug("Constant folding", apply_to_all<mtac::ConstantFolding>(function), function);

    /*//Liveness debugging
    mtac::LiveVariableAnalysisProblem problem;
    auto results = mtac::data_flow(function, problem);

    for(auto& block : function->getBasicBlocks()){
        auto it = block->statements.begin();
        auto end = block->statements.end();

        while(it != end){
            auto statement = *it;

            mtac::Printer printer;
            printer.printStatement(statement);
            std::cout << "OUT{";
            for(auto& var : results->OUT_S[statement].values()){
                std::cout << var->name() << ", ";
            }
            std::cout << "}" << std::endl;
            std::cout << "IN{";
            for(auto& var : results->IN_S[statement].values()){
                std::cout << var->name() << ", ";
            }
            std::cout << "}" << std::endl;

            ++it;
        }
    }*/
}

void optimize_all_functions(std::shared_ptr<mtac::Program> program, std::shared_ptr<StringPool> string_pool){
    PerfsTimer timer("Whole optimizations");

    auto& functions = program->functions;

    //Find a better heuristic to configure the number of threads
    std::size_t threads = std::min(functions.size(), static_cast<std::size_t>(MAX_THREADS));

    if(option_defined("dev")){
        threads = 1;
    }

    std::vector<std::thread> pool;
    for(std::size_t tid = 0; tid < threads; ++tid){
        pool.push_back(std::thread([tid, threads, &string_pool, &functions](){
            std::size_t i = tid;

            while(i < functions.size()){
                optimize_function(functions[i], string_pool);

                i += threads;
            }
        }));
    }

    //Wait for all the threads to finish
    std::for_each(pool.begin(), pool.end(), [](std::thread& thread){thread.join();});
}

void mtac::Optimizer::optimize(std::shared_ptr<mtac::Program> program, std::shared_ptr<StringPool> string_pool) const {
    bool optimized = false;

    do{
        mtac::remove_unused_functions(program);

        optimize_all_functions(program, string_pool);

        optimized = mtac::remove_empty_functions(program);
        optimized = mtac::inline_functions(program);
    } while(optimized);
}

void mtac::Optimizer::basic_optimize(std::shared_ptr<mtac::Program> program, std::shared_ptr<StringPool> /*string_pool*/) const {
    PerfsTimer timer("Whole basic optimizations");

    auto& functions = program->functions;

    //Find a better heuristic to configure the number of threads
    std::size_t threads = std::min(functions.size(), static_cast<std::size_t>(MAX_THREADS));

    if(option_defined("dev")){
        threads = 1;
    }

    std::vector<std::thread> pool;
    for(std::size_t tid = 0; tid < threads; ++tid){
        pool.push_back(std::thread([tid, threads, &functions](){
            std::size_t i = tid;

            while(i < functions.size()){
                basic_optimize_function(functions[i]);

                i += threads;
            }
        }));
    }

    //Wait for all the threads to finish
    std::for_each(pool.begin(), pool.end(), [](std::thread& thread){thread.join();});
}
<commit_msg>Cleanup<commit_after>//=======================================================================
// Copyright Baptiste Wicht 2011.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================

#include <memory>
#include <thread>

#include "VisitorUtils.hpp"
#include "Options.hpp"
#include "PerfsTimer.hpp"
#include "iterators.hpp"
#include "likely.hpp"

#include "mtac/Pass.hpp"
#include "mtac/Optimizer.hpp"
#include "mtac/Program.hpp"
#include "mtac/Printer.hpp"

//The custom optimizations
#include "mtac/VariableCleaner.hpp"
#include "mtac/FunctionOptimizations.hpp"
#include "mtac/DeadCodeElimination.hpp"
#include "mtac/BasicBlockOptimizations.hpp"
#include "mtac/BranchOptimizations.hpp"
#include "mtac/ConcatReduction.hpp"
#include "mtac/inlining.hpp"

//The optimization visitors
#include "mtac/ArithmeticIdentities.hpp"
#include "mtac/ReduceInStrength.hpp"
#include "mtac/ConstantFolding.hpp"
#include "mtac/RemoveAssign.hpp"
#include "mtac/RemoveMultipleAssign.hpp"
#include "mtac/MathPropagation.hpp"

//The data-flow problems
#include "mtac/GlobalOptimizations.hpp"
#include "mtac/ConstantPropagationProblem.hpp"
#include "mtac/OffsetConstantPropagationProblem.hpp"
#include "mtac/CommonSubexpressionElimination.hpp"

using namespace eddic;

namespace {

static const unsigned int MAX_THREADS = 2;

template<typename Visitor>
bool apply_to_all(std::shared_ptr<mtac::Function> function){
    Visitor visitor;

    for(auto& block : function->getBasicBlocks()){
        visit_each(visitor, block->statements);
    }

    return visitor.optimized;
}

template<typename Visitor>
bool apply_to_basic_blocks_two_pass(std::shared_ptr<mtac::Function> function){
    bool optimized = false;

    for(auto& block : function->getBasicBlocks()){
        Visitor visitor;
        visitor.pass = mtac::Pass::DATA_MINING;

        visit_each(visitor, block->statements);

        visitor.pass = mtac::Pass::OPTIMIZE;

        visit_each(visitor, block->statements);

        optimized |= visitor.optimized;
    }

    return optimized;
}

template<typename Problem>
bool data_flow_optimization(std::shared_ptr<mtac::Function> function){
    bool optimized = false;

    Problem problem;

    auto results = mtac::data_flow(function, problem);

    //Once the data-flow problem is fixed, statements can be optimized
    for(auto& block : function->getBasicBlocks()){
        for(auto& statement : block->statements){
            optimized |= problem.optimize(statement, results);
        }
    }

    return optimized;
}

bool debug(const std::string& name, bool b, std::shared_ptr<mtac::Function> function){
    if(option_defined("dev")){
        if(b){
            std::cout << "optimization " << name << " returned true" << std::endl;

            //Print the function
            print(function);
        } else {
            std::cout << "optimization " << name << " returned false" << std::endl;
        }
    }

    return b;
}

template<typename Functor>
bool debug(const std::string& name, Functor functor, std::shared_ptr<mtac::Function> function){
    bool b;
    {
        PerfsTimer timer(name);

        b = functor(function);
    }

    return debug(name, b, function);
}

template<typename Functor, typename Arg2>
bool debug(const std::string& name, Functor functor, std::shared_ptr<mtac::Function> function, Arg2& arg2){
    bool b;
    {
        PerfsTimer timer(name);

        b = functor(function, arg2);
    }

    return debug(name, b, function);
}

void remove_nop(std::shared_ptr<mtac::Function> function){
    for(auto& block : function->getBasicBlocks()){
        auto it = iterate(block->statements);

        while(it.has_next()){
            if(unlikely(boost::get<std::shared_ptr<mtac::NoOp>>(&*it))){
                it.erase();
                continue;
            } else if(auto* ptr = boost::get<std::shared_ptr<mtac::Quadruple>>(&*it)){
                if((*ptr)->op == mtac::Operator::NOP){
                    it.erase();
                    continue;
                }
            }

            ++it;
        }
    }
}

}

void optimize_function(std::shared_ptr<mtac::Function> function, std::shared_ptr<StringPool> pool){
    if(option_defined("dev")){
        std::cout << "Start optimizations on " << function->getName() << std::endl;

        print(function);
    }

    bool optimized;
    do {
        optimized = false;

        optimized |= debug("Aritmetic Identities", &apply_to_all<mtac::ArithmeticIdentities>, function);
        optimized |= debug("Reduce in Strength", &apply_to_all<mtac::ReduceInStrength>, function);
        optimized |= debug("Constant folding", &apply_to_all<mtac::ConstantFolding>, function);

        optimized |= debug("Constant propagation", &data_flow_optimization<mtac::ConstantPropagationProblem>, function);
        optimized |= debug("Offset Constant Propagation", &data_flow_optimization<mtac::OffsetConstantPropagationProblem>, function);

        //If there was optimizations here, better to try again before perfoming common subexpression
        if(optimized){
            continue;
        }

        optimized |= debug("Common Subexpression Elimination", &data_flow_optimization<mtac::CommonSubexpressionElimination>, function);

        optimized |= debug("Math Propagation", &apply_to_basic_blocks_two_pass<mtac::MathPropagation>, function);

        optimized |= debug("Optimize Branches", &mtac::optimize_branches, function);
        optimized |= debug("Optimize Concat", &mtac::optimize_concat, function, pool);
        optimized |= debug("Remove dead basic block", &mtac::remove_dead_basic_blocks, function);
        optimized |= debug("Merge basic block", &mtac::merge_basic_blocks, function);

        remove_nop(function);

        optimized |= debug("Dead-Code Elimination", &mtac::dead_code_elimination, function);
    } while (optimized);

    //Remove variables that are not used after optimizations
    clean_variables(function);
}

void basic_optimize_function(std::shared_ptr<mtac::Function> function){
    if(option_defined("dev")){
        std::cout << "Start basic optimizations on " << function->getName() << std::endl;

        print(function);
    }

    debug("Constant folding", apply_to_all<mtac::ConstantFolding>(function), function);

    /*//Liveness debugging
    mtac::LiveVariableAnalysisProblem problem;
    auto results = mtac::data_flow(function, problem);

    for(auto& block : function->getBasicBlocks()){
        auto it = block->statements.begin();
        auto end = block->statements.end();

        while(it != end){
            auto statement = *it;

            mtac::Printer printer;
            printer.printStatement(statement);
            std::cout << "OUT{";
            for(auto& var : results->OUT_S[statement].values()){
                std::cout << var->name() << ", ";
            }
            std::cout << "}" << std::endl;
            std::cout << "IN{";
            for(auto& var : results->IN_S[statement].values()){
                std::cout << var->name() << ", ";
            }
            std::cout << "}" << std::endl;

            ++it;
        }
    }*/
}

void optimize_all_functions(std::shared_ptr<mtac::Program> program, std::shared_ptr<StringPool> string_pool){
    PerfsTimer timer("Whole optimizations");

    auto& functions = program->functions;

    //Find a better heuristic to configure the number of threads
    std::size_t threads = std::min(functions.size(), static_cast<std::size_t>(MAX_THREADS));

    if(option_defined("dev")){
        threads = 1;
    }

    std::vector<std::thread> pool;
    for(std::size_t tid = 0; tid < threads; ++tid){
        pool.push_back(std::thread([tid, threads, &string_pool, &functions](){
            std::size_t i = tid;

            while(i < functions.size()){
                optimize_function(functions[i], string_pool);

                i += threads;
            }
        }));
    }

    //Wait for all the threads to finish
    std::for_each(pool.begin(), pool.end(), [](std::thread& thread){thread.join();});
}

void mtac::Optimizer::optimize(std::shared_ptr<mtac::Program> program, std::shared_ptr<StringPool> string_pool) const {
    bool optimized = false;

    do{
        mtac::remove_unused_functions(program);

        optimize_all_functions(program, string_pool);

        optimized = mtac::remove_empty_functions(program);
        optimized = mtac::inline_functions(program);
    } while(optimized);
}

void mtac::Optimizer::basic_optimize(std::shared_ptr<mtac::Program> program, std::shared_ptr<StringPool> /*string_pool*/) const {
    PerfsTimer timer("Whole basic optimizations");

    auto& functions = program->functions;

    //Find a better heuristic to configure the number of threads
    std::size_t threads = std::min(functions.size(), static_cast<std::size_t>(MAX_THREADS));

    if(option_defined("dev")){
        threads = 1;
    }

    std::vector<std::thread> pool;
    for(std::size_t tid = 0; tid < threads; ++tid){
        pool.push_back(std::thread([tid, threads, &functions](){
            std::size_t i = tid;

            while(i < functions.size()){
                basic_optimize_function(functions[i]);

                i += threads;
            }
        }));
    }

    //Wait for all the threads to finish
    std::for_each(pool.begin(), pool.end(), [](std::thread& thread){thread.join();});
}
<|endoftext|>