<commit_before>/****************************************************************************
**
** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of the Qt Build Suite.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia.  For licensing terms and
** conditions see http://qt.digia.com/licensing.  For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights.  These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
****************************************************************************/

#include "scriptengine.h"

#include "item.h"
#include "filecontext.h"
#include "propertymapinternal.h"
#include "scriptpropertyobserver.h"
#include <tools/error.h>
#include <tools/qbsassert.h>

#include <QDebug>
#include <QFile>
#include <QScriptProgram>
#include <QScriptValueIterator>
#include <QSet>
#include <QTextStream>

namespace qbs {
namespace Internal {

const bool debugJSImports = false;

ScriptEngine::ScriptEngine(const Logger &logger, QObject *parent)
    : QScriptEngine(parent), m_logger(logger)
{
    QScriptValue objectProto = globalObject().property(QLatin1String("Object"));
    m_definePropertyFunction = objectProto.property(QLatin1String("defineProperty"));
    QBS_ASSERT(m_definePropertyFunction.isFunction(), /* ignore */);
    m_emptyFunction = evaluate("(function(){})");
    QBS_ASSERT(m_emptyFunction.isFunction(), /* ignore */);
    // Initially push a new context to turn off scope chain insanity mode.
    QScriptEngine::pushContext();
    extendJavaScriptBuiltins();
}

ScriptEngine::~ScriptEngine()
{
}

void ScriptEngine::import(const JsImports &jsImports, QScriptValue scope, QScriptValue targetObject)
{
    for (JsImports::const_iterator it = jsImports.begin(); it != jsImports.end(); ++it)
        import(*it, scope, targetObject);
}

void ScriptEngine::import(const JsImport &jsImport, QScriptValue scope, QScriptValue targetObject)
{
    QBS_ASSERT(!scope.isValid() || scope.isObject(), return);
    QBS_ASSERT(targetObject.isObject(), return);
    QBS_ASSERT(targetObject.engine() == this, return);

    if (debugJSImports)
        m_logger.qbsDebug() << "[ENGINE] import into " << jsImport.scopeName;

    foreach (const QString &fileName, jsImport.fileNames) {
        QScriptValue jsImportValue;
        jsImportValue = m_jsImportCache.value(fileName);
        if (jsImportValue.isValid()) {
            if (debugJSImports)
                m_logger.qbsDebug() << "[ENGINE] " << fileName << " (cache hit)";
            targetObject.setProperty(jsImport.scopeName, jsImportValue);
        } else {
            if (debugJSImports)
                m_logger.qbsDebug() << "[ENGINE] " << fileName << " (cache miss)";
            QFile file(fileName);
            if (Q_UNLIKELY(!file.open(QFile::ReadOnly)))
                throw ErrorInfo(tr("Cannot open '%1'.").arg(fileName));
            const QString sourceCode = QTextStream(&file).readAll();
            file.close();
            QScriptProgram program(sourceCode, fileName);
            importProgram(program, scope, jsImportValue);
            targetObject.setProperty(jsImport.scopeName, jsImportValue);
            m_jsImportCache.insert(fileName, jsImportValue);
        }
    }
}

void ScriptEngine::clearImportsCache()
{
    m_jsImportCache.clear();
}

void ScriptEngine::addToPropertyCache(const QString &moduleName, const QString &propertyName,
        const PropertyMapConstPtr &propertyMap, QScriptValue value)
{
    m_propertyCache.insert(qMakePair(moduleName + QLatin1Char('.') + propertyName, propertyMap),
                           value);
}

QScriptValue ScriptEngine::retrieveFromPropertyCache(const QString &moduleName,
        const QString &propertyName, const PropertyMapConstPtr &propertyMap)
{
    return m_propertyCache.value(qMakePair(moduleName + QLatin1Char('.') + propertyName,
                                           propertyMap));
}

void ScriptEngine::defineProperty(QScriptValue &object, const QString &name,
                                  const QScriptValue &descriptor)
{
    QScriptValue arguments = newArray();
    arguments.setProperty(0, object);
    arguments.setProperty(1, name);
    arguments.setProperty(2, descriptor);
    QScriptValue result = m_definePropertyFunction.call(QScriptValue(), arguments);
    QBS_ASSERT(!result.isError(), qDebug() << name << result.toString());
}

static QScriptValue js_observedGet(QScriptContext *context, QScriptEngine *, void *arg)
{
    ScriptPropertyObserver * const observer = static_cast<ScriptPropertyObserver *>(arg);
    const QScriptValue data = context->callee().property(QLatin1String("qbsdata"));
    const QScriptValue value = data.property(2);
    observer->onPropertyRead(data.property(0), data.property(1).toVariant().toString(), value);
    return value;
}

void ScriptEngine::setObservedProperty(QScriptValue &object, const QString &name,
                                       const QScriptValue &value, ScriptPropertyObserver *observer)
{
    if (!observer) {
        object.setProperty(name, value);
        return;
    }

    QScriptValue data = newArray();
    data.setProperty(0, object);
    data.setProperty(1, name);
    data.setProperty(2, value);
    QScriptValue getterFunc = newFunction(js_observedGet, observer);
    getterFunc.setProperty(QLatin1String("qbsdata"), data);
    QScriptValue descriptor = newObject();
    descriptor.setProperty(QLatin1String("get"), getterFunc);
    descriptor.setProperty(QLatin1String("set"), m_emptyFunction);
    defineProperty(object, name, descriptor);
}

QProcessEnvironment ScriptEngine::environment() const
{
    return m_environment;
}

void ScriptEngine::setEnvironment(const QProcessEnvironment &env)
{
    m_environment = env;
}

void ScriptEngine::importProgram(const QScriptProgram &program, const QScriptValue &scope,
                               QScriptValue &targetObject)
{
    QSet<QString> globalPropertyNames;
    {
        QScriptValueIterator it(globalObject());
        while (it.hasNext()) {
            it.next();
            globalPropertyNames += it.name();
        }
    }

    pushContext();
    if (scope.isObject())
        currentContext()->pushScope(scope);
    QScriptValue result = evaluate(program);
    QScriptValue activationObject = currentContext()->activationObject();
    if (scope.isObject())
        currentContext()->popScope();
    popContext();
    if (Q_UNLIKELY(result.isError()))
        throw ErrorInfo(tr("Error when importing '%1': %2").arg(program.fileName(), result.toString()));

    // If targetObject is already an object, it doesn't get overwritten but enhanced by the
    // contents of the .js file.
    // This is necessary for library imports that consist of multiple js files.
    if (!targetObject.isObject())
        targetObject = newObject();

    // Copy every property of the activation object to the target object.
    // We do not just save a reference to the activation object, because QScriptEngine contains
    // special magic for activation objects that leads to unanticipated results.
    {
        QScriptValueIterator it(activationObject);
        while (it.hasNext()) {
            it.next();
            if (debugJSImports)
                m_logger.qbsDebug() << "[ENGINE] Copying property " << it.name();
            targetObject.setProperty(it.name(), it.value());
        }
    }

    // Copy new global properties to the target object and remove them from
    // the global object. This is to support direct variable assignments
    // without the 'var' keyword in JavaScript files.
    QScriptValueIterator it(globalObject());
    while (it.hasNext()) {
        it.next();
        if (globalPropertyNames.contains(it.name()))
            continue;

        if (debugJSImports) {
            m_logger.qbsDebug() << "[ENGINE] inserting global property "
                                << it.name() << " " << it.value().toString();
        }

        targetObject.setProperty(it.name(), it.value());
        it.remove();
    }
}

void ScriptEngine::addEnvironmentVariable(const QString &name, const QString &value)
{
    m_usedEnvironment.insert(name, value);
}

QSet<QString> ScriptEngine::imports() const
{
    return QSet<QString>::fromList(m_jsImportCache.keys());
}

class JSTypeExtender
{
public:
    JSTypeExtender(ScriptEngine *engine, const QString &typeName)
        : m_engine(engine)
    {
        m_proto = engine->globalObject().property(typeName)
                .property(QLatin1String("prototype"));
        QBS_ASSERT(m_proto.isObject(), return);
        m_descriptor = engine->newObject();
    }

    void addFunction(const QString &name, const QString &code)
    {
        QScriptValue f = m_engine->evaluate(code);
        QBS_ASSERT(f.isFunction(), return);
        m_descriptor.setProperty(QLatin1String("value"), f);
        m_engine->defineProperty(m_proto, name, m_descriptor);
    }

private:
    ScriptEngine *const m_engine;
    QScriptValue m_proto;
    QScriptValue m_descriptor;
};

void ScriptEngine::extendJavaScriptBuiltins()
{
    JSTypeExtender arrayExtender(this, QLatin1String("Array"));
    arrayExtender.addFunction(QLatin1String("contains"),
        QLatin1String("(function(e){return this.indexOf(e) !== -1;})"));
    JSTypeExtender stringExtender(this, QLatin1String("String"));
    stringExtender.addFunction(QLatin1String("contains"),
        QLatin1String("(function(e){return this.indexOf(e) !== -1;})"));
}

} // namespace Internal
} // namespace qbs
<commit_msg>Add String.startsWith and String.endsWith to JavaScript.<commit_after>/****************************************************************************
**
** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
** Contact: http://www.qt-project.org/legal
**
** This file is part of the Qt Build Suite.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia.  For licensing terms and
** conditions see http://qt.digia.com/licensing.  For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights.  These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
****************************************************************************/

#include "scriptengine.h"

#include "item.h"
#include "filecontext.h"
#include "propertymapinternal.h"
#include "scriptpropertyobserver.h"
#include <tools/error.h>
#include <tools/qbsassert.h>

#include <QDebug>
#include <QFile>
#include <QScriptProgram>
#include <QScriptValueIterator>
#include <QSet>
#include <QTextStream>

namespace qbs {
namespace Internal {

const bool debugJSImports = false;

ScriptEngine::ScriptEngine(const Logger &logger, QObject *parent)
    : QScriptEngine(parent), m_logger(logger)
{
    QScriptValue objectProto = globalObject().property(QLatin1String("Object"));
    m_definePropertyFunction = objectProto.property(QLatin1String("defineProperty"));
    QBS_ASSERT(m_definePropertyFunction.isFunction(), /* ignore */);
    m_emptyFunction = evaluate("(function(){})");
    QBS_ASSERT(m_emptyFunction.isFunction(), /* ignore */);
    // Initially push a new context to turn off scope chain insanity mode.
    QScriptEngine::pushContext();
    extendJavaScriptBuiltins();
}

ScriptEngine::~ScriptEngine()
{
}

void ScriptEngine::import(const JsImports &jsImports, QScriptValue scope, QScriptValue targetObject)
{
    for (JsImports::const_iterator it = jsImports.begin(); it != jsImports.end(); ++it)
        import(*it, scope, targetObject);
}

void ScriptEngine::import(const JsImport &jsImport, QScriptValue scope, QScriptValue targetObject)
{
    QBS_ASSERT(!scope.isValid() || scope.isObject(), return);
    QBS_ASSERT(targetObject.isObject(), return);
    QBS_ASSERT(targetObject.engine() == this, return);

    if (debugJSImports)
        m_logger.qbsDebug() << "[ENGINE] import into " << jsImport.scopeName;

    foreach (const QString &fileName, jsImport.fileNames) {
        QScriptValue jsImportValue;
        jsImportValue = m_jsImportCache.value(fileName);
        if (jsImportValue.isValid()) {
            if (debugJSImports)
                m_logger.qbsDebug() << "[ENGINE] " << fileName << " (cache hit)";
            targetObject.setProperty(jsImport.scopeName, jsImportValue);
        } else {
            if (debugJSImports)
                m_logger.qbsDebug() << "[ENGINE] " << fileName << " (cache miss)";
            QFile file(fileName);
            if (Q_UNLIKELY(!file.open(QFile::ReadOnly)))
                throw ErrorInfo(tr("Cannot open '%1'.").arg(fileName));
            const QString sourceCode = QTextStream(&file).readAll();
            file.close();
            QScriptProgram program(sourceCode, fileName);
            importProgram(program, scope, jsImportValue);
            targetObject.setProperty(jsImport.scopeName, jsImportValue);
            m_jsImportCache.insert(fileName, jsImportValue);
        }
    }
}

void ScriptEngine::clearImportsCache()
{
    m_jsImportCache.clear();
}

void ScriptEngine::addToPropertyCache(const QString &moduleName, const QString &propertyName,
        const PropertyMapConstPtr &propertyMap, QScriptValue value)
{
    m_propertyCache.insert(qMakePair(moduleName + QLatin1Char('.') + propertyName, propertyMap),
                           value);
}

QScriptValue ScriptEngine::retrieveFromPropertyCache(const QString &moduleName,
        const QString &propertyName, const PropertyMapConstPtr &propertyMap)
{
    return m_propertyCache.value(qMakePair(moduleName + QLatin1Char('.') + propertyName,
                                           propertyMap));
}

void ScriptEngine::defineProperty(QScriptValue &object, const QString &name,
                                  const QScriptValue &descriptor)
{
    QScriptValue arguments = newArray();
    arguments.setProperty(0, object);
    arguments.setProperty(1, name);
    arguments.setProperty(2, descriptor);
    QScriptValue result = m_definePropertyFunction.call(QScriptValue(), arguments);
    QBS_ASSERT(!result.isError(), qDebug() << name << result.toString());
}

static QScriptValue js_observedGet(QScriptContext *context, QScriptEngine *, void *arg)
{
    ScriptPropertyObserver * const observer = static_cast<ScriptPropertyObserver *>(arg);
    const QScriptValue data = context->callee().property(QLatin1String("qbsdata"));
    const QScriptValue value = data.property(2);
    observer->onPropertyRead(data.property(0), data.property(1).toVariant().toString(), value);
    return value;
}

void ScriptEngine::setObservedProperty(QScriptValue &object, const QString &name,
                                       const QScriptValue &value, ScriptPropertyObserver *observer)
{
    if (!observer) {
        object.setProperty(name, value);
        return;
    }

    QScriptValue data = newArray();
    data.setProperty(0, object);
    data.setProperty(1, name);
    data.setProperty(2, value);
    QScriptValue getterFunc = newFunction(js_observedGet, observer);
    getterFunc.setProperty(QLatin1String("qbsdata"), data);
    QScriptValue descriptor = newObject();
    descriptor.setProperty(QLatin1String("get"), getterFunc);
    descriptor.setProperty(QLatin1String("set"), m_emptyFunction);
    defineProperty(object, name, descriptor);
}

QProcessEnvironment ScriptEngine::environment() const
{
    return m_environment;
}

void ScriptEngine::setEnvironment(const QProcessEnvironment &env)
{
    m_environment = env;
}

void ScriptEngine::importProgram(const QScriptProgram &program, const QScriptValue &scope,
                               QScriptValue &targetObject)
{
    QSet<QString> globalPropertyNames;
    {
        QScriptValueIterator it(globalObject());
        while (it.hasNext()) {
            it.next();
            globalPropertyNames += it.name();
        }
    }

    pushContext();
    if (scope.isObject())
        currentContext()->pushScope(scope);
    QScriptValue result = evaluate(program);
    QScriptValue activationObject = currentContext()->activationObject();
    if (scope.isObject())
        currentContext()->popScope();
    popContext();
    if (Q_UNLIKELY(result.isError()))
        throw ErrorInfo(tr("Error when importing '%1': %2").arg(program.fileName(), result.toString()));

    // If targetObject is already an object, it doesn't get overwritten but enhanced by the
    // contents of the .js file.
    // This is necessary for library imports that consist of multiple js files.
    if (!targetObject.isObject())
        targetObject = newObject();

    // Copy every property of the activation object to the target object.
    // We do not just save a reference to the activation object, because QScriptEngine contains
    // special magic for activation objects that leads to unanticipated results.
    {
        QScriptValueIterator it(activationObject);
        while (it.hasNext()) {
            it.next();
            if (debugJSImports)
                m_logger.qbsDebug() << "[ENGINE] Copying property " << it.name();
            targetObject.setProperty(it.name(), it.value());
        }
    }

    // Copy new global properties to the target object and remove them from
    // the global object. This is to support direct variable assignments
    // without the 'var' keyword in JavaScript files.
    QScriptValueIterator it(globalObject());
    while (it.hasNext()) {
        it.next();
        if (globalPropertyNames.contains(it.name()))
            continue;

        if (debugJSImports) {
            m_logger.qbsDebug() << "[ENGINE] inserting global property "
                                << it.name() << " " << it.value().toString();
        }

        targetObject.setProperty(it.name(), it.value());
        it.remove();
    }
}

void ScriptEngine::addEnvironmentVariable(const QString &name, const QString &value)
{
    m_usedEnvironment.insert(name, value);
}

QSet<QString> ScriptEngine::imports() const
{
    return QSet<QString>::fromList(m_jsImportCache.keys());
}

class JSTypeExtender
{
public:
    JSTypeExtender(ScriptEngine *engine, const QString &typeName)
        : m_engine(engine)
    {
        m_proto = engine->globalObject().property(typeName)
                .property(QLatin1String("prototype"));
        QBS_ASSERT(m_proto.isObject(), return);
        m_descriptor = engine->newObject();
    }

    void addFunction(const QString &name, const QString &code)
    {
        QScriptValue f = m_engine->evaluate(code);
        QBS_ASSERT(f.isFunction(), return);
        m_descriptor.setProperty(QLatin1String("value"), f);
        m_engine->defineProperty(m_proto, name, m_descriptor);
    }

private:
    ScriptEngine *const m_engine;
    QScriptValue m_proto;
    QScriptValue m_descriptor;
};

void ScriptEngine::extendJavaScriptBuiltins()
{
    JSTypeExtender arrayExtender(this, QLatin1String("Array"));
    arrayExtender.addFunction(QLatin1String("contains"),
        QLatin1String("(function(e){return this.indexOf(e) !== -1;})"));

    JSTypeExtender stringExtender(this, QLatin1String("String"));
    stringExtender.addFunction(QLatin1String("contains"),
        QLatin1String("(function(e){return this.indexOf(e) !== -1;})"));
    stringExtender.addFunction(QLatin1String("startsWith"),
        QLatin1String("(function(e){return this.slice(0, e.length) === e;})"));
    stringExtender.addFunction(QLatin1String("endsWith"),
        QLatin1String("(function(e){return this.slice(-e.length) === e;})"));
}

} // namespace Internal
} // namespace qbs
<|endoftext|>


<commit_before>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009-2014 Klaus Post

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "io/FileReader.h"
#include "common/Common.h"      // for make_unique
#include "io/Buffer.h"          // for Buffer
#include "io/FileIOException.h" // for FileIOException (ptr only), ThrowFIE
#include <algorithm>            // for move
#include <cstdio>               // for fclose, fseek, fopen, fread, ftell
#include <fcntl.h>              // for SEEK_END, SEEK_SET
#include <memory>               // for unique_ptr

#if !defined(__unix__) && !defined(__APPLE__)
#include <io.h>
#include <tchar.h>
#include <windows.h>
#endif // !defined(__unix__) && !defined(__APPLE__)

namespace rawspeed {

FileReader::FileReader(const char *_filename) : mFilename(_filename) {}

std::unique_ptr<Buffer> FileReader::readFile() {
#if defined(__unix__) || defined(__APPLE__)
  int bytes_read = 0;
  FILE *file;
  long size;

  file = fopen(mFilename, "rb");
  if (file == nullptr)
    ThrowFIE("Could not open file.");
  fseek(file, 0, SEEK_END);
  size = ftell(file);
  if (size <= 0) {
    fclose(file);
    ThrowFIE("File is 0 bytes.");
  }
  fseek(file, 0, SEEK_SET);

  auto dest = Buffer::Create(size);
  bytes_read = fread(dest.get(), 1, size, file);
  fclose(file);
  if (size != bytes_read)
    ThrowFIE("Could not read file.");

  auto fileData = make_unique<Buffer>(move(dest), size);

#else // __unix__

  HANDLE file_h;  // File handle
  file_h = CreateFile(mFilename, GENERIC_READ, FILE_SHARE_READ, nullptr,
                      OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
  if (file_h == INVALID_HANDLE_VALUE) {
    ThrowFIE("Could not open file.");
  }

  LARGE_INTEGER f_size;
  GetFileSizeEx(file_h , &f_size);

  if (!f_size.LowPart)
    ThrowFIE("File is 0 bytes.");

  auto dest = Buffer::Create(f_size.LowPart);

  DWORD bytes_read;
  if (!ReadFile(file_h, dest.get(), f_size.LowPart, &bytes_read, nullptr)) {
    CloseHandle(file_h);
    ThrowFIE("Could not read file.");
  }

  CloseHandle(file_h);

  auto* fileData = make_unique<Buffer>(move(dest), f_size.LowPart);

#endif // __unix__

  return fileData;
}

} // namespace rawspeed
<commit_msg>FileReader: fix win build, add max file size checks.<commit_after>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009-2014 Klaus Post

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "io/FileReader.h"
#include "common/Common.h"      // for make_unique
#include "io/Buffer.h"          // for Buffer
#include "io/FileIOException.h" // for FileIOException (ptr only), ThrowFIE
#include <algorithm>            // for move
#include <cstdio>               // for fclose, fseek, fopen, fread, ftell
#include <fcntl.h>              // for SEEK_END, SEEK_SET
#include <limits>               // for numeric_limits
#include <memory>               // for unique_ptr

#if !defined(__unix__) && !defined(__APPLE__)
#include <io.h>
#include <tchar.h>
#include <windows.h>
#endif // !defined(__unix__) && !defined(__APPLE__)

namespace rawspeed {

FileReader::FileReader(const char *_filename) : mFilename(_filename) {}

std::unique_ptr<Buffer> FileReader::readFile() {
#if defined(__unix__) || defined(__APPLE__)
  int bytes_read = 0;
  FILE *file;
  long size;

  file = fopen(mFilename, "rb");
  if (file == nullptr)
    ThrowFIE("Could not open file.");
  fseek(file, 0, SEEK_END);
  size = ftell(file);
  if (size > std::numeric_limits<Buffer::size_type>::max()) {
    fclose(file);
    ThrowFIE("File is too big.");
  }
  if (size <= 0) {
    fclose(file);
    ThrowFIE("File is 0 bytes.");
  }
  fseek(file, 0, SEEK_SET);

  auto dest = Buffer::Create(size);
  bytes_read = fread(dest.get(), 1, size, file);
  fclose(file);
  if (size != bytes_read)
    ThrowFIE("Could not read file.");

  auto fileData = make_unique<Buffer>(move(dest), size);

#else // __unix__

  HANDLE file_h;  // File handle
  file_h = CreateFile(mFilename, GENERIC_READ, FILE_SHARE_READ, nullptr,
                      OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, nullptr);
  if (file_h == INVALID_HANDLE_VALUE) {
    ThrowFIE("Could not open file.");
  }

  LARGE_INTEGER f_size;
  GetFileSizeEx(file_h , &f_size);

  static_assert(
      std::numeric_limits<Buffer::size_type>::max() ==
          std::numeric_limits<decltype(f_size.LowPart)>::max(),
      "once Buffer migrates to 64-bit index, this needs to be updated.");

  if (HighPart > 0)
    ThrowFIE("File is too big.");
  if (f_size.LowPart <= 0)
    ThrowFIE("File is 0 bytes.");

  auto dest = Buffer::Create(f_size.LowPart);

  DWORD bytes_read;
  if (!ReadFile(file_h, dest.get(), f_size.LowPart, &bytes_read, nullptr)) {
    CloseHandle(file_h);
    ThrowFIE("Could not read file.");
  }

  CloseHandle(file_h);

  if (f_size.LowPart != bytes_read)
    ThrowFIE("Could not read file.");

  auto fileData = make_unique<Buffer>(move(dest), f_size.LowPart);

#endif // __unix__

  return fileData;
}

} // namespace rawspeed
<|endoftext|>


<commit_before>/* Copyright 2009 SPARTA, Inc., dba Cobham Analytic Solutions
 *
 * This file is part of WATCHER.
 *
 *     WATCHER is free software: you can redistribute it and/or modify
 *     it under the terms of the GNU Affero General Public License as published by
 *     the Free Software Foundation, either version 3 of the License, or
 *     (at your option) any later version.
 *
 *     WATCHER is distributed in the hope that it will be useful,
 *     but WITHOUT ANY WARRANTY; without even the implied warranty of
 *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *     GNU Affero General Public License for more details.
 *
 *     You should have received a copy of the GNU Affero General Public License
 *     along with Watcher.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iomanip>
#include <sstream>

#include <boost/lexical_cast.hpp>

#include "dataMarshaller.h"
#include "logger.h"

INIT_LOGGER(watcher::DataMarshaller, "DataMarshaller");

using namespace std;
using namespace watcher;
using namespace watcher::event;

#if 0
/**
 * GTL ERASE START
 * preinitailize the archiving mechanism. This is a hack and should be
 * removed once we figure out how to do this without priming the
 * archive pump
 */
#include "libwatcher/message.h"
#include "libwatcher/labelMessage.h"
#include "libwatcher/edgeMessage.h"
#include "libwatcher/colorMessage.h"
#include "libwatcher/gpsMessage.h"
#include "libwatcher/messageStatus.h"
#include "libwatcher/dataRequestMessage.h"
#include "libwatcher/testMessage.h"
#include "libwatcher/seekWatcherMessage.h"
#include "libwatcher/startWatcherMessage.h"
#include "libwatcher/stopWatcherMessage.h"
#include "libwatcher/speedWatcherMessage.h"
#include "libwatcher/nodeStatusMessage.h"
#include "libwatcher/connectivityMessage.h"
static MessagePtr unused_A_Ptr=MessagePtr(new LabelMessage);
static MessagePtr unused_B_Ptr=MessagePtr(new EdgeMessage);
static MessagePtr unused_C_Ptr=MessagePtr(new ColorMessage);
static MessagePtr unused_D_Ptr=MessagePtr(new GPSMessage);
static MessagePtr unused_E_Ptr=MessagePtr(new MessageStatus);
static MessagePtr unused_F_Ptr=MessagePtr(new DataRequestMessage);
static MessagePtr unused_G_Ptr=MessagePtr(new TestMessage);
static MessagePtr unused_H_Ptr=MessagePtr(new SeekMessage);
static MessagePtr unused_I_Ptr=MessagePtr(new StartMessage);
static MessagePtr unused_J_Ptr=MessagePtr(new StopMessage);
static MessagePtr unused_K_Ptr=MessagePtr(new SpeedMessage);
static MessagePtr unused_L_Ptr=MessagePtr(new NodeStatusMessage);
static MessagePtr unused_M_Ptr=MessagePtr(new ConnectivityMessage);
#endif

/**
 * GTL ERASE END
 *
 **/

#define MARSHALSHORT(a,b) \
	do\
	{\
		*a++=((unsigned char)((b) >> 8)); \
		*a++=((unsigned char)(b)); \
	} while(0)

#define UNMARSHALSHORT(a,b)  {b=(*(a+0)<<8) | *(a+1); a+=2;}

#define MARSHALINT32(a,b) \
    do\
    {\
	*a++=((unsigned char)((b) >> 24));\
	*a++=((unsigned char)((b) >> 16));\
	*a++=((unsigned char)((b) >> 8)); \
	*a++=((unsigned char)(b)); \
    } while(0)

#define UNMARSHALINT32(a,b)  {b = (*(a+0)<<24) | (*(a+1)<<16) | (*(a+2)<<8) | *(a+3); a+=4;}

// static
bool DataMarshaller::unmarshalHeader(const char *buffer, const size_t &bufferSize, size_t &payloadSize, unsigned short &messageNum)
{
    TRACE_ENTER();

    // logger is crashing if given anything other than a string! GTL - Test and see if this is still true.
    LOG_DEBUG("Unmarshalling a header of " << boost::lexical_cast<std::string>(bufferSize) << " bytes.");

    if (bufferSize < DataMarshaller::header_length)
    {
        LOG_ERROR("Not enought data in payload to unmarshal the packet or packet is corrupt");
        TRACE_EXIT_RET("false");
        return false;
    }

    if (bufferSize > 0xffffffff)
    {
	LOG_ERROR("buffer size " << bufferSize << " is too large to fit into an uint32.");
        TRACE_EXIT_RET("false");
        return false;
    }

    const unsigned char *bufPtr=(const unsigned char*)buffer;
    UNMARSHALINT32(bufPtr, payloadSize);
    UNMARSHALSHORT(bufPtr, messageNum);

    LOG_DEBUG("Header data: payload size: " << payloadSize << " messageNum: " << messageNum);

    return true;
}

//static
bool DataMarshaller::unmarshalPayload(MessagePtr &message, const char *buffer, const size_t &bufferSize)
{
    TRACE_ENTER();

    // Extract the Message from the payload
    istringstream s(string(buffer, bufferSize));
    message=Message::unpack(s);

    //LOG_DEBUG("Unmarshalled payload data: " << s.str());

    TRACE_EXIT_RET((message?"true":"false"));
    return static_cast<bool>(message);          // cast may be redundant
}

//static
bool DataMarshaller::unmarshalPayload(std::vector<MessagePtr> &messages, unsigned short &numOfMessages, const char *buffer, const size_t &bufferSize)
{
    TRACE_ENTER();

    istringstream is(string(buffer, bufferSize));
    LOG_DEBUG("Unmarshalling payload data(size=" << bufferSize << "): " << is.str());
    unsigned short i=0;
    for(i=0; i<numOfMessages; i++)
    {
        MessagePtr m=Message::unpack(is);
        if(!m)
        {
            LOG_WARN("Error: Unable to unmarshal message number " << i+1 << " in the packet payload.");
            numOfMessages=i;
            TRACE_EXIT_RET("false");
            return false;
        }
        messages.push_back(m);      // marshalled with push_front, so should get same order on unmarshalling.
    }

    //LOG_DEBUG("Unmarshalled payload data: " << is.str());
    LOG_DEBUG("Successfully unmarshalled " << i << " message" << (i>0?"s":""));

    TRACE_EXIT_RET("true");
    return true;
}

//static
bool DataMarshaller::marshalPayload(const MessagePtr &message, NetworkMarshalBuffers &outBuffers)
{
    TRACE_ENTER();

    std::vector<MessagePtr> messVec;
    messVec.push_back(message);
    bool retVal=marshalPayload(messVec, outBuffers);

    TRACE_EXIT_RET((retVal?"true":"false"));
    return retVal;
}

//static
bool DataMarshaller::marshalPayload(const vector<MessagePtr> &messages, NetworkMarshalBuffers &outBuffers)
{
    TRACE_ENTER();

    // Add serialized Messages first, then prepend the header once we know
    // how large the payload will be.
    size_t payloadSize=0;
    unsigned short messageNum=messages.size();

    // Putting each Message in a separate buffer may speed up sent/recv as
    // each buffer can be scatter-gather sent/recv'd.

    ostringstream os;
    std::string s(os.str());
    for(vector<MessagePtr>::const_iterator m=messages.begin(); m!=messages.end(); ++m)
    {
        (*m)->pack(os);
        s = os.str();
        payloadSize += s.size();
        outBuffers.push_back(NetworkMarshalBuffer(s));
        LOG_DEBUG("Marshalled payload: " << s);
        os.str("");
    }

    LOG_DEBUG("Serialized " << payloadSize << " bytes of message data from " << messageNum << " message" << (messageNum>1?"s":""));

    // Format the header.
    // GTL - may be nice to put the header itself into a class that supports archive/serialization...
    unsigned char header[header_length];
    unsigned char *bufPtr=header;
    MARSHALINT32(bufPtr, payloadSize);
    MARSHALSHORT(bufPtr, messageNum);

    // Put the header on the front.
    outBuffers.push_front(NetworkMarshalBuffer(string((const char*)header, sizeof(header))));

    TRACE_EXIT_RET("true");
    return true;
}

<commit_msg>libwatcher: use uint32_t instead of size_t when you only want 32 bits. Comes in handy on 64 bit systems.<commit_after>/* Copyright 2009 SPARTA, Inc., dba Cobham Analytic Solutions
 *
 * This file is part of WATCHER.
 *
 *     WATCHER is free software: you can redistribute it and/or modify
 *     it under the terms of the GNU Affero General Public License as published by
 *     the Free Software Foundation, either version 3 of the License, or
 *     (at your option) any later version.
 *
 *     WATCHER is distributed in the hope that it will be useful,
 *     but WITHOUT ANY WARRANTY; without even the implied warranty of
 *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *     GNU Affero General Public License for more details.
 *
 *     You should have received a copy of the GNU Affero General Public License
 *     along with Watcher.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <iomanip>
#include <sstream>

#include <boost/lexical_cast.hpp>

#include "dataMarshaller.h"
#include "logger.h"

INIT_LOGGER(watcher::DataMarshaller, "DataMarshaller");

using namespace std;
using namespace watcher;
using namespace watcher::event;

#if 0
/**
 * GTL ERASE START
 * preinitailize the archiving mechanism. This is a hack and should be
 * removed once we figure out how to do this without priming the
 * archive pump
 */
#include "libwatcher/message.h"
#include "libwatcher/labelMessage.h"
#include "libwatcher/edgeMessage.h"
#include "libwatcher/colorMessage.h"
#include "libwatcher/gpsMessage.h"
#include "libwatcher/messageStatus.h"
#include "libwatcher/dataRequestMessage.h"
#include "libwatcher/testMessage.h"
#include "libwatcher/seekWatcherMessage.h"
#include "libwatcher/startWatcherMessage.h"
#include "libwatcher/stopWatcherMessage.h"
#include "libwatcher/speedWatcherMessage.h"
#include "libwatcher/nodeStatusMessage.h"
#include "libwatcher/connectivityMessage.h"
static MessagePtr unused_A_Ptr=MessagePtr(new LabelMessage);
static MessagePtr unused_B_Ptr=MessagePtr(new EdgeMessage);
static MessagePtr unused_C_Ptr=MessagePtr(new ColorMessage);
static MessagePtr unused_D_Ptr=MessagePtr(new GPSMessage);
static MessagePtr unused_E_Ptr=MessagePtr(new MessageStatus);
static MessagePtr unused_F_Ptr=MessagePtr(new DataRequestMessage);
static MessagePtr unused_G_Ptr=MessagePtr(new TestMessage);
static MessagePtr unused_H_Ptr=MessagePtr(new SeekMessage);
static MessagePtr unused_I_Ptr=MessagePtr(new StartMessage);
static MessagePtr unused_J_Ptr=MessagePtr(new StopMessage);
static MessagePtr unused_K_Ptr=MessagePtr(new SpeedMessage);
static MessagePtr unused_L_Ptr=MessagePtr(new NodeStatusMessage);
static MessagePtr unused_M_Ptr=MessagePtr(new ConnectivityMessage);
#endif

/**
 * GTL ERASE END
 *
 **/

#define MARSHALSHORT(a,b) \
	do\
	{\
		*a++=((unsigned char)((b) >> 8)); \
		*a++=((unsigned char)(b)); \
	} while(0)

#define UNMARSHALSHORT(a,b)  {b=(*(a+0)<<8) | *(a+1); a+=2;}

#define MARSHALINT32(a,b) \
    do\
    {\
	*a++=((unsigned char)((b) >> 24));\
	*a++=((unsigned char)((b) >> 16));\
	*a++=((unsigned char)((b) >> 8)); \
	*a++=((unsigned char)(b)); \
    } while(0)

#define UNMARSHALINT32(a,b)  {b = (*(a+0)<<24) | (*(a+1)<<16) | (*(a+2)<<8) | *(a+3); a+=4;}

// static
bool DataMarshaller::unmarshalHeader(const char *buffer, const size_t &bufferSize, size_t &payloadSize, unsigned short &messageNum)
{
    TRACE_ENTER();

    // logger is crashing if given anything other than a string! GTL - Test and see if this is still true.
    LOG_DEBUG("Unmarshalling a header of " << boost::lexical_cast<std::string>(bufferSize) << " bytes.");

    if (bufferSize < DataMarshaller::header_length)
    {
        LOG_ERROR("Not enought data in payload to unmarshal the packet or packet is corrupt");
        TRACE_EXIT_RET("false");
        return false;
    }

    if (bufferSize > 0xffffffff)
    {
	LOG_ERROR("buffer size " << bufferSize << " is too large to fit into an uint32.");
        TRACE_EXIT_RET("false");
        return false;
    }

    const unsigned char *bufPtr=(const unsigned char*)buffer;
    UNMARSHALINT32(bufPtr, payloadSize);
    UNMARSHALSHORT(bufPtr, messageNum);

    LOG_DEBUG("Header data: payload size: " << payloadSize << " messageNum: " << messageNum);

    return true;
}

//static
bool DataMarshaller::unmarshalPayload(MessagePtr &message, const char *buffer, const size_t &bufferSize)
{
    TRACE_ENTER();

    // Extract the Message from the payload
    istringstream s(string(buffer, bufferSize));
    message=Message::unpack(s);

    //LOG_DEBUG("Unmarshalled payload data: " << s.str());

    TRACE_EXIT_RET((message?"true":"false"));
    return static_cast<bool>(message);          // cast may be redundant
}

//static
bool DataMarshaller::unmarshalPayload(std::vector<MessagePtr> &messages, unsigned short &numOfMessages, const char *buffer, const size_t &bufferSize)
{
    TRACE_ENTER();

    istringstream is(string(buffer, bufferSize));
    LOG_DEBUG("Unmarshalling payload data(size=" << bufferSize << "): " << is.str());
    unsigned short i=0;
    for(i=0; i<numOfMessages; i++)
    {
        MessagePtr m=Message::unpack(is);
        if(!m)
        {
            LOG_WARN("Error: Unable to unmarshal message number " << i+1 << " in the packet payload.");
            numOfMessages=i;
            TRACE_EXIT_RET("false");
            return false;
        }
        messages.push_back(m);      // marshalled with push_front, so should get same order on unmarshalling.
    }

    //LOG_DEBUG("Unmarshalled payload data: " << is.str());
    LOG_DEBUG("Successfully unmarshalled " << i << " message" << (i>0?"s":""));

    TRACE_EXIT_RET("true");
    return true;
}

//static
bool DataMarshaller::marshalPayload(const MessagePtr &message, NetworkMarshalBuffers &outBuffers)
{
    TRACE_ENTER();

    std::vector<MessagePtr> messVec;
    messVec.push_back(message);
    bool retVal=marshalPayload(messVec, outBuffers);

    TRACE_EXIT_RET((retVal?"true":"false"));
    return retVal;
}

//static
bool DataMarshaller::marshalPayload(const vector<MessagePtr> &messages, NetworkMarshalBuffers &outBuffers)
{
    TRACE_ENTER();

    // Add serialized Messages first, then prepend the header once we know
    // how large the payload will be.
    uint32_t payloadSize=0;
    unsigned short messageNum=messages.size();

    // Putting each Message in a separate buffer may speed up sent/recv as
    // each buffer can be scatter-gather sent/recv'd.

    ostringstream os;
    std::string s(os.str());
    for(vector<MessagePtr>::const_iterator m=messages.begin(); m!=messages.end(); ++m)
    {
        (*m)->pack(os);
        s = os.str();
        payloadSize += static_cast<uint32_t>(s.size());
        outBuffers.push_back(NetworkMarshalBuffer(s));
        LOG_DEBUG("Marshalled payload: " << s);
        os.str("");
    }

    LOG_DEBUG("Serialized " << payloadSize << " bytes of message data from " << messageNum << " message" << (messageNum>1?"s":""));

    // Format the header.
    // GTL - may be nice to put the header itself into a class that supports archive/serialization...
    unsigned char header[header_length];
    unsigned char *bufPtr=header;
    MARSHALINT32(bufPtr, payloadSize);
    MARSHALSHORT(bufPtr, messageNum);

    // Put the header on the front.
    outBuffers.push_front(NetworkMarshalBuffer(string((const char*)header, sizeof(header))));

    TRACE_EXIT_RET("true");
    return true;
}

<|endoftext|>


<commit_before>
/* Include the SDL main definition header */
#include "SDL_main.h"
#include <stdlib.h>
#include <unistd.h>
#ifdef main
#undef main
#endif
#ifdef QWS
#include <qpe/qpeapplication.h>
#include <qapplication.h>
#include <qpe/qpeapplication.h>
#include <stdlib.h>

// Workaround for OPIE to remove taskbar icon. Also fixes
// some issues in Qtopia where there are left-over qcop files in /tmp/.
// I'm guessing this will also clean up the taskbar in the Sharp version
// of Qtopia.
static inline void cleanupQCop() {
  QString appname(qApp->argv()[0]);
  int slash = appname.findRev("/");
  if(slash != -1) {  appname = appname.mid(slash+1); }
  QString cmd = QPEApplication::qpeDir() + "bin/qcop QPE/System 'closing(QString)' '"+appname+"'";
  system(cmd.latin1());
  cmd = "/tmp/qcop-msg-"+appname;
  unlink(cmd.latin1());
}

static QPEApplication *app;
#endif

extern int SDL_main(int argc, char *argv[]);

int main(int argc, char *argv[])
{
#ifdef QWS
  // This initializes the Qtopia application. It needs to be done here
  // because it parses command line options.
  QWidget dummy;
  app = new QPEApplication(argc, argv);
  app->showMainWidget(&dummy);
  atexit(cleanupQCop);
#endif
  // Exit here because if return is used, the application
  // doesn't seem to quit correctly.
  exit(SDL_main(argc, argv));
}
<commit_msg>Fixed video intitialization problem on Qtopia (thanks David!)<commit_after>
/* Include the SDL main definition header */
#include "SDL_main.h"
#include <stdlib.h>
#include <unistd.h>
#ifdef main
#undef main
#endif
#ifdef QWS
#include <qpe/qpeapplication.h>
#include <qapplication.h>
#include <qpe/qpeapplication.h>
#include <stdlib.h>

// Workaround for OPIE to remove taskbar icon. Also fixes
// some issues in Qtopia where there are left-over qcop files in /tmp/.
// I'm guessing this will also clean up the taskbar in the Sharp version
// of Qtopia.
static inline void cleanupQCop() {
  QString appname(qApp->argv()[0]);
  int slash = appname.findRev("/");
  if(slash != -1) {  appname = appname.mid(slash+1); }
  QString cmd = QPEApplication::qpeDir() + "bin/qcop QPE/System 'closing(QString)' '"+appname+"'";
  system(cmd.latin1());
  cmd = "/tmp/qcop-msg-"+appname;
  unlink(cmd.latin1());
}

static QPEApplication *app;
#endif

extern int SDL_main(int argc, char *argv[]);

int main(int argc, char *argv[])
{
#ifdef QWS
  // This initializes the Qtopia application. It needs to be done here
  // because it parses command line options.
  app = new QPEApplication(argc, argv);
  QWidget dummy;
  app->showMainWidget(&dummy);
  atexit(cleanupQCop);
#endif
  // Exit here because if return is used, the application
  // doesn't seem to quit correctly.
  exit(SDL_main(argc, argv));
}
<|endoftext|>


<commit_before>/*
 * Copyright 2012 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Author: jefftk@google.com (Jeff Kaufman)

#include "ngx_rewrite_driver_factory.h"

#include <cstdio>

#include "log_message_handler.h"
#include "ngx_message_handler.h"
#include "ngx_rewrite_options.h"
#include "ngx_server_context.h"
#include "ngx_thread_system.h"
#include "ngx_url_async_fetcher.h"
#include "pthread_shared_mem.h"

#include "net/instaweb/apache/serf_url_async_fetcher.h"
#include "net/instaweb/http/public/content_type.h"
#include "net/instaweb/http/public/fake_url_async_fetcher.h"
#include "net/instaweb/http/public/wget_url_fetcher.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/static_asset_manager.h"
#include "net/instaweb/system/public/system_caches.h"
#include "net/instaweb/util/public/google_message_handler.h"
#include "net/instaweb/util/public/null_shared_mem.h"
#include "net/instaweb/util/public/property_cache.h"
#include "net/instaweb/util/public/scheduler_thread.h"
#include "net/instaweb/util/public/posix_timer.h"
#include "net/instaweb/util/public/shared_circular_buffer.h"
#include "net/instaweb/util/public/shared_mem_statistics.h"
#include "net/instaweb/util/public/slow_worker.h"
#include "net/instaweb/util/public/stdio_file_system.h"
#include "net/instaweb/util/public/string.h"
#include "net/instaweb/util/public/string_util.h"
#include "net/instaweb/util/public/thread_system.h"

namespace net_instaweb {

class FileSystem;
class Hasher;
class MessageHandler;
class Statistics;
class Timer;
class UrlAsyncFetcher;
class UrlFetcher;
class Writer;

const char NgxRewriteDriverFactory::kStaticAssetPrefix[] =
    "/ngx_pagespeed_static/";

namespace {

const char kShutdownCount[] = "child_shutdown_count";

}  // namespace

NgxRewriteDriverFactory::NgxRewriteDriverFactory(
    NgxThreadSystem* ngx_thread_system)
    : SystemRewriteDriverFactory(ngx_thread_system),
      ngx_thread_system_(ngx_thread_system),
      // TODO(oschaaf): mod_pagespeed ifdefs this:
      shared_mem_runtime_(new ngx::PthreadSharedMem()),
      main_conf_(NULL),
      threads_started_(false),
      use_per_vhost_statistics_(false),
      is_root_process_(true),
      ngx_message_handler_(new NgxMessageHandler(thread_system()->NewMutex())),
      ngx_html_parse_message_handler_(
          new NgxMessageHandler(thread_system()->NewMutex())),
      install_crash_handler_(false),
      message_buffer_size_(0),
      shared_circular_buffer_(NULL),
      statistics_frozen_(false),
      ngx_url_async_fetcher_(NULL),
      log_(NULL),
      resolver_timeout_(NGX_CONF_UNSET_MSEC),
      use_native_fetcher_(false) {
  InitializeDefaultOptions();
  default_options()->set_beacon_url("/ngx_pagespeed_beacon");
  set_message_handler(ngx_message_handler_);
  set_html_parse_message_handler(ngx_html_parse_message_handler_);

  // see https://code.google.com/p/modpagespeed/issues/detail?id=672
  int thread_limit = 1;
  caches_.reset(
      new SystemCaches(this, shared_mem_runtime_.get(), thread_limit));
}

NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
  ShutDown();

  CHECK(uninitialized_server_contexts_.empty() || is_root_process_);
  STLDeleteElements(&uninitialized_server_contexts_);
  shared_mem_statistics_.reset(NULL);
}

Hasher* NgxRewriteDriverFactory::NewHasher() {
  return new MD5Hasher;
}

UrlFetcher* NgxRewriteDriverFactory::DefaultUrlFetcher() {
  CHECK(false) << "Nothing should still be using DefaultUrlFetcher()";
  return NULL;
}

UrlAsyncFetcher* NgxRewriteDriverFactory::DefaultAsyncUrlFetcher() {
  const char* fetcher_proxy = "";
  if (main_conf_ != NULL) {
    fetcher_proxy = main_conf_->fetcher_proxy().c_str();
  }

  if (use_native_fetcher_) {
    net_instaweb::NgxUrlAsyncFetcher* fetcher =
        new net_instaweb::NgxUrlAsyncFetcher(
            fetcher_proxy,
            log_,
            resolver_timeout_,
            25000,
            resolver_,
            thread_system(),
            message_handler());
    ngx_url_async_fetcher_ = fetcher;
    return fetcher;
  } else {
    net_instaweb::UrlAsyncFetcher* fetcher =
        new net_instaweb::SerfUrlAsyncFetcher(
            fetcher_proxy,
            NULL,
            thread_system(),
            statistics(),
            timer(),
            2500,
            message_handler());
    return fetcher;
  }
}

MessageHandler* NgxRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
  return ngx_html_parse_message_handler_;
}

MessageHandler* NgxRewriteDriverFactory::DefaultMessageHandler() {
  return ngx_message_handler_;
}

FileSystem* NgxRewriteDriverFactory::DefaultFileSystem() {
  return new StdioFileSystem();
}

Timer* NgxRewriteDriverFactory::DefaultTimer() {
  return new PosixTimer;
}

NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
  CHECK(false);
  return NULL;
}

void NgxRewriteDriverFactory::SetupCaches(ServerContext* server_context) {
  caches_->SetupCaches(server_context);

  server_context->set_enable_property_cache(true);
  PropertyCache* pcache = server_context->page_property_cache();
  if (pcache->GetCohort(RewriteDriver::kBeaconCohort) == NULL) {
    pcache->AddCohort(RewriteDriver::kBeaconCohort);
  }
  if (pcache->GetCohort(RewriteDriver::kDomCohort) == NULL) {
    pcache->AddCohort(RewriteDriver::kDomCohort);
  }
}

RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
  NgxRewriteOptions* options = new NgxRewriteOptions();
  options->SetRewriteLevel(RewriteOptions::kCoreFilters);
  return options;
}

void NgxRewriteDriverFactory::InitStaticAssetManager(
    StaticAssetManager* static_asset_manager) {
  static_asset_manager->set_library_url_prefix(kStaticAssetPrefix);
}

void NgxRewriteDriverFactory::PrintMemCacheStats(GoogleString* out) {
  // TODO(morlovich): Port the client code to proper API, so it gets
  // shm stats, too.
  caches_->PrintCacheStats(SystemCaches::kIncludeMemcached, out);
}

bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetcher() {
  if (ngx_url_async_fetcher_ == NULL) {
    return true;
  }
  log_ = ngx_cycle->log;
  return ngx_url_async_fetcher_->Init();
}

bool NgxRewriteDriverFactory::CheckResolver() {
  if (use_native_fetcher_ && resolver_ == NULL) {
    return false;
  }
  return true;
}

void NgxRewriteDriverFactory::StopCacheActivity() {
  RewriteDriverFactory::StopCacheActivity();
  caches_->StopCacheActivity();
}

NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext() {
  NgxServerContext* server_context = new NgxServerContext(this);
  uninitialized_server_contexts_.insert(server_context);
  return server_context;
}

ServerContext* NgxRewriteDriverFactory::NewServerContext() {
  LOG(DFATAL) << "MakeNgxServerContext should be used instead";
  return NULL;
}

void NgxRewriteDriverFactory::ShutDown() {
  StopCacheActivity();
  if (!is_root_process_) {
    Variable* child_shutdown_count = statistics()->GetVariable(kShutdownCount);
    child_shutdown_count->Add(1);
    message_handler()->Message(kInfo, "Shutting down ngx_pagespeed child");
  } else {
    message_handler()->Message(kInfo, "Shutting down ngx_pagespeed root");
  }

  RewriteDriverFactory::ShutDown();
  caches_->ShutDown(message_handler());

  ngx_message_handler_->set_buffer(NULL);
  ngx_html_parse_message_handler_->set_buffer(NULL);

  // TODO(oschaaf): enable this once the shared memory cleanup code
  // supports our ordering of events during a configuration reload
  if (is_root_process_) {
    // Cleanup statistics.
    // TODO(morlovich): This looks dangerous with async.
    if (shared_mem_statistics_.get() != NULL) {
      shared_mem_statistics_->GlobalCleanup(message_handler());
    }
    if (shared_circular_buffer_ != NULL) {
      shared_circular_buffer_->GlobalCleanup(message_handler());
    }
  }
}

void NgxRewriteDriverFactory::StartThreads() {
  if (threads_started_) {
    return;
  }
  ngx_thread_system_->PermitThreadStarting();
  // TODO(jefftk): use a native nginx timer instead of running our own thread.
  // See issue #111.
  SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
  bool ok = thread->Start();
  CHECK(ok) << "Unable to start scheduler thread";
  defer_cleanup(thread->MakeDeleter());
  threads_started_ = true;
}

void NgxRewriteDriverFactory::ParentOrChildInit(ngx_log_t* log) {
  if (install_crash_handler_) {
    NgxMessageHandler::InstallCrashHandler(log);
  }
  ngx_message_handler_->set_log(log);
  ngx_html_parse_message_handler_->set_log(log);
  SharedCircularBufferInit(is_root_process_);
}

// TODO(jmarantz): make this per-vhost.
void NgxRewriteDriverFactory::SharedCircularBufferInit(bool is_root) {
  // Set buffer size to 0 means turning it off
  if (shared_mem_runtime() != NULL && (message_buffer_size_ != 0)) {
    // TODO(jmarantz): it appears that filename_prefix() is not actually
    // established at the time of this construction, calling into question
    // whether we are naming our shared-memory segments correctly.
    shared_circular_buffer_.reset(new SharedCircularBuffer(
        shared_mem_runtime(),
        message_buffer_size_,
        filename_prefix().as_string(),
        "foo.com" /*hostname_identifier()*/));
    if (shared_circular_buffer_->InitSegment(is_root, message_handler())) {
      ngx_message_handler_->set_buffer(shared_circular_buffer_.get());
      ngx_html_parse_message_handler_->set_buffer(
          shared_circular_buffer_.get());
    }
  }
}

void NgxRewriteDriverFactory::RootInit(ngx_log_t* log) {
  net_instaweb::log_message_handler::Install(log);

  ParentOrChildInit(log);

  // Let SystemCaches know about the various paths we have in configuration
  // first, as well as the memcached instances.
  for (NgxServerContextSet::iterator p = uninitialized_server_contexts_.begin(),
           e = uninitialized_server_contexts_.end(); p != e; ++p) {
    NgxServerContext* server_context = *p;
    caches_->RegisterConfig(server_context->config());
  }

  caches_->RootInit();
}

void NgxRewriteDriverFactory::ChildInit(ngx_log_t* log) {
  is_root_process_ = false;

  ParentOrChildInit(log);
  if (shared_mem_statistics_.get() != NULL) {
    shared_mem_statistics_->Init(false, message_handler());
  }

  caches_->ChildInit();
  for (NgxServerContextSet::iterator p = uninitialized_server_contexts_.begin(),
           e = uninitialized_server_contexts_.end(); p != e; ++p) {
    NgxServerContext* server_context = *p;
    server_context->ChildInit();
  }

  uninitialized_server_contexts_.clear();
}

// Initializes global statistics object if needed, using factory to
// help with the settings if needed.
// Note: does not call set_statistics() on the factory.
Statistics* NgxRewriteDriverFactory::MakeGlobalSharedMemStatistics(
    bool logging, int64 logging_interval_ms,
    const GoogleString& logging_file_base) {
  if (shared_mem_statistics_.get() == NULL) {
    shared_mem_statistics_.reset(AllocateAndInitSharedMemStatistics(
        "global", logging, logging_interval_ms, logging_file_base));
  }
  DCHECK(!statistics_frozen_);
  statistics_frozen_ = true;
  SetStatistics(shared_mem_statistics_.get());
  return shared_mem_statistics_.get();
}

SharedMemStatistics* NgxRewriteDriverFactory::
AllocateAndInitSharedMemStatistics(
    const StringPiece& name, const bool logging,
    const int64 logging_interval_ms,
    const GoogleString& logging_file_base) {
  // Note that we create the statistics object in the parent process, and
  // it stays around in the kids but gets reinitialized for them
  // inside ChildInit(), called from pagespeed_child_init.
  SharedMemStatistics* stats = new SharedMemStatistics(
      logging_interval_ms, StrCat(logging_file_base, name), logging,
      StrCat(filename_prefix(), name), shared_mem_runtime(), message_handler(),
      file_system(), timer());
  InitStats(stats);
  stats->Init(true, message_handler());
  return stats;
}

void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
  // Init standard PSOL stats.
  RewriteDriverFactory::InitStats(statistics);

  // Init Ngx-specific stats.
  NgxServerContext::InitStats(statistics);
  SystemCaches::InitStats(statistics);
  SerfUrlAsyncFetcher::InitStats(statistics);
  PropertyCache::InitCohortStats(RewriteDriver::kBeaconCohort, statistics);
  PropertyCache::InitCohortStats(RewriteDriver::kDomCohort, statistics);

  statistics->AddVariable(kShutdownCount);
}

}  // namespace net_instaweb
<commit_msg>small cleanup: stale comment, confusing log message<commit_after>/*
 * Copyright 2012 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Author: jefftk@google.com (Jeff Kaufman)

#include "ngx_rewrite_driver_factory.h"

#include <cstdio>

#include "log_message_handler.h"
#include "ngx_message_handler.h"
#include "ngx_rewrite_options.h"
#include "ngx_server_context.h"
#include "ngx_thread_system.h"
#include "ngx_url_async_fetcher.h"
#include "pthread_shared_mem.h"

#include "net/instaweb/apache/serf_url_async_fetcher.h"
#include "net/instaweb/http/public/content_type.h"
#include "net/instaweb/http/public/fake_url_async_fetcher.h"
#include "net/instaweb/http/public/wget_url_fetcher.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/static_asset_manager.h"
#include "net/instaweb/system/public/system_caches.h"
#include "net/instaweb/util/public/google_message_handler.h"
#include "net/instaweb/util/public/null_shared_mem.h"
#include "net/instaweb/util/public/property_cache.h"
#include "net/instaweb/util/public/scheduler_thread.h"
#include "net/instaweb/util/public/posix_timer.h"
#include "net/instaweb/util/public/shared_circular_buffer.h"
#include "net/instaweb/util/public/shared_mem_statistics.h"
#include "net/instaweb/util/public/slow_worker.h"
#include "net/instaweb/util/public/stdio_file_system.h"
#include "net/instaweb/util/public/string.h"
#include "net/instaweb/util/public/string_util.h"
#include "net/instaweb/util/public/thread_system.h"

namespace net_instaweb {

class FileSystem;
class Hasher;
class MessageHandler;
class Statistics;
class Timer;
class UrlAsyncFetcher;
class UrlFetcher;
class Writer;

const char NgxRewriteDriverFactory::kStaticAssetPrefix[] =
    "/ngx_pagespeed_static/";

namespace {

const char kShutdownCount[] = "child_shutdown_count";

}  // namespace

NgxRewriteDriverFactory::NgxRewriteDriverFactory(
    NgxThreadSystem* ngx_thread_system)
    : SystemRewriteDriverFactory(ngx_thread_system),
      ngx_thread_system_(ngx_thread_system),
      // TODO(oschaaf): mod_pagespeed ifdefs this:
      shared_mem_runtime_(new ngx::PthreadSharedMem()),
      main_conf_(NULL),
      threads_started_(false),
      use_per_vhost_statistics_(false),
      is_root_process_(true),
      ngx_message_handler_(new NgxMessageHandler(thread_system()->NewMutex())),
      ngx_html_parse_message_handler_(
          new NgxMessageHandler(thread_system()->NewMutex())),
      install_crash_handler_(false),
      message_buffer_size_(0),
      shared_circular_buffer_(NULL),
      statistics_frozen_(false),
      ngx_url_async_fetcher_(NULL),
      log_(NULL),
      resolver_timeout_(NGX_CONF_UNSET_MSEC),
      use_native_fetcher_(false) {
  InitializeDefaultOptions();
  default_options()->set_beacon_url("/ngx_pagespeed_beacon");
  set_message_handler(ngx_message_handler_);
  set_html_parse_message_handler(ngx_html_parse_message_handler_);

  // see https://code.google.com/p/modpagespeed/issues/detail?id=672
  int thread_limit = 1;
  caches_.reset(
      new SystemCaches(this, shared_mem_runtime_.get(), thread_limit));
}

NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
  ShutDown();

  CHECK(uninitialized_server_contexts_.empty() || is_root_process_);
  STLDeleteElements(&uninitialized_server_contexts_);
  shared_mem_statistics_.reset(NULL);
}

Hasher* NgxRewriteDriverFactory::NewHasher() {
  return new MD5Hasher;
}

UrlFetcher* NgxRewriteDriverFactory::DefaultUrlFetcher() {
  CHECK(false) << "Nothing should still be using DefaultUrlFetcher()";
  return NULL;
}

UrlAsyncFetcher* NgxRewriteDriverFactory::DefaultAsyncUrlFetcher() {
  const char* fetcher_proxy = "";
  if (main_conf_ != NULL) {
    fetcher_proxy = main_conf_->fetcher_proxy().c_str();
  }

  if (use_native_fetcher_) {
    net_instaweb::NgxUrlAsyncFetcher* fetcher =
        new net_instaweb::NgxUrlAsyncFetcher(
            fetcher_proxy,
            log_,
            resolver_timeout_,
            25000,
            resolver_,
            thread_system(),
            message_handler());
    ngx_url_async_fetcher_ = fetcher;
    return fetcher;
  } else {
    net_instaweb::UrlAsyncFetcher* fetcher =
        new net_instaweb::SerfUrlAsyncFetcher(
            fetcher_proxy,
            NULL,
            thread_system(),
            statistics(),
            timer(),
            2500,
            message_handler());
    return fetcher;
  }
}

MessageHandler* NgxRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
  return ngx_html_parse_message_handler_;
}

MessageHandler* NgxRewriteDriverFactory::DefaultMessageHandler() {
  return ngx_message_handler_;
}

FileSystem* NgxRewriteDriverFactory::DefaultFileSystem() {
  return new StdioFileSystem();
}

Timer* NgxRewriteDriverFactory::DefaultTimer() {
  return new PosixTimer;
}

NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
  CHECK(false);
  return NULL;
}

void NgxRewriteDriverFactory::SetupCaches(ServerContext* server_context) {
  caches_->SetupCaches(server_context);

  server_context->set_enable_property_cache(true);
  PropertyCache* pcache = server_context->page_property_cache();
  if (pcache->GetCohort(RewriteDriver::kBeaconCohort) == NULL) {
    pcache->AddCohort(RewriteDriver::kBeaconCohort);
  }
  if (pcache->GetCohort(RewriteDriver::kDomCohort) == NULL) {
    pcache->AddCohort(RewriteDriver::kDomCohort);
  }
}

RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
  NgxRewriteOptions* options = new NgxRewriteOptions();
  options->SetRewriteLevel(RewriteOptions::kCoreFilters);
  return options;
}

void NgxRewriteDriverFactory::InitStaticAssetManager(
    StaticAssetManager* static_asset_manager) {
  static_asset_manager->set_library_url_prefix(kStaticAssetPrefix);
}

void NgxRewriteDriverFactory::PrintMemCacheStats(GoogleString* out) {
  // TODO(morlovich): Port the client code to proper API, so it gets
  // shm stats, too.
  caches_->PrintCacheStats(SystemCaches::kIncludeMemcached, out);
}

bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetcher() {
  if (ngx_url_async_fetcher_ == NULL) {
    return true;
  }
  log_ = ngx_cycle->log;
  return ngx_url_async_fetcher_->Init();
}

bool NgxRewriteDriverFactory::CheckResolver() {
  if (use_native_fetcher_ && resolver_ == NULL) {
    return false;
  }
  return true;
}

void NgxRewriteDriverFactory::StopCacheActivity() {
  RewriteDriverFactory::StopCacheActivity();
  caches_->StopCacheActivity();
}

NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext() {
  NgxServerContext* server_context = new NgxServerContext(this);
  uninitialized_server_contexts_.insert(server_context);
  return server_context;
}

ServerContext* NgxRewriteDriverFactory::NewServerContext() {
  LOG(DFATAL) << "MakeNgxServerContext should be used instead";
  return NULL;
}

void NgxRewriteDriverFactory::ShutDown() {
  StopCacheActivity();
  if (!is_root_process_) {
    Variable* child_shutdown_count = statistics()->GetVariable(kShutdownCount);
    child_shutdown_count->Add(1);
  }

  RewriteDriverFactory::ShutDown();
  caches_->ShutDown(message_handler());

  ngx_message_handler_->set_buffer(NULL);
  ngx_html_parse_message_handler_->set_buffer(NULL);

  if (is_root_process_) {
    // Cleanup statistics.
    // TODO(morlovich): This looks dangerous with async.
    if (shared_mem_statistics_.get() != NULL) {
      shared_mem_statistics_->GlobalCleanup(message_handler());
    }
    if (shared_circular_buffer_ != NULL) {
      shared_circular_buffer_->GlobalCleanup(message_handler());
    }
  }
}

void NgxRewriteDriverFactory::StartThreads() {
  if (threads_started_) {
    return;
  }
  ngx_thread_system_->PermitThreadStarting();
  // TODO(jefftk): use a native nginx timer instead of running our own thread.
  // See issue #111.
  SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
  bool ok = thread->Start();
  CHECK(ok) << "Unable to start scheduler thread";
  defer_cleanup(thread->MakeDeleter());
  threads_started_ = true;
}

void NgxRewriteDriverFactory::ParentOrChildInit(ngx_log_t* log) {
  if (install_crash_handler_) {
    NgxMessageHandler::InstallCrashHandler(log);
  }
  ngx_message_handler_->set_log(log);
  ngx_html_parse_message_handler_->set_log(log);
  SharedCircularBufferInit(is_root_process_);
}

// TODO(jmarantz): make this per-vhost.
void NgxRewriteDriverFactory::SharedCircularBufferInit(bool is_root) {
  // Set buffer size to 0 means turning it off
  if (shared_mem_runtime() != NULL && (message_buffer_size_ != 0)) {
    // TODO(jmarantz): it appears that filename_prefix() is not actually
    // established at the time of this construction, calling into question
    // whether we are naming our shared-memory segments correctly.
    shared_circular_buffer_.reset(new SharedCircularBuffer(
        shared_mem_runtime(),
        message_buffer_size_,
        filename_prefix().as_string(),
        "foo.com" /*hostname_identifier()*/));
    if (shared_circular_buffer_->InitSegment(is_root, message_handler())) {
      ngx_message_handler_->set_buffer(shared_circular_buffer_.get());
      ngx_html_parse_message_handler_->set_buffer(
          shared_circular_buffer_.get());
    }
  }
}

void NgxRewriteDriverFactory::RootInit(ngx_log_t* log) {
  net_instaweb::log_message_handler::Install(log);

  ParentOrChildInit(log);

  // Let SystemCaches know about the various paths we have in configuration
  // first, as well as the memcached instances.
  for (NgxServerContextSet::iterator p = uninitialized_server_contexts_.begin(),
           e = uninitialized_server_contexts_.end(); p != e; ++p) {
    NgxServerContext* server_context = *p;
    caches_->RegisterConfig(server_context->config());
  }

  caches_->RootInit();
}

void NgxRewriteDriverFactory::ChildInit(ngx_log_t* log) {
  is_root_process_ = false;

  ParentOrChildInit(log);
  if (shared_mem_statistics_.get() != NULL) {
    shared_mem_statistics_->Init(false, message_handler());
  }

  caches_->ChildInit();
  for (NgxServerContextSet::iterator p = uninitialized_server_contexts_.begin(),
           e = uninitialized_server_contexts_.end(); p != e; ++p) {
    NgxServerContext* server_context = *p;
    server_context->ChildInit();
  }

  uninitialized_server_contexts_.clear();
}

// Initializes global statistics object if needed, using factory to
// help with the settings if needed.
// Note: does not call set_statistics() on the factory.
Statistics* NgxRewriteDriverFactory::MakeGlobalSharedMemStatistics(
    bool logging, int64 logging_interval_ms,
    const GoogleString& logging_file_base) {
  if (shared_mem_statistics_.get() == NULL) {
    shared_mem_statistics_.reset(AllocateAndInitSharedMemStatistics(
        "global", logging, logging_interval_ms, logging_file_base));
  }
  DCHECK(!statistics_frozen_);
  statistics_frozen_ = true;
  SetStatistics(shared_mem_statistics_.get());
  return shared_mem_statistics_.get();
}

SharedMemStatistics* NgxRewriteDriverFactory::
AllocateAndInitSharedMemStatistics(
    const StringPiece& name, const bool logging,
    const int64 logging_interval_ms,
    const GoogleString& logging_file_base) {
  // Note that we create the statistics object in the parent process, and
  // it stays around in the kids but gets reinitialized for them
  // inside ChildInit(), called from pagespeed_child_init.
  SharedMemStatistics* stats = new SharedMemStatistics(
      logging_interval_ms, StrCat(logging_file_base, name), logging,
      StrCat(filename_prefix(), name), shared_mem_runtime(), message_handler(),
      file_system(), timer());
  InitStats(stats);
  stats->Init(true, message_handler());
  return stats;
}

void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
  // Init standard PSOL stats.
  RewriteDriverFactory::InitStats(statistics);

  // Init Ngx-specific stats.
  NgxServerContext::InitStats(statistics);
  SystemCaches::InitStats(statistics);
  SerfUrlAsyncFetcher::InitStats(statistics);
  PropertyCache::InitCohortStats(RewriteDriver::kBeaconCohort, statistics);
  PropertyCache::InitCohortStats(RewriteDriver::kDomCohort, statistics);

  statistics->AddVariable(kShutdownCount);
}

}  // namespace net_instaweb
<|endoftext|>


<commit_before>/**********************************************************************
 *
 * GEOS - Geometry Engine Open Source
 * http://geos.osgeo.org
 *
 * Copyright 2009-2010 Sandro Santilli <strk@kbt.io>
 * Copyright (C) 2019 Even Rouault <even.rouault@spatialys.com>
 *
 * This is free software; you can redistribute and/or modify it under
 * the terms of the GNU Lesser General Public Licence as published
 * by the Free Software Foundation.
 * See the COPYING file for more information.
 **********************************************************************
 *
 * Ported from rtgeom_geos.c from
 *   rttopo - topology library
 *   http://git.osgeo.org/gitea/rttopo/librttopo
 * with relicensing from GPL to LGPL with Copyright holder permission.
 *
 **********************************************************************/

#include <geos/operation/valid/MakeValid.h>
#include <geos/operation/valid/IsValidOp.h>
#include <geos/operation/polygonize/BuildArea.h>
#include <geos/geom/Geometry.h>
#include <geos/geom/GeometryCollection.h>
#include <geos/geom/GeometryFactory.h>
#include <geos/geom/LineString.h>
#include <geos/geom/Point.h>
#include <geos/geom/Polygon.h>
#include <geos/geom/MultiLineString.h>
#include <geos/geom/MultiPolygon.h>
#include <geos/util/UniqueCoordinateArrayFilter.h>
#include <geos/util/UnsupportedOperationException.h>

// std
#include <cassert>
#include <algorithm>
#include <utility>
#include <vector>

#ifdef _MSC_VER
#pragma warning(disable:4355)
#endif

using namespace geos::geom;

namespace geos {
namespace operation { // geos.operation
namespace valid { // geos.operation.valid

/*
 * Fully node given linework
 */
static std::unique_ptr<geom::Geometry>
nodeLineWithFirstCoordinate(const geom::Geometry* geom)
{
  /*
   * Union with first geometry point, obtaining full noding
   * and dissolving of duplicated repeated points
   *
   * TODO: substitute this with UnaryUnion?
   */

  if( geom->isEmpty() )
      return nullptr;

  const auto geomType = geom->getGeometryTypeId();
  assert( geomType == GEOS_LINESTRING || geomType == GEOS_MULTILINESTRING );

  std::unique_ptr<geom::Geometry> point;
  if( geomType == GEOS_LINESTRING ) {
      auto line = dynamic_cast<const geom::LineString*>(geom);
      assert(line);
      point = line->getPointN(0);
  } else {
      auto mls = dynamic_cast<const geom::MultiLineString*>(geom);
      assert(mls);
      auto line = mls->getGeometryN(0);
      assert(line);
      point = line->getPointN(0);
  }

  return geom->Union(point.get());
}

static std::unique_ptr<geom::Geometry> MakeValidLine(const geom::LineString* line)
{
    return nodeLineWithFirstCoordinate(line);
}

static std::unique_ptr<geom::Geometry> MakeValidMultiLine(const geom::MultiLineString* mls)
{
    std::vector<std::unique_ptr<geom::Geometry>> points;
    std::vector<std::unique_ptr<geom::Geometry>> lines;

    for(const auto& subgeom: *mls) {
        auto line = dynamic_cast<const geom::LineString*>(subgeom.get());
        assert(line);
        auto validSubGeom = MakeValidLine(line);
        if( !validSubGeom || validSubGeom->isEmpty() ) {
            continue;
        }
        auto validLineType = validSubGeom->getGeometryTypeId();
        if( validLineType == GEOS_POINT ) {
            points.emplace_back(std::move(validSubGeom));
        }
        else if( validLineType == GEOS_LINESTRING ) {
            lines.emplace_back(std::move(validSubGeom));
        } else if( validLineType == GEOS_MULTILINESTRING ) {
            auto mlsValid = dynamic_cast<const geom::MultiLineString*>(validSubGeom.get());
            for(const auto& subgeomMlsValid: *mlsValid) {
                lines.emplace_back(subgeomMlsValid->clone());
            }
        } else {
            throw util::UnsupportedOperationException();
        }
    }

    std::unique_ptr<geom::Geometry> pointsRet;
    if( !points.empty() ) {
        if( points.size() > 1 ) {
            pointsRet = mls->getFactory()->createMultiPoint(std::move(points));
        } else {
            pointsRet = std::move(points[0]);
        }
    }

    std::unique_ptr<geom::Geometry> linesRet;
    if( !lines.empty() ) {
        if( lines.size() > 1 ) {
            linesRet = mls->getFactory()->createMultiLineString(std::move(lines));
        } else {
            linesRet = std::move(lines[0]);
        }
    }

    if( pointsRet && linesRet ) {
        std::vector<std::unique_ptr<Geometry>> geoms(2);
        geoms[0] = std::move(pointsRet);
        geoms[1] = std::move(linesRet);
        return mls->getFactory()->createGeometryCollection(std::move(geoms));
    } else if( pointsRet ) {
        return pointsRet;
    } else if( linesRet ) {
        return linesRet;
    }

    return nullptr;
}

static std::unique_ptr<geom::Geometry> extractUniquePoints(const geom::Geometry* geom)
{

    // Code taken from GEOSGeom_extractUniquePoints_r()

    /* 1: extract points */
    std::vector<const geom::Coordinate*> coords;
    geos::util::UniqueCoordinateArrayFilter filter(coords);
    geom->apply_ro(&filter);

    /* 2: for each point, create a geometry and put into a vector */
    std::vector<std::unique_ptr<Geometry>> points;
    points.reserve(coords.size());
    const GeometryFactory* factory = geom->getFactory();
    for(const Coordinate* c : coords) {
        points.emplace_back(factory->createPoint(*c));
    }

    /* 3: create a multipoint */
    return factory->createMultiPoint(std::move(points));
}

static std::unique_ptr<geom::Geometry> MakeValidPoly(const geom::Geometry* geom)
{
    assert( geom->getGeometryTypeId() == GEOS_POLYGON ||
            geom->getGeometryTypeId() == GEOS_MULTIPOLYGON );

    std::unique_ptr<geom::Geometry> bound(geom->getBoundary());
    if( !bound )
        return nullptr;

    /* Use noded boundaries as initial "cut" edges */
    auto cut_edges = nodeLineWithFirstCoordinate(bound.get());
    if( !cut_edges )
        return nullptr;

    /* NOTE: the noding process may drop lines collapsing to points.
    *       We want to retrieve any of those */
    auto pi = extractUniquePoints(bound.get());
    auto po = extractUniquePoints(cut_edges.get());
    std::unique_ptr<geom::Geometry> collapse_points(pi->difference(po.get()));
    assert(collapse_points);
    pi.reset();
    po.reset();

    /* And use an empty geometry as initial "area" */
    const GeometryFactory* factory = geom->getFactory();
    std::unique_ptr<geom::Geometry> area(factory->createPolygon());
    assert(area);

    /*
    * See if an area can be build with the remaining edges
    * and if it can, symdifference with the original area.
    * Iterate this until no more polygons can be created
    * with left-over edges.
    */
    while( cut_edges->getNumGeometries() ) {

        // ASSUMPTION: cut_edges should already be fully noded
        auto new_area = geos::operation::polygonize::BuildArea().build(cut_edges.get());
        assert(new_area); // never return nullptr, but exception
        if( new_area->isEmpty() ) {
            /* no more rings can be build with thes edges */
            break;
        }

        // We succeeded in building a ring !
        // Save the new ring boundaries first (to compute further cut edges later)
        std::unique_ptr<geom::Geometry> new_area_bound = new_area->getBoundary();
        assert(new_area_bound);

        // Now symdif new and old area
        std::unique_ptr<geom::Geometry> symdif(area->symDifference(new_area.get()));
        assert(symdif);

        area = std::move(symdif);

        /*
        * Now let's re-set cut_edges with what's left
        * from the original boundary.
        * ASSUMPTION: only the previous cut-edges can be
        *             left, so we don't need to reconsider
        *             the whole original boundaries
        *
        * NOTE: this is an expensive operation.
        *
        */
        std::unique_ptr<geom::Geometry> new_cut_edges(cut_edges->difference(new_area_bound.get()));
        assert(new_cut_edges);

        cut_edges = std::move(new_cut_edges);
    }

    std::vector<std::unique_ptr<Geometry>> vgeoms(3);
    unsigned int nvgeoms=0;

    if( !area->isEmpty() ) {
        vgeoms[nvgeoms++] = std::move(area);
    }
    if( !cut_edges->isEmpty() ) {
        vgeoms[nvgeoms++] = std::move(cut_edges);
    }
    if( !collapse_points->isEmpty() ) {
        vgeoms[nvgeoms++] = std::move(collapse_points);
    }

    if( nvgeoms == 1 ) {
        /* Return cut edges */
        return std::move(vgeoms[0]);
    }

    /* Collect areas and lines (if any line) */
    vgeoms.resize(nvgeoms);
    return factory->createGeometryCollection(std::move(vgeoms));
}

static std::unique_ptr<geom::Geometry> MakeValidCollection(const geom::GeometryCollection* coll)
{
    std::vector<std::unique_ptr<Geometry>> validGeoms;
    for(const auto& geom: *coll) {
        validGeoms.push_back(MakeValid().build(geom.get()));
    }
    return coll->getFactory()->createGeometryCollection(std::move(validGeoms));
}

/** Return a valid version of the input geometry. */
std::unique_ptr<geom::Geometry> MakeValid::build(const geom::Geometry* geom)
{

    IsValidOp ivo(geom);
    if( ivo.getValidationError() == nullptr ) {
        return std::unique_ptr<geom::Geometry>(geom->clone());
    }

    auto typeId = geom->getGeometryTypeId();
    if( typeId == GEOS_LINESTRING ) {
        auto lineString = dynamic_cast<const LineString*>(geom);
        return MakeValidLine(lineString);
    }
    if( typeId == GEOS_MULTILINESTRING ) {
        auto mls = dynamic_cast<const MultiLineString*>(geom);
        return MakeValidMultiLine(mls);
    }
    if( typeId == GEOS_POLYGON ||
        typeId == GEOS_MULTIPOLYGON ) {
        return MakeValidPoly(geom);
    }
    if( typeId == GEOS_GEOMETRYCOLLECTION ) {
        auto coll = dynamic_cast<const GeometryCollection*>(geom);
        return MakeValidCollection(coll);
    }

    throw util::UnsupportedOperationException();
}

} // namespace geos.operation.valid
} // namespace geos.operation
} // namespace geos

<commit_msg>Tie MakeValid implementation directly to the old overlay operations<commit_after>/**********************************************************************
 *
 * GEOS - Geometry Engine Open Source
 * http://geos.osgeo.org
 *
 * Copyright 2009-2010 Sandro Santilli <strk@kbt.io>
 * Copyright (C) 2019 Even Rouault <even.rouault@spatialys.com>
 *
 * This is free software; you can redistribute and/or modify it under
 * the terms of the GNU Lesser General Public Licence as published
 * by the Free Software Foundation.
 * See the COPYING file for more information.
 **********************************************************************
 *
 * Ported from rtgeom_geos.c from
 *   rttopo - topology library
 *   http://git.osgeo.org/gitea/rttopo/librttopo
 * with relicensing from GPL to LGPL with Copyright holder permission.
 *
 **********************************************************************/

#include <geos/operation/valid/MakeValid.h>
#include <geos/operation/valid/IsValidOp.h>
#include <geos/operation/overlay/OverlayOp.h>
#include <geos/operation/polygonize/BuildArea.h>
#include <geos/operation/union/UnaryUnionOp.h>
#include <geos/geom/Geometry.h>
#include <geos/geom/GeometryCollection.h>
#include <geos/geom/GeometryFactory.h>
#include <geos/geom/LineString.h>
#include <geos/geom/Point.h>
#include <geos/geom/Polygon.h>
#include <geos/geom/MultiLineString.h>
#include <geos/geom/MultiPolygon.h>
#include <geos/util/UniqueCoordinateArrayFilter.h>
#include <geos/util/UnsupportedOperationException.h>

// std
#include <cassert>
#include <algorithm>
#include <utility>
#include <vector>

#ifdef _MSC_VER
#pragma warning(disable:4355)
#endif

using namespace geos::geom;
using geos::operation::overlay::OverlayOp;
using geos::operation::geounion::UnaryUnionOp;

namespace geos {
namespace operation { // geos.operation
namespace valid { // geos.operation.valid

static std::unique_ptr<geom::Geometry>
makeValidSymDifference(const geom::Geometry* g0, const geom::Geometry* g1)
{
    std::unique_ptr<Geometry> r(OverlayOp::overlayOp(g0, g1, OverlayOp::opSYMDIFFERENCE));
    return r;
}

static std::unique_ptr<geom::Geometry>
makeValidDifference(const geom::Geometry* g0, const geom::Geometry* g1)
{
    std::unique_ptr<Geometry> r(OverlayOp::overlayOp(g0, g1, OverlayOp::opDIFFERENCE));
    return r;
}

static std::unique_ptr<geom::Geometry>
makeValidUnion(const geom::Geometry* g0, const geom::Geometry* g1)
{
    std::unique_ptr<Geometry> r(OverlayOp::overlayOp(g0, g1, OverlayOp::opUNION));
    return r;
}

/*
 * Fully node given linework
 */
static std::unique_ptr<geom::Geometry>
nodeLineWithFirstCoordinate(const geom::Geometry* geom)
{
  /*
   * Union with first geometry point, obtaining full noding
   * and dissolving of duplicated repeated points
   *
   * TODO: substitute this with UnaryUnion?
   */

  if( geom->isEmpty() )
      return nullptr;

  const auto geomType = geom->getGeometryTypeId();
  assert( geomType == GEOS_LINESTRING || geomType == GEOS_MULTILINESTRING );

  std::unique_ptr<geom::Geometry> point;
  if( geomType == GEOS_LINESTRING ) {
      auto line = dynamic_cast<const geom::LineString*>(geom);
      assert(line);
      point = line->getPointN(0);
  } else {
      auto mls = dynamic_cast<const geom::MultiLineString*>(geom);
      assert(mls);
      auto line = mls->getGeometryN(0);
      assert(line);
      point = line->getPointN(0);
  }

  return makeValidUnion(geom, point.get());
}

static std::unique_ptr<geom::Geometry> MakeValidLine(const geom::LineString* line)
{
    return nodeLineWithFirstCoordinate(line);
}

static std::unique_ptr<geom::Geometry> MakeValidMultiLine(const geom::MultiLineString* mls)
{
    std::vector<std::unique_ptr<geom::Geometry>> points;
    std::vector<std::unique_ptr<geom::Geometry>> lines;

    for(const auto& subgeom: *mls) {
        auto line = dynamic_cast<const geom::LineString*>(subgeom.get());
        assert(line);
        auto validSubGeom = MakeValidLine(line);
        if( !validSubGeom || validSubGeom->isEmpty() ) {
            continue;
        }
        auto validLineType = validSubGeom->getGeometryTypeId();
        if( validLineType == GEOS_POINT ) {
            points.emplace_back(std::move(validSubGeom));
        }
        else if( validLineType == GEOS_LINESTRING ) {
            lines.emplace_back(std::move(validSubGeom));
        } else if( validLineType == GEOS_MULTILINESTRING ) {
            auto mlsValid = dynamic_cast<const geom::MultiLineString*>(validSubGeom.get());
            for(const auto& subgeomMlsValid: *mlsValid) {
                lines.emplace_back(subgeomMlsValid->clone());
            }
        } else {
            throw util::UnsupportedOperationException();
        }
    }

    std::unique_ptr<geom::Geometry> pointsRet;
    if( !points.empty() ) {
        if( points.size() > 1 ) {
            pointsRet = mls->getFactory()->createMultiPoint(std::move(points));
        } else {
            pointsRet = std::move(points[0]);
        }
    }

    std::unique_ptr<geom::Geometry> linesRet;
    if( !lines.empty() ) {
        if( lines.size() > 1 ) {
            linesRet = mls->getFactory()->createMultiLineString(std::move(lines));
        } else {
            linesRet = std::move(lines[0]);
        }
    }

    if( pointsRet && linesRet ) {
        std::vector<std::unique_ptr<Geometry>> geoms(2);
        geoms[0] = std::move(pointsRet);
        geoms[1] = std::move(linesRet);
        return mls->getFactory()->createGeometryCollection(std::move(geoms));
    } else if( pointsRet ) {
        return pointsRet;
    } else if( linesRet ) {
        return linesRet;
    }

    return nullptr;
}

static std::unique_ptr<geom::Geometry> extractUniquePoints(const geom::Geometry* geom)
{

    // Code taken from GEOSGeom_extractUniquePoints_r()

    /* 1: extract points */
    std::vector<const geom::Coordinate*> coords;
    geos::util::UniqueCoordinateArrayFilter filter(coords);
    geom->apply_ro(&filter);

    /* 2: for each point, create a geometry and put into a vector */
    std::vector<std::unique_ptr<Geometry>> points;
    points.reserve(coords.size());
    const GeometryFactory* factory = geom->getFactory();
    for(const Coordinate* c : coords) {
        points.emplace_back(factory->createPoint(*c));
    }

    /* 3: create a multipoint */
    return factory->createMultiPoint(std::move(points));
}

static std::unique_ptr<geom::Geometry> MakeValidPoly(const geom::Geometry* geom)
{
    assert( geom->getGeometryTypeId() == GEOS_POLYGON ||
            geom->getGeometryTypeId() == GEOS_MULTIPOLYGON );

    std::unique_ptr<geom::Geometry> bound(geom->getBoundary());
    if( !bound )
        return nullptr;

    /* Use noded boundaries as initial "cut" edges */
    auto cut_edges = nodeLineWithFirstCoordinate(bound.get());
    if( !cut_edges )
        return nullptr;

    /* NOTE: the noding process may drop lines collapsing to points.
    *       We want to retrieve any of those */
    auto pi = extractUniquePoints(bound.get());
    auto po = extractUniquePoints(cut_edges.get());
    std::unique_ptr<geom::Geometry> collapse_points = makeValidDifference(pi.get(), po.get());
    assert(collapse_points);
    pi.reset();
    po.reset();

    /* And use an empty geometry as initial "area" */
    const GeometryFactory* factory = geom->getFactory();
    std::unique_ptr<geom::Geometry> area(factory->createPolygon());
    assert(area);

    /*
    * See if an area can be build with the remaining edges
    * and if it can, symdifference with the original area.
    * Iterate this until no more polygons can be created
    * with left-over edges.
    */
    while( cut_edges->getNumGeometries() ) {

        // ASSUMPTION: cut_edges should already be fully noded
        auto new_area = geos::operation::polygonize::BuildArea().build(cut_edges.get());
        assert(new_area); // never return nullptr, but exception
        if( new_area->isEmpty() ) {
            /* no more rings can be build with thes edges */
            break;
        }

        // We succeeded in building a ring !
        // Save the new ring boundaries first (to compute further cut edges later)
        std::unique_ptr<geom::Geometry> new_area_bound = new_area->getBoundary();
        assert(new_area_bound);

        // Now symdif new and old area
        std::unique_ptr<geom::Geometry> symdif = makeValidSymDifference(area.get(), new_area.get());
        assert(symdif);

        area = std::move(symdif);

        /*
        * Now let's re-set cut_edges with what's left
        * from the original boundary.
        * ASSUMPTION: only the previous cut-edges can be
        *             left, so we don't need to reconsider
        *             the whole original boundaries
        *
        * NOTE: this is an expensive operation.
        *
        */
        std::unique_ptr<geom::Geometry> new_cut_edges = makeValidDifference(cut_edges.get(), new_area_bound.get());
        assert(new_cut_edges);

        cut_edges = std::move(new_cut_edges);
    }

    std::vector<std::unique_ptr<Geometry>> vgeoms(3);
    unsigned int nvgeoms=0;

    if( !area->isEmpty() ) {
        vgeoms[nvgeoms++] = std::move(area);
    }
    if( !cut_edges->isEmpty() ) {
        vgeoms[nvgeoms++] = std::move(cut_edges);
    }
    if( !collapse_points->isEmpty() ) {
        vgeoms[nvgeoms++] = std::move(collapse_points);
    }

    if( nvgeoms == 1 ) {
        /* Return cut edges */
        return std::move(vgeoms[0]);
    }

    /* Collect areas and lines (if any line) */
    vgeoms.resize(nvgeoms);
    return factory->createGeometryCollection(std::move(vgeoms));
}

static std::unique_ptr<geom::Geometry> MakeValidCollection(const geom::GeometryCollection* coll)
{
    std::vector<std::unique_ptr<Geometry>> validGeoms;
    for(const auto& geom: *coll) {
        validGeoms.push_back(MakeValid().build(geom.get()));
    }
    return coll->getFactory()->createGeometryCollection(std::move(validGeoms));
}

/** Return a valid version of the input geometry. */
std::unique_ptr<geom::Geometry> MakeValid::build(const geom::Geometry* geom)
{

    IsValidOp ivo(geom);
    if( ivo.getValidationError() == nullptr ) {
        return std::unique_ptr<geom::Geometry>(geom->clone());
    }

    auto typeId = geom->getGeometryTypeId();
    if( typeId == GEOS_LINESTRING ) {
        auto lineString = dynamic_cast<const LineString*>(geom);
        return MakeValidLine(lineString);
    }
    if( typeId == GEOS_MULTILINESTRING ) {
        auto mls = dynamic_cast<const MultiLineString*>(geom);
        return MakeValidMultiLine(mls);
    }
    if( typeId == GEOS_POLYGON ||
        typeId == GEOS_MULTIPOLYGON ) {
        return MakeValidPoly(geom);
    }
    if( typeId == GEOS_GEOMETRYCOLLECTION ) {
        auto coll = dynamic_cast<const GeometryCollection*>(geom);
        return MakeValidCollection(coll);
    }

    throw util::UnsupportedOperationException();
}

} // namespace geos.operation.valid
} // namespace geos.operation
} // namespace geos

<|endoftext|>