<commit_before>// Copyright 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "config.h"
#include "Request.h"

#include "bindings/core/v8/Dictionary.h"
#include "core/dom/ExecutionContext.h"
#include "core/fetch/FetchUtils.h"
#include "core/fetch/ResourceLoaderOptions.h"
#include "core/loader/ThreadableLoader.h"
#include "core/xml/XMLHttpRequest.h"
#include "modules/serviceworkers/FetchManager.h"
#include "modules/serviceworkers/RequestInit.h"
#include "platform/NotImplemented.h"
#include "platform/network/HTTPParsers.h"
#include "platform/network/ResourceRequest.h"
#include "platform/weborigin/Referrer.h"
#include "public/platform/WebServiceWorkerRequest.h"

namespace blink {

namespace {

PassRefPtrWillBeRawPtr<Request> createRequestWithRequestData(PassRefPtrWillBeRawPtr<FetchRequestData> request, const RequestInit& init, FetchRequestData::Mode mode, FetchRequestData::Credentials credentials, ExceptionState& exceptionState)
{
    // "6. Let |mode| be |init|'s mode member if it is present, and
    // |fallbackMode| otherwise."
    // "7. If |mode| is non-null, set |request|'s mode to |mode|."
    if (init.mode == "same-origin") {
        request->setMode(FetchRequestData::SameOriginMode);
    } else if (init.mode == "no-cors") {
        request->setMode(mode = FetchRequestData::NoCORSMode);
    } else if (init.mode == "cors") {
        request->setMode(FetchRequestData::CORSMode);
    } else {
        // Instead of using null as a special fallback value, we pass the
        // current mode in Request::create(). So we just set here.
        request->setMode(mode);
    }

    // "8. Let |credentials| be |init|'s credentials member if it is present,
    // and |fallbackCredentials| otherwise."
    // "9. If |credentials| is non-null, set |request|'s credentials mode to
    // |credentials|.
    if (init.credentials == "omit") {
        request->setCredentials(FetchRequestData::OmitCredentials);
    } else if (init.credentials == "same-origin") {
        request->setCredentials(FetchRequestData::SameOriginCredentials);
    } else if (init.credentials == "include") {
        request->setCredentials(FetchRequestData::IncludeCredentials);
    } else {
        // Instead of using null as a special fallback value, we pass the
        // current credentials in Request::create(). So we just set here.
        request->setCredentials(credentials);
    }

    // "10. If |init|'s method member is present, let |method| be it and run
    // these substeps:"
    if (!init.method.isEmpty()) {
        // "1. If |method| is not a useful method, throw a TypeError."
        if (!FetchUtils::isUsefulMethod(init.method)) {
            exceptionState.throwTypeError("'" + init.method + "' HTTP method is unsupported.");
            return nullptr;
        }
        if (!isValidHTTPToken(init.method)) {
            exceptionState.throwTypeError("'" + init.method + "' is not a valid HTTP method.");
            return nullptr;
        }
        // FIXME: "2. Add case correction as in XMLHttpRequest?"
        // "3. Set |request|'s method to |method|."
        request->setMethod(XMLHttpRequest::uppercaseKnownHTTPMethod(AtomicString(init.method)));
    }
    // "11. Let |r| be a new Request object associated with |request|, Headers
    // object, and FetchBodyStream object."
    RefPtrWillBeRawPtr<Request> r = Request::create(request);

    // "12. Let |headers| be a copy of |r|'s Headers object."
    // "13. If |init|'s headers member is present, set |headers| to |init|'s
    // headers member."
    // We don't create a copy of r's Headers object when init's headers member
    // is present.
    RefPtrWillBeRawPtr<Headers> headers = nullptr;
    if (!init.headers && init.headersDictionary.isUndefinedOrNull()) {
        headers = r->headers()->createCopy();
    }
    // "14. Empty |r|'s request's header list."
    r->request()->headerList()->clearList();

    // "15. If |r|'s request's mode is no CORS, run these substeps:
    if (r->request()->mode() == FetchRequestData::NoCORSMode) {
        // "1. If |r|'s request's method is not a simple method, throw a
        // TypeError."
        if (!FetchUtils::isSimpleMethod(r->request()->method())) {
            exceptionState.throwTypeError("'" + r->request()->method() + "' is unsupported in no-cors mode.");
            return nullptr;
        }
        // "Set |r|'s Headers object's guard to |request-no-CORS|.
        r->headers()->setGuard(Headers::RequestNoCORSGuard);
    }

    // "16. Fill |r|'s Headers object with |headers|. Rethrow any exceptions."
    if (init.headers) {
        ASSERT(init.headersDictionary.isUndefinedOrNull());
        r->headers()->fillWith(init.headers.get(), exceptionState);
    } else if (!init.headersDictionary.isUndefinedOrNull()) {
        r->headers()->fillWith(init.headersDictionary, exceptionState);
    } else {
        ASSERT(headers);
        r->headers()->fillWith(headers.get(), exceptionState);
    }
    if (exceptionState.hadException())
        return nullptr;
    // FIXME: Support body.
    // "20. Return |r|."
    return r.release();
}

} // namespace

DEFINE_EMPTY_DESTRUCTOR_WILL_BE_REMOVED(Request);

PassRefPtrWillBeRawPtr<Request> Request::create(ExecutionContext* context, const String& input, ExceptionState& exceptionState)
{
    return create(context, input, Dictionary(), exceptionState);
}

PassRefPtrWillBeRawPtr<Request> Request::create(ExecutionContext* context, const String& input, const Dictionary& init, ExceptionState& exceptionState)
{
    // "1. Let |request| be |input|'s associated request, if |input| is a
    // Request object, and a new request otherwise."
    RefPtrWillBeRawPtr<FetchRequestData> request(FetchRequestData::create(context));
    // "2. Set |request| to a restricted copy of itself."
    request = request->createRestrictedCopy(context, SecurityOrigin::create(context->url()));
    // "5. If |input| is a string, run these substeps:"
    // "1. Let |parsedURL| be the result of parsing |input| with entry settings
    // object's API base URL."
    KURL parsedURL = context->completeURL(input);
    // "2. If |parsedURL| is failure, throw a TypeError."
    if (!parsedURL.isValid()) {
        exceptionState.throwTypeError("Invalid URL");
        return nullptr;
    }
    // "3. Set |request|'s url to |parsedURL|."
    request->setURL(parsedURL);
    // "4. Set |fallbackMode| to CORS."
    // "5. Set |fallbackCredentials| to omit."
    return createRequestWithRequestData(request.release(), RequestInit(init), FetchRequestData::CORSMode, FetchRequestData::OmitCredentials, exceptionState);
}

PassRefPtrWillBeRawPtr<Request> Request::create(ExecutionContext* context, Request* input, ExceptionState& exceptionState)
{
    return create(context, input, Dictionary(), exceptionState);
}

PassRefPtrWillBeRawPtr<Request> Request::create(ExecutionContext* context, Request* input, const Dictionary& init, ExceptionState& exceptionState)
{
    // "1. Let |request| be |input|'s associated request, if |input| is a
    // Request object, and a new request otherwise."
    // "2. Set |request| to a restricted copy of itself."
    RefPtrWillBeRawPtr<FetchRequestData> request(input->request()->createRestrictedCopy(context, SecurityOrigin::create(context->url())));
    // "3. Let |fallbackMode| be null."
    // "4. Let |fallbackCredentials| be null."
    // Instead of using null as a special fallback value, just pass the current
    // mode and credentials; it has the same effect.
    const FetchRequestData::Mode currentMode = request->mode();
    const FetchRequestData::Credentials currentCredentials = request->credentials();
    return createRequestWithRequestData(request.release(), RequestInit(init), currentMode, currentCredentials, exceptionState);
}

PassRefPtrWillBeRawPtr<Request> Request::create(PassRefPtrWillBeRawPtr<FetchRequestData> request)
{
    return adoptRefWillBeNoop(new Request(request));
}

Request::Request(PassRefPtrWillBeRawPtr<FetchRequestData> request)
    : m_request(request)
    , m_headers(Headers::create(m_request->headerList()))
{
    m_headers->setGuard(Headers::RequestGuard);
    ScriptWrappable::init(this);
}

PassRefPtrWillBeRawPtr<Request> Request::create(const WebServiceWorkerRequest& webRequest)
{
    return adoptRefWillBeNoop(new Request(webRequest));
}

Request::Request(const WebServiceWorkerRequest& webRequest)
    : m_request(FetchRequestData::create(webRequest))
    , m_headers(Headers::create(m_request->headerList()))
{
    m_headers->setGuard(Headers::RequestGuard);
    ScriptWrappable::init(this);
}

String Request::method() const
{
    // "The method attribute's getter must return request's method."
    return m_request->method();
}

String Request::url() const
{
    // The url attribute's getter must return request's url, serialized with the exclude fragment flag set.
    if (!m_request->url().hasFragmentIdentifier())
        return m_request->url();
    KURL url(m_request->url());
    url.removeFragmentIdentifier();
    return url;
}

String Request::referrer() const
{
    // "The referrer attribute's getter must return the empty string if
    // request's referrer is none, and request's referrer, serialized,
    // otherwise."
    return m_request->referrer().referrer().referrer;
}

String Request::mode() const
{
    // "The mode attribute's getter must return the value corresponding to the
    // first matching statement, switching on request's mode:"
    switch (m_request->mode()) {
    case FetchRequestData::SameOriginMode:
        return "same-origin";
    case FetchRequestData::NoCORSMode:
        return "no-cors";
    case FetchRequestData::CORSMode:
    case FetchRequestData::CORSWithForcedPreflight:
        return "cors";
    }
    ASSERT_NOT_REACHED();
    return "";
}

String Request::credentials() const
{
    // "The credentials attribute's getter must return the value corresponding
    // to the first matching statement, switching on request's credentials
    // mode:"
    switch (m_request->credentials()) {
    case FetchRequestData::OmitCredentials:
        return "omit";
    case FetchRequestData::SameOriginCredentials:
        return "same-origin";
    case FetchRequestData::IncludeCredentials:
        return "include";
    }
    ASSERT_NOT_REACHED();
    return "";
}

void Request::trace(Visitor* visitor)
{
    visitor->trace(m_request);
    visitor->trace(m_headers);
}

} // namespace blink
<commit_msg>Service Worker: Remove unnecessary newline from Request.cpp<commit_after>// Copyright 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "config.h"
#include "Request.h"

#include "bindings/core/v8/Dictionary.h"
#include "core/dom/ExecutionContext.h"
#include "core/fetch/FetchUtils.h"
#include "core/fetch/ResourceLoaderOptions.h"
#include "core/loader/ThreadableLoader.h"
#include "core/xml/XMLHttpRequest.h"
#include "modules/serviceworkers/FetchManager.h"
#include "modules/serviceworkers/RequestInit.h"
#include "platform/NotImplemented.h"
#include "platform/network/HTTPParsers.h"
#include "platform/network/ResourceRequest.h"
#include "platform/weborigin/Referrer.h"
#include "public/platform/WebServiceWorkerRequest.h"

namespace blink {

namespace {

PassRefPtrWillBeRawPtr<Request> createRequestWithRequestData(PassRefPtrWillBeRawPtr<FetchRequestData> request, const RequestInit& init, FetchRequestData::Mode mode, FetchRequestData::Credentials credentials, ExceptionState& exceptionState)
{
    // "6. Let |mode| be |init|'s mode member if it is present, and
    // |fallbackMode| otherwise."
    // "7. If |mode| is non-null, set |request|'s mode to |mode|."
    if (init.mode == "same-origin") {
        request->setMode(FetchRequestData::SameOriginMode);
    } else if (init.mode == "no-cors") {
        request->setMode(mode = FetchRequestData::NoCORSMode);
    } else if (init.mode == "cors") {
        request->setMode(FetchRequestData::CORSMode);
    } else {
        // Instead of using null as a special fallback value, we pass the
        // current mode in Request::create(). So we just set here.
        request->setMode(mode);
    }

    // "8. Let |credentials| be |init|'s credentials member if it is present,
    // and |fallbackCredentials| otherwise."
    // "9. If |credentials| is non-null, set |request|'s credentials mode to
    // |credentials|.
    if (init.credentials == "omit") {
        request->setCredentials(FetchRequestData::OmitCredentials);
    } else if (init.credentials == "same-origin") {
        request->setCredentials(FetchRequestData::SameOriginCredentials);
    } else if (init.credentials == "include") {
        request->setCredentials(FetchRequestData::IncludeCredentials);
    } else {
        // Instead of using null as a special fallback value, we pass the
        // current credentials in Request::create(). So we just set here.
        request->setCredentials(credentials);
    }

    // "10. If |init|'s method member is present, let |method| be it and run
    // these substeps:"
    if (!init.method.isEmpty()) {
        // "1. If |method| is not a useful method, throw a TypeError."
        if (!FetchUtils::isUsefulMethod(init.method)) {
            exceptionState.throwTypeError("'" + init.method + "' HTTP method is unsupported.");
            return nullptr;
        }
        if (!isValidHTTPToken(init.method)) {
            exceptionState.throwTypeError("'" + init.method + "' is not a valid HTTP method.");
            return nullptr;
        }
        // FIXME: "2. Add case correction as in XMLHttpRequest?"
        // "3. Set |request|'s method to |method|."
        request->setMethod(XMLHttpRequest::uppercaseKnownHTTPMethod(AtomicString(init.method)));
    }
    // "11. Let |r| be a new Request object associated with |request|, Headers
    // object, and FetchBodyStream object."
    RefPtrWillBeRawPtr<Request> r = Request::create(request);

    // "12. Let |headers| be a copy of |r|'s Headers object."
    // "13. If |init|'s headers member is present, set |headers| to |init|'s
    // headers member."
    // We don't create a copy of r's Headers object when init's headers member
    // is present.
    RefPtrWillBeRawPtr<Headers> headers = nullptr;
    if (!init.headers && init.headersDictionary.isUndefinedOrNull()) {
        headers = r->headers()->createCopy();
    }
    // "14. Empty |r|'s request's header list."
    r->request()->headerList()->clearList();

    // "15. If |r|'s request's mode is no CORS, run these substeps:
    if (r->request()->mode() == FetchRequestData::NoCORSMode) {
        // "1. If |r|'s request's method is not a simple method, throw a
        // TypeError."
        if (!FetchUtils::isSimpleMethod(r->request()->method())) {
            exceptionState.throwTypeError("'" + r->request()->method() + "' is unsupported in no-cors mode.");
            return nullptr;
        }
        // "Set |r|'s Headers object's guard to |request-no-CORS|.
        r->headers()->setGuard(Headers::RequestNoCORSGuard);
    }

    // "16. Fill |r|'s Headers object with |headers|. Rethrow any exceptions."
    if (init.headers) {
        ASSERT(init.headersDictionary.isUndefinedOrNull());
        r->headers()->fillWith(init.headers.get(), exceptionState);
    } else if (!init.headersDictionary.isUndefinedOrNull()) {
        r->headers()->fillWith(init.headersDictionary, exceptionState);
    } else {
        ASSERT(headers);
        r->headers()->fillWith(headers.get(), exceptionState);
    }
    if (exceptionState.hadException())
        return nullptr;
    // FIXME: Support body.
    // "20. Return |r|."
    return r.release();
}

} // namespace

DEFINE_EMPTY_DESTRUCTOR_WILL_BE_REMOVED(Request);

PassRefPtrWillBeRawPtr<Request> Request::create(ExecutionContext* context, const String& input, ExceptionState& exceptionState)
{
    return create(context, input, Dictionary(), exceptionState);
}

PassRefPtrWillBeRawPtr<Request> Request::create(ExecutionContext* context, const String& input, const Dictionary& init, ExceptionState& exceptionState)
{
    // "1. Let |request| be |input|'s associated request, if |input| is a
    // Request object, and a new request otherwise."
    RefPtrWillBeRawPtr<FetchRequestData> request(FetchRequestData::create(context));
    // "2. Set |request| to a restricted copy of itself."
    request = request->createRestrictedCopy(context, SecurityOrigin::create(context->url()));
    // "5. If |input| is a string, run these substeps:"
    // "1. Let |parsedURL| be the result of parsing |input| with entry settings
    // object's API base URL."
    KURL parsedURL = context->completeURL(input);
    // "2. If |parsedURL| is failure, throw a TypeError."
    if (!parsedURL.isValid()) {
        exceptionState.throwTypeError("Invalid URL");
        return nullptr;
    }
    // "3. Set |request|'s url to |parsedURL|."
    request->setURL(parsedURL);
    // "4. Set |fallbackMode| to CORS."
    // "5. Set |fallbackCredentials| to omit."
    return createRequestWithRequestData(request.release(), RequestInit(init), FetchRequestData::CORSMode, FetchRequestData::OmitCredentials, exceptionState);
}

PassRefPtrWillBeRawPtr<Request> Request::create(ExecutionContext* context, Request* input, ExceptionState& exceptionState)
{
    return create(context, input, Dictionary(), exceptionState);
}

PassRefPtrWillBeRawPtr<Request> Request::create(ExecutionContext* context, Request* input, const Dictionary& init, ExceptionState& exceptionState)
{
    // "1. Let |request| be |input|'s associated request, if |input| is a
    // Request object, and a new request otherwise."
    // "2. Set |request| to a restricted copy of itself."
    RefPtrWillBeRawPtr<FetchRequestData> request(input->request()->createRestrictedCopy(context, SecurityOrigin::create(context->url())));
    // "3. Let |fallbackMode| be null."
    // "4. Let |fallbackCredentials| be null."
    // Instead of using null as a special fallback value, just pass the current
    // mode and credentials; it has the same effect.
    const FetchRequestData::Mode currentMode = request->mode();
    const FetchRequestData::Credentials currentCredentials = request->credentials();
    return createRequestWithRequestData(request.release(), RequestInit(init), currentMode, currentCredentials, exceptionState);
}

PassRefPtrWillBeRawPtr<Request> Request::create(PassRefPtrWillBeRawPtr<FetchRequestData> request)
{
    return adoptRefWillBeNoop(new Request(request));
}

Request::Request(PassRefPtrWillBeRawPtr<FetchRequestData> request)
    : m_request(request)
    , m_headers(Headers::create(m_request->headerList()))
{
    m_headers->setGuard(Headers::RequestGuard);
    ScriptWrappable::init(this);
}

PassRefPtrWillBeRawPtr<Request> Request::create(const WebServiceWorkerRequest& webRequest)
{
    return adoptRefWillBeNoop(new Request(webRequest));
}

Request::Request(const WebServiceWorkerRequest& webRequest)
    : m_request(FetchRequestData::create(webRequest))
    , m_headers(Headers::create(m_request->headerList()))
{
    m_headers->setGuard(Headers::RequestGuard);
    ScriptWrappable::init(this);
}

String Request::method() const
{
    // "The method attribute's getter must return request's method."
    return m_request->method();
}

String Request::url() const
{
    // The url attribute's getter must return request's url, serialized with the exclude fragment flag set.
    if (!m_request->url().hasFragmentIdentifier())
        return m_request->url();
    KURL url(m_request->url());
    url.removeFragmentIdentifier();
    return url;
}

String Request::referrer() const
{
    // "The referrer attribute's getter must return the empty string if
    // request's referrer is none, and request's referrer, serialized,
    // otherwise."
    return m_request->referrer().referrer().referrer;
}

String Request::mode() const
{
    // "The mode attribute's getter must return the value corresponding to the
    // first matching statement, switching on request's mode:"
    switch (m_request->mode()) {
    case FetchRequestData::SameOriginMode:
        return "same-origin";
    case FetchRequestData::NoCORSMode:
        return "no-cors";
    case FetchRequestData::CORSMode:
    case FetchRequestData::CORSWithForcedPreflight:
        return "cors";
    }
    ASSERT_NOT_REACHED();
    return "";
}

String Request::credentials() const
{
    // "The credentials attribute's getter must return the value corresponding
    // to the first matching statement, switching on request's credentials
    // mode:"
    switch (m_request->credentials()) {
    case FetchRequestData::OmitCredentials:
        return "omit";
    case FetchRequestData::SameOriginCredentials:
        return "same-origin";
    case FetchRequestData::IncludeCredentials:
        return "include";
    }
    ASSERT_NOT_REACHED();
    return "";
}

void Request::trace(Visitor* visitor)
{
    visitor->trace(m_request);
    visitor->trace(m_headers);
}

} // namespace blink
<|endoftext|>


<commit_before>/*  MConfig - C++ library for working with configuration files
    Copyright (C) 2011 Dmitry Shatrov

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include <libmary/types.h>
#include <cctype>

#include <mycpp/native_file.h>
#include <mycpp/cached_file.h>

#include <pargen/parser.h>

#include <scruffy/preprocessor.h>
#include <scruffy/list_pp_item_stream.h>
#include <scruffy/pp_item_stream_token_stream.h>
#include <scruffy/checkpoint_tracker.h>

#include <mconfig/mconfig_pargen.h>

#include <mconfig/config_parser.h>

using namespace M;

namespace MConfig {

namespace {
LogGroup libMary_logGroup_mconfig ("mconfig", LogLevel::N);
}

class ConfigParser
{
public:
    MyCpp::List<Section*> sections;

    Scruffy::CheckpointTracker checkpoint_tracker;

    ConfigParser (Config * const config)
    {
	sections.append (config->getRootSection());
    }
};

// Characters banned from words: " { } , ; # =
//
// " is used for string literals, which are provided by pargen with quotes
// included.
//
bool
mconfig_word_token_match_func (MyCpp::ConstMemoryDesc const &token_mem,
			       void * const /* token_user_ptr */,
			       void * const /* user_data */)
{
    logD (mconfig, _func, ConstMemory (token_mem));

    if (token_mem.getLength() == 0)
	return false;

    char const *token = (char const*) token_mem.getMemory();
    if (token [0] == '{' ||
	token [0] == '}' ||
	token [0] == ',' ||
	token [0] == ';' ||
	token [0] == '#' ||
	token [0] == '=')
    {
	return false;
    }

    if (token [0] == '"'
	&& (token_mem.getLength() == 1
	    || token [token_mem.getLength() - 1] != '"'))
    {
	return false;
    }

    return true;
}

static bool
whitespaceBetweenWordsNeeded (ConstMemory const &left,
			      ConstMemory const &right)
{
    if (left .len() == 0 ||
	right.len() == 0)
    {
	return false;
    }

    if (isalnum (left.mem() [left.len() - 1]) &&
	isalnum (right.mem() [0]))
    {
	return true;
    }

    return false;
}

static Ref<String>
wordsToString (MyCpp::List<MConfig_Word*> * const mt_nonnull words)
{
    Size str_len = 0;
    {
	ConstMemory prv_word;
	MyCpp::List<MConfig_Word*>::DataIterator iter (*words);
	while (!iter.done ()) {
	    MConfig_Word * const word_ = iter.next();

	    ConstMemory word = word_->any_token->token;
	    // Stripping quotes from string literals.
	    if (word.len() && word.mem() [0] == '"') {
		assert (word.len() >= 2);
		word = word.region (1, word.len() - 2);
	    }

	    str_len += word.len();

	    if (whitespaceBetweenWordsNeeded (prv_word, word))
		++str_len;

	    prv_word = word;
	}
    }

    logD (mconfig, _func, "str_len: ", str_len);

    Ref<String> str = grab (new String (str_len));

    {
	Size pos = 0;
	ConstMemory prv_word;
	MyCpp::List<MConfig_Word*>::DataIterator iter (*words);
	while (!iter.done ()) {
	    MConfig_Word * const word_ = iter.next();

	    ConstMemory word = word_->any_token->token;
	    logD (mconfig, _func, "word: ", word);
	    // Stripping quotes from string literals.
	    if (word.len() && word.mem() [0] == '"') {
		assert (word.len() >= 2);
		word = word.region (1, word.len() - 2);
	    }

	    if (whitespaceBetweenWordsNeeded (prv_word, word)) {
		str->mem().mem() [pos] = ' ';
		++pos;
	    }

	    memcpy (str->mem().mem() + pos, word.mem(), word.len());
	    pos += word.len();

	    prv_word = word;
	}
    }

    logD (mconfig, _func, "str: ", str);

    return str;
}

static Ref<String>
keyToString (MConfig_Key * const key)
{
    if (!key)
	return grab (new String());

    return wordsToString (&key->words);
}

bool
mconfig_begin_section (MConfig_Section       * const section,
		       Pargen::ParserControl * const /* parser_control */,
		       void                  * const _self)
{
    ConfigParser * const self = static_cast <ConfigParser*> (_self);

    Ref<String> section_name = keyToString (section->name);

    logD (mconfig, _func, "section: ", section_name);

// Section value replacement was disabled to allow lists of sections
// with the same name (for Moment's mod_file).
//  Section *opts_section = self->sections.getLast()->getSection (section_name->mem());
    Section *opts_section = NULL;
    if (!opts_section) {
	opts_section = new Section (keyToString (section->name)->mem());
	self->sections.getLast()->addSection (opts_section);
    }

    self->sections.append (opts_section);
    self->checkpoint_tracker.addUnconditionalCancellable (
	    MyCpp::grab (new Scruffy::Cancellable_ListElement<Section*> (
		    self->sections,
		    self->sections.last)));

    return true;
}

bool
mconfig_accept_option (MConfig_Option         * const option,
		       Pargen::ParserControl  * const /* parser_control */,
		       void                   * const _self)
{
    ConfigParser * const self = static_cast <ConfigParser*> (_self);

    if (self->sections.isEmpty()) {
	logE_ (_func, "option not within a section");
	return false;
    }

    Section * const section = self->sections.getLast();

    MConfig_Option_KeyValue *option__key_value = NULL;

    MConfig_Key *key_elem = NULL;
    switch (option->option_type) {
	case MConfig_Option::_KeyValue: {
	    option__key_value = static_cast <MConfig_Option_KeyValue*> (option);
	    key_elem = option__key_value->key;
	} break;
	case MConfig_Option::_Key: {
	    MConfig_Option_Key * const option__key = static_cast <MConfig_Option_Key*> (option);
	    key_elem = option__key->key;
	} break;
	default:
	    unreachable ();
    }

    Ref<String> key = keyToString (key_elem);

    logD (mconfig, _func, "option: ", key);

    Option *opts_option = section->getOption (key->mem());
    if (opts_option) {
	opts_option->removeValues ();
    } else {
	opts_option = new Option (key->mem());
	self->sections.getLast()->addOption (opts_option);
    }

    if (option__key_value) {
	MConfig_Value *value = option__key_value->value;
	for (;;) {
	    switch (value->value_type) {
		case MConfig_Value::_List: {
		    MConfig_Value_List * const value__list = static_cast <MConfig_Value_List*> (value);
		    logD (mconfig, _func, "value: ", wordsToString (&value__list->words));
		    opts_option->addValue (wordsToString (&value__list->words)->mem());
		    value = value__list->value;
		    logD (mconfig, _func, "new value: 0x", fmt_hex, (UintPtr) value);
		} break;
		case MConfig_Value::_Word: {
		    MConfig_Value_Word * const value__word = static_cast <MConfig_Value_Word*> (value);
		    logD (mconfig, _func, "value: ", wordsToString (&value__word->words));
		    opts_option->addValue (wordsToString (&value__word->words)->mem());
		    value = NULL;
		} break;
		default:
		    unreachable ();
	    }

	    if (!value)
		break;
	}
    }

    logD (mconfig, _func, "done");

    return true;
}

Result parseConfig (ConstMemory const &filename,
		    Config * const config)
try {
    MyCpp::Ref<Pargen::Grammar> const grammar = create_mconfig_grammar ();

    MyCpp::Ref<MyCpp::File> file = MyCpp::File::createDefault (filename,
							       0 /* open_flags */,
							       MyCpp::AccessMode::ReadOnly);
    file = MyCpp::grab (new MyCpp::CachedFile (file, (1 << 14) /* page_size */, 64 /* max_pages */));

    MyCpp::Ref<Scruffy::CppPreprocessor> const preprocessor = MyCpp::grab (new Scruffy::CppPreprocessor (file));
    preprocessor->performPreprocessing ();

    MyCpp::Ref< MyCpp::List_< MyCpp::Ref<Scruffy::PpItem>, MyCpp::SimplyReferenced > > pp_items =
	    preprocessor->getPpItems ();

    MyCpp::Ref<Scruffy::PpItemStream> pp_stream =
	    MyCpp::grab (static_cast <Scruffy::PpItemStream*> (
		    new Scruffy::ListPpItemStream (pp_items->first, MyLang::FilePosition ())));

    MyCpp::Ref<Scruffy::PpItemStreamTokenStream> token_stream =
	    MyCpp::grab (new Scruffy::PpItemStreamTokenStream (pp_stream));
    token_stream->setNewlineReplacement (";");

    ConfigParser config_parser (config);

    Pargen::ParserElement *mconfig_elem = NULL;
    Pargen::parse (token_stream,
		   &config_parser.checkpoint_tracker,
		   &config_parser,
		   grammar,
		   &mconfig_elem,
		   MyCpp::ConstMemoryDesc::forString ("default"),
		   // TODO Set 'upwards_jumps' to true for non-AST mode.
		   Pargen::createParserConfig (false /* upwards_jumps */),
		   false /* debug_dump */);

    if (mconfig_elem == NULL ||
	token_stream->getNextToken ().getLength () > 0)
    {
	logE_ (_func, "Syntax error in configuration file ", filename);
	return Result::Failure;
    }

    if (logLevelOn (mconfig, LogLevel::Debug)) {
	dump_mconfig_grammar (static_cast <MConfig_Grammar*> (mconfig_elem));
	MyCpp::errf->pflush ();
    }

    return Result::Success;
} catch (...) {
    logE_ (_func, "Exception");
    return Result::Failure;
}

}

<commit_msg>Close config file after parsing<commit_after>/*  MConfig - C++ library for working with configuration files
    Copyright (C) 2011 Dmitry Shatrov

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include <libmary/types.h>
#include <cctype>

#include <mycpp/native_file.h>
#include <mycpp/cached_file.h>

#include <pargen/parser.h>

#include <scruffy/preprocessor.h>
#include <scruffy/list_pp_item_stream.h>
#include <scruffy/pp_item_stream_token_stream.h>
#include <scruffy/checkpoint_tracker.h>

#include <mconfig/mconfig_pargen.h>

#include <mconfig/config_parser.h>

using namespace M;

namespace MConfig {

namespace {
LogGroup libMary_logGroup_mconfig ("mconfig", LogLevel::N);
}

class ConfigParser
{
public:
    MyCpp::List<Section*> sections;

    Scruffy::CheckpointTracker checkpoint_tracker;

    ConfigParser (Config * const config)
    {
	sections.append (config->getRootSection());
    }
};

// Characters banned from words: " { } , ; # =
//
// " is used for string literals, which are provided by pargen with quotes
// included.
//
bool
mconfig_word_token_match_func (MyCpp::ConstMemoryDesc const &token_mem,
			       void * const /* token_user_ptr */,
			       void * const /* user_data */)
{
    logD (mconfig, _func, ConstMemory (token_mem));

    if (token_mem.getLength() == 0)
	return false;

    char const *token = (char const*) token_mem.getMemory();
    if (token [0] == '{' ||
	token [0] == '}' ||
	token [0] == ',' ||
	token [0] == ';' ||
	token [0] == '#' ||
	token [0] == '=')
    {
	return false;
    }

    if (token [0] == '"'
	&& (token_mem.getLength() == 1
	    || token [token_mem.getLength() - 1] != '"'))
    {
	return false;
    }

    return true;
}

static bool
whitespaceBetweenWordsNeeded (ConstMemory const &left,
			      ConstMemory const &right)
{
    if (left .len() == 0 ||
	right.len() == 0)
    {
	return false;
    }

    if (isalnum (left.mem() [left.len() - 1]) &&
	isalnum (right.mem() [0]))
    {
	return true;
    }

    return false;
}

static Ref<String>
wordsToString (MyCpp::List<MConfig_Word*> * const mt_nonnull words)
{
    Size str_len = 0;
    {
	ConstMemory prv_word;
	MyCpp::List<MConfig_Word*>::DataIterator iter (*words);
	while (!iter.done ()) {
	    MConfig_Word * const word_ = iter.next();

	    ConstMemory word = word_->any_token->token;
	    // Stripping quotes from string literals.
	    if (word.len() && word.mem() [0] == '"') {
		assert (word.len() >= 2);
		word = word.region (1, word.len() - 2);
	    }

	    str_len += word.len();

	    if (whitespaceBetweenWordsNeeded (prv_word, word))
		++str_len;

	    prv_word = word;
	}
    }

    logD (mconfig, _func, "str_len: ", str_len);

    Ref<String> str = grab (new String (str_len));

    {
	Size pos = 0;
	ConstMemory prv_word;
	MyCpp::List<MConfig_Word*>::DataIterator iter (*words);
	while (!iter.done ()) {
	    MConfig_Word * const word_ = iter.next();

	    ConstMemory word = word_->any_token->token;
	    logD (mconfig, _func, "word: ", word);
	    // Stripping quotes from string literals.
	    if (word.len() && word.mem() [0] == '"') {
		assert (word.len() >= 2);
		word = word.region (1, word.len() - 2);
	    }

	    if (whitespaceBetweenWordsNeeded (prv_word, word)) {
		str->mem().mem() [pos] = ' ';
		++pos;
	    }

	    memcpy (str->mem().mem() + pos, word.mem(), word.len());
	    pos += word.len();

	    prv_word = word;
	}
    }

    logD (mconfig, _func, "str: ", str);

    return str;
}

static Ref<String>
keyToString (MConfig_Key * const key)
{
    if (!key)
	return grab (new String());

    return wordsToString (&key->words);
}

bool
mconfig_begin_section (MConfig_Section       * const section,
		       Pargen::ParserControl * const /* parser_control */,
		       void                  * const _self)
{
    ConfigParser * const self = static_cast <ConfigParser*> (_self);

    Ref<String> section_name = keyToString (section->name);

    logD (mconfig, _func, "section: ", section_name);

// Section value replacement was disabled to allow lists of sections
// with the same name (for Moment's mod_file).
//  Section *opts_section = self->sections.getLast()->getSection (section_name->mem());
    Section *opts_section = NULL;
    if (!opts_section) {
	opts_section = new Section (keyToString (section->name)->mem());
	self->sections.getLast()->addSection (opts_section);
    }

    self->sections.append (opts_section);
    self->checkpoint_tracker.addUnconditionalCancellable (
	    MyCpp::grab (new Scruffy::Cancellable_ListElement<Section*> (
		    self->sections,
		    self->sections.last)));

    return true;
}

bool
mconfig_accept_option (MConfig_Option         * const option,
		       Pargen::ParserControl  * const /* parser_control */,
		       void                   * const _self)
{
    ConfigParser * const self = static_cast <ConfigParser*> (_self);

    if (self->sections.isEmpty()) {
	logE_ (_func, "option not within a section");
	return false;
    }

    Section * const section = self->sections.getLast();

    MConfig_Option_KeyValue *option__key_value = NULL;

    MConfig_Key *key_elem = NULL;
    switch (option->option_type) {
	case MConfig_Option::_KeyValue: {
	    option__key_value = static_cast <MConfig_Option_KeyValue*> (option);
	    key_elem = option__key_value->key;
	} break;
	case MConfig_Option::_Key: {
	    MConfig_Option_Key * const option__key = static_cast <MConfig_Option_Key*> (option);
	    key_elem = option__key->key;
	} break;
	default:
	    unreachable ();
    }

    Ref<String> key = keyToString (key_elem);

    logD (mconfig, _func, "option: ", key);

    Option *opts_option = section->getOption (key->mem());
    if (opts_option) {
	opts_option->removeValues ();
    } else {
	opts_option = new Option (key->mem());
	self->sections.getLast()->addOption (opts_option);
    }

    if (option__key_value) {
	MConfig_Value *value = option__key_value->value;
	for (;;) {
	    switch (value->value_type) {
		case MConfig_Value::_List: {
		    MConfig_Value_List * const value__list = static_cast <MConfig_Value_List*> (value);
		    logD (mconfig, _func, "value: ", wordsToString (&value__list->words));
		    opts_option->addValue (wordsToString (&value__list->words)->mem());
		    value = value__list->value;
		    logD (mconfig, _func, "new value: 0x", fmt_hex, (UintPtr) value);
		} break;
		case MConfig_Value::_Word: {
		    MConfig_Value_Word * const value__word = static_cast <MConfig_Value_Word*> (value);
		    logD (mconfig, _func, "value: ", wordsToString (&value__word->words));
		    opts_option->addValue (wordsToString (&value__word->words)->mem());
		    value = NULL;
		} break;
		default:
		    unreachable ();
	    }

	    if (!value)
		break;
	}
    }

    logD (mconfig, _func, "done");

    return true;
}

Result parseConfig (ConstMemory const &filename,
		    Config * const config)
{
    MyCpp::Ref<MyCpp::File> file;

try {
    MyCpp::Ref<Pargen::Grammar> const grammar = create_mconfig_grammar ();

    file = MyCpp::File::createDefault (filename,
				       0 /* open_flags */,
				       MyCpp::AccessMode::ReadOnly);
    file = MyCpp::grab (new MyCpp::CachedFile (file, (1 << 14) /* page_size */, 64 /* max_pages */));

    MyCpp::Ref<Scruffy::CppPreprocessor> const preprocessor = MyCpp::grab (new Scruffy::CppPreprocessor (file));
    preprocessor->performPreprocessing ();

    MyCpp::Ref< MyCpp::List_< MyCpp::Ref<Scruffy::PpItem>, MyCpp::SimplyReferenced > > pp_items =
	    preprocessor->getPpItems ();

    MyCpp::Ref<Scruffy::PpItemStream> pp_stream =
	    MyCpp::grab (static_cast <Scruffy::PpItemStream*> (
		    new Scruffy::ListPpItemStream (pp_items->first, MyLang::FilePosition ())));

    MyCpp::Ref<Scruffy::PpItemStreamTokenStream> token_stream =
	    MyCpp::grab (new Scruffy::PpItemStreamTokenStream (pp_stream));
    token_stream->setNewlineReplacement (";");

    ConfigParser config_parser (config);

    Pargen::ParserElement *mconfig_elem = NULL;
    Pargen::parse (token_stream,
		   &config_parser.checkpoint_tracker,
		   &config_parser,
		   grammar,
		   &mconfig_elem,
		   MyCpp::ConstMemoryDesc::forString ("default"),
		   // TODO Set 'upwards_jumps' to true for non-AST mode.
		   Pargen::createParserConfig (false /* upwards_jumps */),
		   false /* debug_dump */);

    file->close (true /* flush_data */);

    if (mconfig_elem == NULL ||
	token_stream->getNextToken ().getLength () > 0)
    {
	logE_ (_func, "Syntax error in configuration file ", filename);
	return Result::Failure;
    }

    if (logLevelOn (mconfig, LogLevel::Debug)) {
	dump_mconfig_grammar (static_cast <MConfig_Grammar*> (mconfig_elem));
	MyCpp::errf->pflush ();
    }

    return Result::Success;
} catch (...) {
    logE_ (_func, "Exception");

    if (file) {
	try {
	    file->close (true /* flush_data */);
	} catch (...) {
	}
    }

    return Result::Failure;
}
}

}

<|endoftext|>


<commit_before>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This webpage shows layout of YV12 and other YUV formats
// http://www.fourcc.org/yuv.php
// The actual conversion is best described here
// http://en.wikipedia.org/wiki/YUV
// An article on optimizing YUV conversion using tables instead of multiplies
// http://lestourtereaux.free.fr/papers/data/yuvrgb.pdf
//
// YV12 is a full plane of Y and a half height, half width chroma planes
// YV16 is a full plane of Y and a full height, half width chroma planes
//
// ARGB pixel format is output, which on little endian is stored as BGRA.
// The alpha is set to 255, allowing the application to use RGBA or RGB32.

#include "media/base/yuv_convert.h"

#include "base/logging.h"
#include "build/build_config.h"
#include "media/base/cpu_features.h"
#include "media/base/simd/convert_rgb_to_yuv.h"
#include "media/base/simd/convert_yuv_to_rgb.h"
#include "media/base/simd/filter_yuv.h"
#include "media/base/yuv_convert_internal.h"
#include "media/base/yuv_row.h"

#if defined(ARCH_CPU_X86_FAMILY)
#if defined(_MSC_VER)
#include <intrin.h>
#else
#include <emmintrin.h>
#include <mmintrin.h>
#endif
#endif

namespace media {

static FilterYUVRowsProc ChooseFilterYUVRowsProc() {
#if defined(ARCH_CPU_X86_FAMILY)
  if (hasSSE2())
    return &FilterYUVRows_SSE2;
  if (hasMMX())
    return &FilterYUVRows_MMX;
#endif
  return &FilterYUVRows_C;
}

static ConvertYUVToRGB32RowProc ChooseConvertYUVToRGB32RowProc() {
#if defined(ARCH_CPU_X86_FAMILY)
  if (hasSSE())
    return &ConvertYUVToRGB32Row_SSE;
  if (hasMMX())
    return &ConvertYUVToRGB32Row_MMX;
#endif
  return &ConvertYUVToRGB32Row_C;
}

static ScaleYUVToRGB32RowProc ChooseScaleYUVToRGB32RowProc() {
#if defined(ARCH_CPU_X86_FAMILY)
#if defined(ARCH_CPU_X86_64)
  // Use 64-bits version if possible.
  return &ScaleYUVToRGB32Row_SSE2_X64;
#endif
  // Choose the best one on 32-bits system.
  if (hasSSE())
    return &ScaleYUVToRGB32Row_SSE;
  if (hasMMX())
    return &ScaleYUVToRGB32Row_MMX;
#endif
  return &ScaleYUVToRGB32Row_C;
}

static ScaleYUVToRGB32RowProc ChooseLinearScaleYUVToRGB32RowProc() {
#if defined(ARCH_CPU_X86_FAMILY)
#if defined(ARCH_CPU_X86_64)
  // Use 64-bits version if possible.
  return &LinearScaleYUVToRGB32Row_MMX_X64;
#endif
  // 32-bits system.
  if (hasSSE())
    return &LinearScaleYUVToRGB32Row_SSE;
  if (hasMMX())
    return &LinearScaleYUVToRGB32Row_MMX;
#endif
  return &LinearScaleYUVToRGB32Row_C;
}

// Empty SIMD registers state after using them.
void EmptyRegisterState() {
#if defined(ARCH_CPU_X86_FAMILY)
  static bool checked = false;
  static bool has_mmx = false;
  if (!checked) {
    has_mmx = hasMMX();
    checked = true;
  }
  if (has_mmx)
    _mm_empty();
#endif
}

// 16.16 fixed point arithmetic
const int kFractionBits = 16;
const int kFractionMax = 1 << kFractionBits;
const int kFractionMask = ((1 << kFractionBits) - 1);

// Scale a frame of YUV to 32 bit ARGB.
void ScaleYUVToRGB32(const uint8* y_buf,
                     const uint8* u_buf,
                     const uint8* v_buf,
                     uint8* rgb_buf,
                     int source_width,
                     int source_height,
                     int width,
                     int height,
                     int y_pitch,
                     int uv_pitch,
                     int rgb_pitch,
                     YUVType yuv_type,
                     Rotate view_rotate,
                     ScaleFilter filter) {
  static FilterYUVRowsProc filter_proc = NULL;
  static ConvertYUVToRGB32RowProc convert_proc = NULL;
  static ScaleYUVToRGB32RowProc scale_proc = NULL;
  static ScaleYUVToRGB32RowProc linear_scale_proc = NULL;

  if (!filter_proc)
    filter_proc = ChooseFilterYUVRowsProc();
  if (!convert_proc)
    convert_proc = ChooseConvertYUVToRGB32RowProc();
  if (!scale_proc)
    scale_proc = ChooseScaleYUVToRGB32RowProc();
  if (!linear_scale_proc)
    linear_scale_proc = ChooseLinearScaleYUVToRGB32RowProc();

  // Handle zero sized sources and destinations.
  if ((yuv_type == YV12 && (source_width < 2 || source_height < 2)) ||
      (yuv_type == YV16 && (source_width < 2 || source_height < 1)) ||
      width == 0 || height == 0)
    return;

  // 4096 allows 3 buffers to fit in 12k.
  // Helps performance on CPU with 16K L1 cache.
  // Large enough for 3830x2160 and 30" displays which are 2560x1600.
  const int kFilterBufferSize = 4096;
  // Disable filtering if the screen is too big (to avoid buffer overflows).
  // This should never happen to regular users: they don't have monitors
  // wider than 4096 pixels.
  // TODO(fbarchard): Allow rotated videos to filter.
  if (source_width > kFilterBufferSize || view_rotate)
    filter = FILTER_NONE;

  unsigned int y_shift = yuv_type;
  // Diagram showing origin and direction of source sampling.
  // ->0   4<-
  // 7       3
  //
  // 6       5
  // ->1   2<-
  // Rotations that start at right side of image.
  if ((view_rotate == ROTATE_180) ||
      (view_rotate == ROTATE_270) ||
      (view_rotate == MIRROR_ROTATE_0) ||
      (view_rotate == MIRROR_ROTATE_90)) {
    y_buf += source_width - 1;
    u_buf += source_width / 2 - 1;
    v_buf += source_width / 2 - 1;
    source_width = -source_width;
  }
  // Rotations that start at bottom of image.
  if ((view_rotate == ROTATE_90) ||
      (view_rotate == ROTATE_180) ||
      (view_rotate == MIRROR_ROTATE_90) ||
      (view_rotate == MIRROR_ROTATE_180)) {
    y_buf += (source_height - 1) * y_pitch;
    u_buf += ((source_height >> y_shift) - 1) * uv_pitch;
    v_buf += ((source_height >> y_shift) - 1) * uv_pitch;
    source_height = -source_height;
  }

  int source_dx = source_width * kFractionMax / width;
  int source_dy = source_height * kFractionMax / height;

  if ((view_rotate == ROTATE_90) ||
      (view_rotate == ROTATE_270)) {
    int tmp = height;
    height = width;
    width = tmp;
    tmp = source_height;
    source_height = source_width;
    source_width = tmp;
    int original_dx = source_dx;
    int original_dy = source_dy;
    source_dx = ((original_dy >> kFractionBits) * y_pitch) << kFractionBits;
    source_dy = original_dx;
    if (view_rotate == ROTATE_90) {
      y_pitch = -1;
      uv_pitch = -1;
      source_height = -source_height;
    } else {
      y_pitch = 1;
      uv_pitch = 1;
    }
  }

  // Need padding because FilterRows() will write 1 to 16 extra pixels
  // after the end for SSE2 version.
  uint8 yuvbuf[16 + kFilterBufferSize * 3 + 16];
  uint8* ybuf =
      reinterpret_cast<uint8*>(reinterpret_cast<uintptr_t>(yuvbuf + 15) & ~15);
  uint8* ubuf = ybuf + kFilterBufferSize;
  uint8* vbuf = ubuf + kFilterBufferSize;
  // TODO(fbarchard): Fixed point math is off by 1 on negatives.
  int yscale_fixed = (source_height << kFractionBits) / height;

  // TODO(fbarchard): Split this into separate function for better efficiency.
  for (int y = 0; y < height; ++y) {
    uint8* dest_pixel = rgb_buf + y * rgb_pitch;
    int source_y_subpixel = (y * yscale_fixed);
    if (yscale_fixed >= (kFractionMax * 2)) {
      source_y_subpixel += kFractionMax / 2;  // For 1/2 or less, center filter.
    }
    int source_y = source_y_subpixel >> kFractionBits;

    const uint8* y0_ptr = y_buf + source_y * y_pitch;
    const uint8* y1_ptr = y0_ptr + y_pitch;

    const uint8* u0_ptr = u_buf + (source_y >> y_shift) * uv_pitch;
    const uint8* u1_ptr = u0_ptr + uv_pitch;
    const uint8* v0_ptr = v_buf + (source_y >> y_shift) * uv_pitch;
    const uint8* v1_ptr = v0_ptr + uv_pitch;

    // vertical scaler uses 16.8 fixed point
    int source_y_fraction = (source_y_subpixel & kFractionMask) >> 8;
    int source_uv_fraction =
        ((source_y_subpixel >> y_shift) & kFractionMask) >> 8;

    const uint8* y_ptr = y0_ptr;
    const uint8* u_ptr = u0_ptr;
    const uint8* v_ptr = v0_ptr;
    // Apply vertical filtering if necessary.
    // TODO(fbarchard): Remove memcpy when not necessary.
    if (filter & media::FILTER_BILINEAR_V) {
      if (yscale_fixed != kFractionMax &&
          source_y_fraction && ((source_y + 1) < source_height)) {
        filter_proc(ybuf, y0_ptr, y1_ptr, source_width, source_y_fraction);
      } else {
        memcpy(ybuf, y0_ptr, source_width);
      }
      y_ptr = ybuf;
      ybuf[source_width] = ybuf[source_width-1];
      int uv_source_width = (source_width + 1) / 2;
      if (yscale_fixed != kFractionMax &&
          source_uv_fraction &&
          (((source_y >> y_shift) + 1) < (source_height >> y_shift))) {
        filter_proc(ubuf, u0_ptr, u1_ptr, uv_source_width, source_uv_fraction);
        filter_proc(vbuf, v0_ptr, v1_ptr, uv_source_width, source_uv_fraction);
      } else {
        memcpy(ubuf, u0_ptr, uv_source_width);
        memcpy(vbuf, v0_ptr, uv_source_width);
      }
      u_ptr = ubuf;
      v_ptr = vbuf;
      ubuf[uv_source_width] = ubuf[uv_source_width - 1];
      vbuf[uv_source_width] = vbuf[uv_source_width - 1];
    }
    if (source_dx == kFractionMax) {  // Not scaled
      convert_proc(y_ptr, u_ptr, v_ptr, dest_pixel, width);
    } else {
      if (filter & FILTER_BILINEAR_H) {
        linear_scale_proc(y_ptr, u_ptr, v_ptr, dest_pixel, width, source_dx);
      } else {
        scale_proc(y_ptr, u_ptr, v_ptr, dest_pixel, width, source_dx);
      }
    }
  }

  EmptyRegisterState();
}

void ConvertRGB32ToYUV(const uint8* rgbframe,
                       uint8* yplane,
                       uint8* uplane,
                       uint8* vplane,
                       int width,
                       int height,
                       int rgbstride,
                       int ystride,
                       int uvstride) {
  static void (*convert_proc)(const uint8*, uint8*, uint8*, uint8*,
                              int, int, int, int, int) = NULL;
  if (!convert_proc) {
#if defined(ARCH_CPU_ARM_FAMILY)
    // For ARM processors, always use C version.
    // TODO(hclam): Implement a NEON version.
    convert_proc = &ConvertRGB32ToYUV_C;
#else
    // For x86 processors, check if SSSE3 (or SSE2) is supported.
    if (hasSSSE3())
      convert_proc = &ConvertRGB32ToYUV_SSSE3;
    else if (hasSSE2())
      convert_proc = &ConvertRGB32ToYUV_SSE2;
    else
      convert_proc = &ConvertRGB32ToYUV_C;
#endif
  }

  convert_proc(rgbframe, yplane, uplane, vplane, width, height,
               rgbstride, ystride, uvstride);
}

void ConvertRGB24ToYUV(const uint8* rgbframe,
                       uint8* yplane,
                       uint8* uplane,
                       uint8* vplane,
                       int width,
                       int height,
                       int rgbstride,
                       int ystride,
                       int uvstride) {
#if defined(ARCH_CPU_ARM_FAMILY)
  ConvertRGB24ToYUV_C(rgbframe, yplane, uplane, vplane, width, height,
                      rgbstride, ystride, uvstride);
#else
  static void (*convert_proc)(const uint8*, uint8*, uint8*, uint8*,
                              int, int, int, int, int) = NULL;
  if (!convert_proc) {
    if (hasSSSE3())
      convert_proc = &ConvertRGB24ToYUV_SSSE3;
    else
      convert_proc = &ConvertRGB24ToYUV_C;
  }
  convert_proc(rgbframe, yplane, uplane, vplane, width, height,
               rgbstride, ystride, uvstride);
#endif
}

void ConvertYUY2ToYUV(const uint8* src,
                      uint8* yplane,
                      uint8* uplane,
                      uint8* vplane,
                      int width,
                      int height) {
  ConvertYUY2ToYUV_C(src, yplane, uplane, vplane, width, height);
}

void ConvertYUVToRGB32(const uint8* yplane,
                       const uint8* uplane,
                       const uint8* vplane,
                       uint8* rgbframe,
                       int width,
                       int height,
                       int ystride,
                       int uvstride,
                       int rgbstride,
                       YUVType yuv_type) {
#if defined(ARCH_CPU_ARM_FAMILY)
  ConvertYUVToRGB32_C(yplane, uplane, vplane, rgbframe,
                      width, height, ystride, uvstride, rgbstride, yuv_type);
#else
  static ConvertYUVToRGB32Proc convert_proc = NULL;
  if (!convert_proc) {
    if (hasSSE())
      convert_proc = &ConvertYUVToRGB32_SSE;
    else if (hasMMX())
      convert_proc = &ConvertYUVToRGB32_MMX;
    else
      convert_proc = &ConvertYUVToRGB32_C;
  }

  convert_proc(yplane, uplane, vplane, rgbframe,
               width, height, ystride, uvstride, rgbstride, yuv_type);
#endif
}

}  // namespace media
<commit_msg>Not use SSSE3 code that was merged accidentally<commit_after>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This webpage shows layout of YV12 and other YUV formats
// http://www.fourcc.org/yuv.php
// The actual conversion is best described here
// http://en.wikipedia.org/wiki/YUV
// An article on optimizing YUV conversion using tables instead of multiplies
// http://lestourtereaux.free.fr/papers/data/yuvrgb.pdf
//
// YV12 is a full plane of Y and a half height, half width chroma planes
// YV16 is a full plane of Y and a full height, half width chroma planes
//
// ARGB pixel format is output, which on little endian is stored as BGRA.
// The alpha is set to 255, allowing the application to use RGBA or RGB32.

#include "media/base/yuv_convert.h"

#include "base/logging.h"
#include "build/build_config.h"
#include "media/base/cpu_features.h"
#include "media/base/simd/convert_rgb_to_yuv.h"
#include "media/base/simd/convert_yuv_to_rgb.h"
#include "media/base/simd/filter_yuv.h"
#include "media/base/yuv_convert_internal.h"
#include "media/base/yuv_row.h"

#if defined(ARCH_CPU_X86_FAMILY)
#if defined(_MSC_VER)
#include <intrin.h>
#else
#include <emmintrin.h>
#include <mmintrin.h>
#endif
#endif

namespace media {

static FilterYUVRowsProc ChooseFilterYUVRowsProc() {
#if defined(ARCH_CPU_X86_FAMILY)
  if (hasSSE2())
    return &FilterYUVRows_SSE2;
  if (hasMMX())
    return &FilterYUVRows_MMX;
#endif
  return &FilterYUVRows_C;
}

static ConvertYUVToRGB32RowProc ChooseConvertYUVToRGB32RowProc() {
#if defined(ARCH_CPU_X86_FAMILY)
  if (hasSSE())
    return &ConvertYUVToRGB32Row_SSE;
  if (hasMMX())
    return &ConvertYUVToRGB32Row_MMX;
#endif
  return &ConvertYUVToRGB32Row_C;
}

static ScaleYUVToRGB32RowProc ChooseScaleYUVToRGB32RowProc() {
#if defined(ARCH_CPU_X86_FAMILY)
#if defined(ARCH_CPU_X86_64)
  // Use 64-bits version if possible.
  return &ScaleYUVToRGB32Row_SSE2_X64;
#endif
  // Choose the best one on 32-bits system.
  if (hasSSE())
    return &ScaleYUVToRGB32Row_SSE;
  if (hasMMX())
    return &ScaleYUVToRGB32Row_MMX;
#endif
  return &ScaleYUVToRGB32Row_C;
}

static ScaleYUVToRGB32RowProc ChooseLinearScaleYUVToRGB32RowProc() {
#if defined(ARCH_CPU_X86_FAMILY)
#if defined(ARCH_CPU_X86_64)
  // Use 64-bits version if possible.
  return &LinearScaleYUVToRGB32Row_MMX_X64;
#endif
  // 32-bits system.
  if (hasSSE())
    return &LinearScaleYUVToRGB32Row_SSE;
  if (hasMMX())
    return &LinearScaleYUVToRGB32Row_MMX;
#endif
  return &LinearScaleYUVToRGB32Row_C;
}

// Empty SIMD registers state after using them.
void EmptyRegisterState() {
#if defined(ARCH_CPU_X86_FAMILY)
  static bool checked = false;
  static bool has_mmx = false;
  if (!checked) {
    has_mmx = hasMMX();
    checked = true;
  }
  if (has_mmx)
    _mm_empty();
#endif
}

// 16.16 fixed point arithmetic
const int kFractionBits = 16;
const int kFractionMax = 1 << kFractionBits;
const int kFractionMask = ((1 << kFractionBits) - 1);

// Scale a frame of YUV to 32 bit ARGB.
void ScaleYUVToRGB32(const uint8* y_buf,
                     const uint8* u_buf,
                     const uint8* v_buf,
                     uint8* rgb_buf,
                     int source_width,
                     int source_height,
                     int width,
                     int height,
                     int y_pitch,
                     int uv_pitch,
                     int rgb_pitch,
                     YUVType yuv_type,
                     Rotate view_rotate,
                     ScaleFilter filter) {
  static FilterYUVRowsProc filter_proc = NULL;
  static ConvertYUVToRGB32RowProc convert_proc = NULL;
  static ScaleYUVToRGB32RowProc scale_proc = NULL;
  static ScaleYUVToRGB32RowProc linear_scale_proc = NULL;

  if (!filter_proc)
    filter_proc = ChooseFilterYUVRowsProc();
  if (!convert_proc)
    convert_proc = ChooseConvertYUVToRGB32RowProc();
  if (!scale_proc)
    scale_proc = ChooseScaleYUVToRGB32RowProc();
  if (!linear_scale_proc)
    linear_scale_proc = ChooseLinearScaleYUVToRGB32RowProc();

  // Handle zero sized sources and destinations.
  if ((yuv_type == YV12 && (source_width < 2 || source_height < 2)) ||
      (yuv_type == YV16 && (source_width < 2 || source_height < 1)) ||
      width == 0 || height == 0)
    return;

  // 4096 allows 3 buffers to fit in 12k.
  // Helps performance on CPU with 16K L1 cache.
  // Large enough for 3830x2160 and 30" displays which are 2560x1600.
  const int kFilterBufferSize = 4096;
  // Disable filtering if the screen is too big (to avoid buffer overflows).
  // This should never happen to regular users: they don't have monitors
  // wider than 4096 pixels.
  // TODO(fbarchard): Allow rotated videos to filter.
  if (source_width > kFilterBufferSize || view_rotate)
    filter = FILTER_NONE;

  unsigned int y_shift = yuv_type;
  // Diagram showing origin and direction of source sampling.
  // ->0   4<-
  // 7       3
  //
  // 6       5
  // ->1   2<-
  // Rotations that start at right side of image.
  if ((view_rotate == ROTATE_180) ||
      (view_rotate == ROTATE_270) ||
      (view_rotate == MIRROR_ROTATE_0) ||
      (view_rotate == MIRROR_ROTATE_90)) {
    y_buf += source_width - 1;
    u_buf += source_width / 2 - 1;
    v_buf += source_width / 2 - 1;
    source_width = -source_width;
  }
  // Rotations that start at bottom of image.
  if ((view_rotate == ROTATE_90) ||
      (view_rotate == ROTATE_180) ||
      (view_rotate == MIRROR_ROTATE_90) ||
      (view_rotate == MIRROR_ROTATE_180)) {
    y_buf += (source_height - 1) * y_pitch;
    u_buf += ((source_height >> y_shift) - 1) * uv_pitch;
    v_buf += ((source_height >> y_shift) - 1) * uv_pitch;
    source_height = -source_height;
  }

  int source_dx = source_width * kFractionMax / width;
  int source_dy = source_height * kFractionMax / height;

  if ((view_rotate == ROTATE_90) ||
      (view_rotate == ROTATE_270)) {
    int tmp = height;
    height = width;
    width = tmp;
    tmp = source_height;
    source_height = source_width;
    source_width = tmp;
    int original_dx = source_dx;
    int original_dy = source_dy;
    source_dx = ((original_dy >> kFractionBits) * y_pitch) << kFractionBits;
    source_dy = original_dx;
    if (view_rotate == ROTATE_90) {
      y_pitch = -1;
      uv_pitch = -1;
      source_height = -source_height;
    } else {
      y_pitch = 1;
      uv_pitch = 1;
    }
  }

  // Need padding because FilterRows() will write 1 to 16 extra pixels
  // after the end for SSE2 version.
  uint8 yuvbuf[16 + kFilterBufferSize * 3 + 16];
  uint8* ybuf =
      reinterpret_cast<uint8*>(reinterpret_cast<uintptr_t>(yuvbuf + 15) & ~15);
  uint8* ubuf = ybuf + kFilterBufferSize;
  uint8* vbuf = ubuf + kFilterBufferSize;
  // TODO(fbarchard): Fixed point math is off by 1 on negatives.
  int yscale_fixed = (source_height << kFractionBits) / height;

  // TODO(fbarchard): Split this into separate function for better efficiency.
  for (int y = 0; y < height; ++y) {
    uint8* dest_pixel = rgb_buf + y * rgb_pitch;
    int source_y_subpixel = (y * yscale_fixed);
    if (yscale_fixed >= (kFractionMax * 2)) {
      source_y_subpixel += kFractionMax / 2;  // For 1/2 or less, center filter.
    }
    int source_y = source_y_subpixel >> kFractionBits;

    const uint8* y0_ptr = y_buf + source_y * y_pitch;
    const uint8* y1_ptr = y0_ptr + y_pitch;

    const uint8* u0_ptr = u_buf + (source_y >> y_shift) * uv_pitch;
    const uint8* u1_ptr = u0_ptr + uv_pitch;
    const uint8* v0_ptr = v_buf + (source_y >> y_shift) * uv_pitch;
    const uint8* v1_ptr = v0_ptr + uv_pitch;

    // vertical scaler uses 16.8 fixed point
    int source_y_fraction = (source_y_subpixel & kFractionMask) >> 8;
    int source_uv_fraction =
        ((source_y_subpixel >> y_shift) & kFractionMask) >> 8;

    const uint8* y_ptr = y0_ptr;
    const uint8* u_ptr = u0_ptr;
    const uint8* v_ptr = v0_ptr;
    // Apply vertical filtering if necessary.
    // TODO(fbarchard): Remove memcpy when not necessary.
    if (filter & media::FILTER_BILINEAR_V) {
      if (yscale_fixed != kFractionMax &&
          source_y_fraction && ((source_y + 1) < source_height)) {
        filter_proc(ybuf, y0_ptr, y1_ptr, source_width, source_y_fraction);
      } else {
        memcpy(ybuf, y0_ptr, source_width);
      }
      y_ptr = ybuf;
      ybuf[source_width] = ybuf[source_width-1];
      int uv_source_width = (source_width + 1) / 2;
      if (yscale_fixed != kFractionMax &&
          source_uv_fraction &&
          (((source_y >> y_shift) + 1) < (source_height >> y_shift))) {
        filter_proc(ubuf, u0_ptr, u1_ptr, uv_source_width, source_uv_fraction);
        filter_proc(vbuf, v0_ptr, v1_ptr, uv_source_width, source_uv_fraction);
      } else {
        memcpy(ubuf, u0_ptr, uv_source_width);
        memcpy(vbuf, v0_ptr, uv_source_width);
      }
      u_ptr = ubuf;
      v_ptr = vbuf;
      ubuf[uv_source_width] = ubuf[uv_source_width - 1];
      vbuf[uv_source_width] = vbuf[uv_source_width - 1];
    }
    if (source_dx == kFractionMax) {  // Not scaled
      convert_proc(y_ptr, u_ptr, v_ptr, dest_pixel, width);
    } else {
      if (filter & FILTER_BILINEAR_H) {
        linear_scale_proc(y_ptr, u_ptr, v_ptr, dest_pixel, width, source_dx);
      } else {
        scale_proc(y_ptr, u_ptr, v_ptr, dest_pixel, width, source_dx);
      }
    }
  }

  EmptyRegisterState();
}

void ConvertRGB32ToYUV(const uint8* rgbframe,
                       uint8* yplane,
                       uint8* uplane,
                       uint8* vplane,
                       int width,
                       int height,
                       int rgbstride,
                       int ystride,
                       int uvstride) {
  static void (*convert_proc)(const uint8*, uint8*, uint8*, uint8*,
                              int, int, int, int, int) = NULL;
  if (!convert_proc) {
#if defined(ARCH_CPU_ARM_FAMILY)
    // For ARM processors, always use C version.
    // TODO(hclam): Implement a NEON version.
    convert_proc = &ConvertRGB32ToYUV_C;
#else
    if (hasSSE2())
      convert_proc = &ConvertRGB32ToYUV_SSE2;
    else
      convert_proc = &ConvertRGB32ToYUV_C;
#endif
  }

  convert_proc(rgbframe, yplane, uplane, vplane, width, height,
               rgbstride, ystride, uvstride);
}

void ConvertRGB24ToYUV(const uint8* rgbframe,
                       uint8* yplane,
                       uint8* uplane,
                       uint8* vplane,
                       int width,
                       int height,
                       int rgbstride,
                       int ystride,
                       int uvstride) {
  ConvertRGB24ToYUV_C(rgbframe, yplane, uplane, vplane, width, height,
                      rgbstride, ystride, uvstride);
}

void ConvertYUY2ToYUV(const uint8* src,
                      uint8* yplane,
                      uint8* uplane,
                      uint8* vplane,
                      int width,
                      int height) {
  ConvertYUY2ToYUV_C(src, yplane, uplane, vplane, width, height);
}

void ConvertYUVToRGB32(const uint8* yplane,
                       const uint8* uplane,
                       const uint8* vplane,
                       uint8* rgbframe,
                       int width,
                       int height,
                       int ystride,
                       int uvstride,
                       int rgbstride,
                       YUVType yuv_type) {
#if defined(ARCH_CPU_ARM_FAMILY)
  ConvertYUVToRGB32_C(yplane, uplane, vplane, rgbframe,
                      width, height, ystride, uvstride, rgbstride, yuv_type);
#else
  static ConvertYUVToRGB32Proc convert_proc = NULL;
  if (!convert_proc) {
    if (hasSSE())
      convert_proc = &ConvertYUVToRGB32_SSE;
    else if (hasMMX())
      convert_proc = &ConvertYUVToRGB32_MMX;
    else
      convert_proc = &ConvertYUVToRGB32_C;
  }

  convert_proc(yplane, uplane, vplane, rgbframe,
               width, height, ystride, uvstride, rgbstride, yuv_type);
#endif
}

}  // namespace media
<|endoftext|>


<commit_before>/*
 * An example program that takes a GloVe
 * (http://nlp.stanford.edu/projects/glove/) dataset and builds a cross polytope
 * LSH table with the following property: for a random subset of NUM_QUERIES
 * points, we would like to find a nearest neighbor (w.r.t. cosine similarity)
 * with probability at least 0.9.
 *
 * You need to specify:
 *   - NUM_HASH_TABLES, which affects the memory usage, that larger it is, the
 *     better (unless it's too large). Despite that, it's usually a good idea
 *     to start with say 10 tables, and then increase it gradually, while
 *     observing the effect it makes.
 *   - NUM_HASH_BITS, that controls the number of buckets per table,
 *     usually it should be around the logarithm of the number of data points
 *   - NUM_ROTATIONS, which controls the number of pseudo-random rotations for
 *     the cross-polytope LSH, set it to 1 for the dense data, and 2 for the
 *     sparse data
 *
 * The code sets the number of probes automatically. Also, it recenters the
 * dataset for speeding-up hashing. Since after recentering vectors are not
 * unit anymore we should use the Euclidean distance in the data structure.
 */

#include <falconn/lsh_nn_table.h>

#include <algorithm>
#include <chrono>
#include <iostream>
#include <random>
#include <stdexcept>
#include <string>
#include <vector>

#include <cstdio>

using std::cerr;
using std::cout;
using std::endl;
using std::exception;
using std::max;
using std::mt19937_64;
using std::runtime_error;
using std::string;
using std::uniform_int_distribution;
using std::vector;

using std::chrono::duration;
using std::chrono::duration_cast;
using std::chrono::high_resolution_clock;

using falconn::construct_table;
using falconn::compute_number_of_hash_functions;
using falconn::DenseVector;
using falconn::DistanceFunction;
using falconn::LSHConstructionParameters;
using falconn::LSHFamily;
using falconn::LSHNearestNeighborTable;

typedef DenseVector<float> Point;

const string FILE_NAME = "dataset/glove.840B.300d.dat";
const int NUM_QUERIES = 100;
const int SEED = 4057218;
const int NUM_HASH_TABLES = 10;
const int NUM_HASH_BITS = 20;
const int NUM_ROTATIONS = 1;

/*
 * An auxiliary function that reads a point from a binary file that is produced
 * by a script 'prepare-dataset.sh'
 */
bool read_point(FILE *file, Point *point) {
  int d;
  if (fread(&d, sizeof(int), 1, file) != 1) {
    return false;
  }
  float *buf = new float[d];
  if (fread(buf, sizeof(float), d, file) != (size_t)d) {
    throw runtime_error("can't read a point");
  }
  point->resize(d);
  for (int i = 0; i < d; ++i) {
    (*point)[i] = buf[i];
  }
  delete[] buf;
  return true;
}

/*
 * An auxiliary function that reads a dataset from a binary file that is
 * produced by a script 'prepare-dataset.sh'
 */
void read_dataset(string file_name, vector<Point> *dataset) {
  FILE *file = fopen(file_name.c_str(), "rb");
  if (!file) {
    throw runtime_error("can't open the file with the dataset");
  }
  Point p;
  dataset->clear();
  while (read_point(file, &p)) {
    dataset->push_back(p);
  }
  if (fclose(file)) {
    throw runtime_error("fclose() error");
  }
}

/*
 * Normalizes the dataset.
 */
void normalize(vector<Point> *dataset) {
  for (auto &p: *dataset) {
    p.normalize();
  }
}

/*
 * Chooses a random subset of the dataset to be the queries. The queries are
 * taken out of the dataset.
 */
void gen_queries(vector<Point> *dataset, vector<Point> *queries) {
  mt19937_64 gen(SEED);
  queries->clear();
  for (int i = 0; i < NUM_QUERIES; ++i) {
    uniform_int_distribution<> u(0, dataset->size() - 1);
    int ind = u(gen);
    queries->push_back((*dataset)[ind]);
    (*dataset)[ind] = dataset->back();
    dataset->pop_back();
  }
}

/*
 * Generates answers for the queries using the (optimized) linear scan.
 */
void gen_answers(const vector<Point> &dataset,
		 const vector<Point> &queries,
		 vector<int> *answers) {
  answers->resize(queries.size());
  int outer_counter = 0;
  for (const auto &query: queries) {
    float best = -10.0;
    int inner_counter = 0;
    for (const auto &datapoint: dataset) {
      float score = query.dot(datapoint);
      if (score > best) {
	(*answers)[outer_counter] = inner_counter;
	best = score;
      }
      ++inner_counter;
    }
    ++outer_counter;
  }
}

/*
 * Computes the probability of success using a given number of probes.
 */
double evaluate_num_probes(LSHNearestNeighborTable<Point> *table,
			   const vector<Point> &queries,
			   const vector<int> &answers,
			   int num_probes) {
  table->set_num_probes(num_probes);
  int outer_counter = 0;
  int num_matches = 0;
  vector<int32_t> candidates;
  for (const auto &query: queries) {
    table->get_candidates_with_duplicates(query, &candidates);
    for (auto x: candidates) {
      if (x == answers[outer_counter]) {
	++num_matches;
	break;
      }
    }
    ++outer_counter;
  }
  return (num_matches + 0.0) / (queries.size() + 0.0);
}

/*
 * Queries the data structure using a given number of probes.
 * It is much slower than 'evaluate_num_probes' and should be used to
 * measure the time.
 */
double evaluate_query_time(LSHNearestNeighborTable<Point> *table,
			   const vector<Point> &queries,
			   const vector<int> &answers,
			   int num_probes) {
  table->set_num_probes(num_probes);
  int outer_counter = 0;
  int num_matches = 0;
  for (const auto &query: queries) {
    if (table->find_closest(query) == answers[outer_counter]) {
      ++num_matches;
    }
    ++outer_counter;
  }
  return (num_matches + 0.0) / (queries.size() + 0.0);
}

/*
 * Finds the smallest number of probes that gives the probability of success
 * at least 0.9 using binary search.
 */
int find_num_probes(LSHNearestNeighborTable<Point> *table,
		    const vector<Point> &queries,
		    const vector<int> &answers,
		    int start_num_probes) {
  int num_probes = start_num_probes;
  for (;;) {
    cout << "trying " << num_probes << " probes" << endl;
    double precision =
      evaluate_num_probes(table, queries, answers, num_probes);
    if (precision >= 0.9) {
      break;
    }
    num_probes *= 2;
  }

  int r = num_probes;
  int l = r / 2;

  while (r - l > 1) {
    int num_probes = (l + r) / 2;
    cout << "trying " << num_probes << " probes" << endl;
    double precision =
      evaluate_num_probes(table, queries, answers, num_probes);
    if (precision >= 0.9) {
      r = num_probes;
    }
    else {
      l = num_probes;
    }
  }

  return r;
}

int main() {
  try {
    vector<Point> dataset, queries;
    vector<int> answers;

    // read the dataset
    cout << "reading points" << endl;
    read_dataset(FILE_NAME, &dataset);
    cout << dataset.size() << " points read" << endl;

    // normalize the data points
    cout << "normalizing points" << endl;
    normalize(&dataset);
    cout << "done" << endl;

    // find the center of mass
    Point center = dataset[0];
    for (size_t i = 1; i < dataset.size(); ++i) {
      center += dataset[i];
    }
    center /= dataset.size();

    // selecting NUM_QUERIES data points as queries
    cout << "selecting " << NUM_QUERIES << " queries" << endl;
    gen_queries(&dataset, &queries);
    cout << "done" << endl;

    // running the linear scan
    cout << "running linear scan (to generate nearest neighbors)" << endl;
    auto t1 = high_resolution_clock::now();
    gen_answers(dataset, queries, &answers);
    auto t2 = high_resolution_clock::now();
    double elapsed_time = duration_cast<duration<double>>(t2 - t1).count();
    cout << "done" << endl;
    cout << elapsed_time / queries.size() << " s per query" << endl;

    // re-centering the data to make it more isotropic
    cout << "re-centering" << endl;
    for (auto &datapoint: dataset) {
      datapoint -= center;
    }
    for (auto &query: queries) {
      query -= center;
    }
    cout << "done" << endl;

    // setting parameters and constructing the table
    LSHConstructionParameters params;
    params.dimension = dataset[0].size();
    params.lsh_family = LSHFamily::CrossPolytope;
    params.l = NUM_HASH_TABLES;
    params.distance_function = DistanceFunction::EuclideanSquared;
    compute_number_of_hash_functions<Point>(NUM_HASH_BITS, &params);
    params.num_rotations = NUM_ROTATIONS;
    cout << "building the index based on the cross-polytope LSH" << endl;
    t1 = high_resolution_clock::now();
    auto table = construct_table<Point>(dataset, params);
    t2 = high_resolution_clock::now();
    elapsed_time = duration_cast<duration<double>>(t2 - t1).count();
    cout << "done" << endl;
    cout << "construction time: " << elapsed_time << endl;

    // finding the number of probes via the binary search
    cout << "finding the appropriate number of probes" << endl;
    int num_probes = find_num_probes(&*table, queries, answers, params.l);
    cout << "done" << endl;
    cout << num_probes << " probes" << endl;

    // executing the queries using the found number of probes to gather
    // statistics
    table->reset_query_statistics();
    double score = evaluate_query_time(&*table, queries, answers, num_probes);
    auto statistics = table->get_query_statistics();
    cout << "average total query time: "
	 << statistics.average_total_query_time << endl;
    cout << "average lsh time: "
	 << statistics.average_lsh_time << endl;
    cout << "average hash table time: "
	 << statistics.average_hash_table_time << endl;
    cout << "average distance time: "
	 << statistics.average_distance_time << endl;
    cout << "average number of candidates: "
	 << statistics.average_num_candidates << endl;
    cout << "average number of unique candidates: "
	 << statistics.average_num_unique_candidates << endl;
    cout << "score: " << score << endl;
  }
  catch (runtime_error &e) {
    cerr << "Runtime error: " << e.what() << endl;
    return 1;
  }
  catch (exception &e) {
    cerr << "Exception: " << e.what() << endl;
    return 1;
  }
  catch (...) {
    cerr << "ERROR" << endl;
    return 1;
  }
  return 0;
}
<commit_msg>edited comments<commit_after>/*
 * An example program that takes a GloVe
 * (http://nlp.stanford.edu/projects/glove/) dataset and builds a cross-polytope
 * LSH table with the following goal in mind: for a random subset of NUM_QUERIES
 * points, we would like to find a nearest neighbor (w.r.t. cosine similarity)
 * with probability at least 0.9.
 *
 * You need to specify:
 *   - NUM_HASH_TABLES, which affects the memory usage: the larger it is, the
 *     better (unless it's too large). Despite that, it's usually a good idea
 *     to start with say 10 tables, and then increase it gradually, while
 *     observing the effect it makes.
 *   - NUM_HASH_BITS, that controls the number of buckets per table,
 *     usually it should be around the binary logarithm of the number of data
 *     points
 *   - NUM_ROTATIONS, which controls the number of pseudo-random rotations for
 *     the cross-polytope LSH, set it to 1 for the dense data, and 2 for the
 *     sparse data (for GloVe we set it to 1)
 *
 * The code sets the number of probes automatically. Also, it recenters the
 * dataset for speeding-up hashing. Since after recentering vectors are not
 * unit anymore we should use the Euclidean distance in the data structure.
 */

#include <falconn/lsh_nn_table.h>

#include <algorithm>
#include <chrono>
#include <iostream>
#include <random>
#include <stdexcept>
#include <string>
#include <vector>

#include <cstdio>

using std::cerr;
using std::cout;
using std::endl;
using std::exception;
using std::max;
using std::mt19937_64;
using std::runtime_error;
using std::string;
using std::uniform_int_distribution;
using std::vector;

using std::chrono::duration;
using std::chrono::duration_cast;
using std::chrono::high_resolution_clock;

using falconn::construct_table;
using falconn::compute_number_of_hash_functions;
using falconn::DenseVector;
using falconn::DistanceFunction;
using falconn::LSHConstructionParameters;
using falconn::LSHFamily;
using falconn::LSHNearestNeighborTable;

typedef DenseVector<float> Point;

const string FILE_NAME = "dataset/glove.840B.300d.dat";
const int NUM_QUERIES = 100;
const int SEED = 4057218;
const int NUM_HASH_TABLES = 10;
const int NUM_HASH_BITS = 20;
const int NUM_ROTATIONS = 1;

/*
 * An auxiliary function that reads a point from a binary file that is produced
 * by a script 'prepare-dataset.sh'
 */
bool read_point(FILE *file, Point *point) {
  int d;
  if (fread(&d, sizeof(int), 1, file) != 1) {
    return false;
  }
  float *buf = new float[d];
  if (fread(buf, sizeof(float), d, file) != (size_t)d) {
    throw runtime_error("can't read a point");
  }
  point->resize(d);
  for (int i = 0; i < d; ++i) {
    (*point)[i] = buf[i];
  }
  delete[] buf;
  return true;
}

/*
 * An auxiliary function that reads a dataset from a binary file that is
 * produced by a script 'prepare-dataset.sh'
 */
void read_dataset(string file_name, vector<Point> *dataset) {
  FILE *file = fopen(file_name.c_str(), "rb");
  if (!file) {
    throw runtime_error("can't open the file with the dataset");
  }
  Point p;
  dataset->clear();
  while (read_point(file, &p)) {
    dataset->push_back(p);
  }
  if (fclose(file)) {
    throw runtime_error("fclose() error");
  }
}

/*
 * Normalizes the dataset.
 */
void normalize(vector<Point> *dataset) {
  for (auto &p: *dataset) {
    p.normalize();
  }
}

/*
 * Chooses a random subset of the dataset to be the queries. The queries are
 * taken out of the dataset.
 */
void gen_queries(vector<Point> *dataset, vector<Point> *queries) {
  mt19937_64 gen(SEED);
  queries->clear();
  for (int i = 0; i < NUM_QUERIES; ++i) {
    uniform_int_distribution<> u(0, dataset->size() - 1);
    int ind = u(gen);
    queries->push_back((*dataset)[ind]);
    (*dataset)[ind] = dataset->back();
    dataset->pop_back();
  }
}

/*
 * Generates answers for the queries using the (optimized) linear scan.
 */
void gen_answers(const vector<Point> &dataset,
		 const vector<Point> &queries,
		 vector<int> *answers) {
  answers->resize(queries.size());
  int outer_counter = 0;
  for (const auto &query: queries) {
    float best = -10.0;
    int inner_counter = 0;
    for (const auto &datapoint: dataset) {
      float score = query.dot(datapoint);
      if (score > best) {
	(*answers)[outer_counter] = inner_counter;
	best = score;
      }
      ++inner_counter;
    }
    ++outer_counter;
  }
}

/*
 * Computes the probability of success using a given number of probes.
 */
double evaluate_num_probes(LSHNearestNeighborTable<Point> *table,
			   const vector<Point> &queries,
			   const vector<int> &answers,
			   int num_probes) {
  table->set_num_probes(num_probes);
  int outer_counter = 0;
  int num_matches = 0;
  vector<int32_t> candidates;
  for (const auto &query: queries) {
    table->get_candidates_with_duplicates(query, &candidates);
    for (auto x: candidates) {
      if (x == answers[outer_counter]) {
	++num_matches;
	break;
      }
    }
    ++outer_counter;
  }
  return (num_matches + 0.0) / (queries.size() + 0.0);
}

/*
 * Queries the data structure using a given number of probes.
 * It is much slower than 'evaluate_num_probes' and should be used to
 * measure the time.
 */
double evaluate_query_time(LSHNearestNeighborTable<Point> *table,
			   const vector<Point> &queries,
			   const vector<int> &answers,
			   int num_probes) {
  table->set_num_probes(num_probes);
  int outer_counter = 0;
  int num_matches = 0;
  for (const auto &query: queries) {
    if (table->find_closest(query) == answers[outer_counter]) {
      ++num_matches;
    }
    ++outer_counter;
  }
  return (num_matches + 0.0) / (queries.size() + 0.0);
}

/*
 * Finds the smallest number of probes that gives the probability of success
 * at least 0.9 using binary search.
 */
int find_num_probes(LSHNearestNeighborTable<Point> *table,
		    const vector<Point> &queries,
		    const vector<int> &answers,
		    int start_num_probes) {
  int num_probes = start_num_probes;
  for (;;) {
    cout << "trying " << num_probes << " probes" << endl;
    double precision =
      evaluate_num_probes(table, queries, answers, num_probes);
    if (precision >= 0.9) {
      break;
    }
    num_probes *= 2;
  }

  int r = num_probes;
  int l = r / 2;

  while (r - l > 1) {
    int num_probes = (l + r) / 2;
    cout << "trying " << num_probes << " probes" << endl;
    double precision =
      evaluate_num_probes(table, queries, answers, num_probes);
    if (precision >= 0.9) {
      r = num_probes;
    }
    else {
      l = num_probes;
    }
  }

  return r;
}

int main() {
  try {
    vector<Point> dataset, queries;
    vector<int> answers;

    // read the dataset
    cout << "reading points" << endl;
    read_dataset(FILE_NAME, &dataset);
    cout << dataset.size() << " points read" << endl;

    // normalize the data points
    cout << "normalizing points" << endl;
    normalize(&dataset);
    cout << "done" << endl;

    // find the center of mass
    Point center = dataset[0];
    for (size_t i = 1; i < dataset.size(); ++i) {
      center += dataset[i];
    }
    center /= dataset.size();

    // selecting NUM_QUERIES data points as queries
    cout << "selecting " << NUM_QUERIES << " queries" << endl;
    gen_queries(&dataset, &queries);
    cout << "done" << endl;

    // running the linear scan
    cout << "running linear scan (to generate nearest neighbors)" << endl;
    auto t1 = high_resolution_clock::now();
    gen_answers(dataset, queries, &answers);
    auto t2 = high_resolution_clock::now();
    double elapsed_time = duration_cast<duration<double>>(t2 - t1).count();
    cout << "done" << endl;
    cout << elapsed_time / queries.size() << " s per query" << endl;

    // re-centering the data to make it more isotropic
    cout << "re-centering" << endl;
    for (auto &datapoint: dataset) {
      datapoint -= center;
    }
    for (auto &query: queries) {
      query -= center;
    }
    cout << "done" << endl;

    // setting parameters and constructing the table
    LSHConstructionParameters params;
    params.dimension = dataset[0].size();
    params.lsh_family = LSHFamily::CrossPolytope;
    params.l = NUM_HASH_TABLES;
    params.distance_function = DistanceFunction::EuclideanSquared;
    compute_number_of_hash_functions<Point>(NUM_HASH_BITS, &params);
    params.num_rotations = NUM_ROTATIONS;
    cout << "building the index based on the cross-polytope LSH" << endl;
    t1 = high_resolution_clock::now();
    auto table = construct_table<Point>(dataset, params);
    t2 = high_resolution_clock::now();
    elapsed_time = duration_cast<duration<double>>(t2 - t1).count();
    cout << "done" << endl;
    cout << "construction time: " << elapsed_time << endl;

    // finding the number of probes via the binary search
    cout << "finding the appropriate number of probes" << endl;
    int num_probes = find_num_probes(&*table, queries, answers, params.l);
    cout << "done" << endl;
    cout << num_probes << " probes" << endl;

    // executing the queries using the found number of probes to gather
    // statistics
    table->reset_query_statistics();
    double score = evaluate_query_time(&*table, queries, answers, num_probes);
    auto statistics = table->get_query_statistics();
    cout << "average total query time: "
	 << statistics.average_total_query_time << endl;
    cout << "average lsh time: "
	 << statistics.average_lsh_time << endl;
    cout << "average hash table time: "
	 << statistics.average_hash_table_time << endl;
    cout << "average distance time: "
	 << statistics.average_distance_time << endl;
    cout << "average number of candidates: "
	 << statistics.average_num_candidates << endl;
    cout << "average number of unique candidates: "
	 << statistics.average_num_unique_candidates << endl;
    cout << "score: " << score << endl;
  }
  catch (runtime_error &e) {
    cerr << "Runtime error: " << e.what() << endl;
    return 1;
  }
  catch (exception &e) {
    cerr << "Exception: " << e.what() << endl;
    return 1;
  }
  catch (...) {
    cerr << "ERROR" << endl;
    return 1;
  }
  return 0;
}
<|endoftext|>


<commit_before><commit_msg>Update Miller Rabin.cpp<commit_after><|endoftext|>


<commit_before>// SockMsg.cpp : implementation file
//

#include "stdafx.h"
#include "LANTalk.h"
#include "SockMsg.h"
#include "LANTalkDlg.h"

// SockMsg

SockMsg::SockMsg()
{
}

SockMsg::~SockMsg()
{
}

unsigned int ip2int(CString mask)
{
	wchar_t * m1;
	m1 = mask.GetBuffer(mask.GetLength());
	unsigned int imask = 0;
	int len = mask.GetLength();
	int x[4][3];
	for (int m = 0; m < 4; m++)
	{
		for (int n = 0; n < 3; n++)
			x[m][n] = 0;
	}
	int mul[3] = { 1, 10, 100 };
	int i = 0, j = 0;
	for (int k = 0; k < len; k++)
	{
		if (m1[k] == L'.')
		{
			j = 0;
			i = i + 1;
		}
		else
		{
			for (int s = j; s >= 0; s--)
				x[i][s + 1] = x[i][s];
			x[i][0] = int(m1[k] - 48);
			j++;

		}
	}
	for (int m = 3; m >= 0; m--)
	{
		imask = imask << 8;
		int temp = 0;
		for (int n = 0; n < 3; n++)
			temp = temp + x[m][n] * mul[n];
		imask = imask + temp;

	}
	return imask;
}

CString int2ip(unsigned int imask)
{
	char sMask[16];
	memset(sMask, 0, 16);
	unsigned int b[4];
	b[0] = (imask >> 24) & 0x000000ff;
	b[1] = (imask >> 16) & 0x000000ff;
	b[2] = (imask >> 8) & 0x000000ff;
	b[3] = (imask) & 0x000000ff;

	int i = 0, k = 0;
	for (k = 0; k < 16; k++)
	{
		if (i > 3)
			break;
		sMask[k] = char(b[i] % 10 + 48);
		b[i] = b[i] / 10;
		if (b[i] == 0)
		{
			sMask[++k] = '.';
			i++;
		}
	}
	k--;
	sMask[k] = 0;

	char tmp;
	for (int j = 0; j <= (k - 1) / 2; j++)
	{
		tmp = sMask[j];
		sMask[j] = sMask[k - 1 - j];
		sMask[k - 1 - j] = tmp;
	}

	return CString(sMask);
}

// SockMsg member functions

void SockMsg::OnReceive(int nErrorCode)
{
	// TODO: Add your specialized code here and/or call the base class
	CLANTalkDlg * dlg = (CLANTalkDlg *)AfxGetMainWnd();
	UDP_Pack pack1;
	memset(&pack1, 0, sizeof(UDP_Pack));
	CString  sIP;
	UINT port = UDP_PORT;

	while (ReceiveFrom(&pack1, sizeof(pack1),sIP,port))
	{
		if (pack1.nCmd = SEND_ON)
		if (pack1.nCmd == SEND_ON)
		{
			StrInfo UserInfo;
			memcpy(&UserInfo, pack1.data, sizeof(UserInfo));
			CString sName(UserInfo.name);
			CString sPCName(UserInfo.pc);
			CString sMask(UserInfo.mask);
			CString sIP(UserInfo.ip);
			dlg->InsertUser(sName, sPCName, sIP, sMask);
			if (sIP != CString(theApp.sInfo.ip))
				theApp.ReplyHello(sIP);
			break;
		}
		if (pack1.nCmd = SEND_REPLY)
		else if (pack1.nCmd == SEND_REPLY)
		{
			StrInfo UserInfo;
			memcpy(&UserInfo, pack1.data, sizeof(UserInfo));
			CString sName(UserInfo.name);
			CString sPCName(UserInfo.pc);
			CString sMask(UserInfo.mask);
			CString sIP(UserInfo.ip);
			dlg->InsertUser(sName, sPCName, sIP, sMask);
			break;
		}
		if (pack1.nCmd == SEND_MSG)
		else if (pack1.nCmd == SEND_MSG)
		{
			wchar_t * wMsg = (wchar_t *)pack1.data;
			//UINT16 MsgLen = (UINT16)wMsg[0];
			CString csMsg(wMsg + 1);
			//dlg->InserMessage(sIP,csMsg);
		}
	}

	CAsyncSocket::OnReceive(nErrorCode);
}

<commit_msg>version 1.0.0<commit_after>// SockMsg.cpp : implementation file
//

#include "stdafx.h"
#include "LANTalk.h"
#include "SockMsg.h"
#include "LANTalkDlg.h"

// SockMsg

SockMsg::SockMsg()
{
}

SockMsg::~SockMsg()
{
}

unsigned int ip2int(CString mask)
{
	wchar_t * m1;
	m1 = mask.GetBuffer(mask.GetLength());
	unsigned int imask = 0;
	int len = mask.GetLength();
	int x[4][3];
	for (int m = 0; m < 4; m++)
	{
		for (int n = 0; n < 3; n++)
			x[m][n] = 0;
	}
	int mul[3] = { 1, 10, 100 };
	int i = 0, j = 0;
	for (int k = 0; k < len; k++)
	{
		if (m1[k] == L'.')
		{
			j = 0;
			i = i + 1;
		}
		else
		{
			for (int s = j; s >= 0; s--)
				x[i][s + 1] = x[i][s];
			x[i][0] = int(m1[k] - 48);
			j++;

		}
	}
	for (int m = 3; m >= 0; m--)
	{
		imask = imask << 8;
		int temp = 0;
		for (int n = 0; n < 3; n++)
			temp = temp + x[m][n] * mul[n];
		imask = imask + temp;

	}
	return imask;
}

CString int2ip(unsigned int imask)
{
	char sMask[16];
	memset(sMask, 0, 16);
	unsigned int b[4];
	b[0] = (imask >> 24) & 0x000000ff;
	b[1] = (imask >> 16) & 0x000000ff;
	b[2] = (imask >> 8) & 0x000000ff;
	b[3] = (imask) & 0x000000ff;

	int i = 0, k = 0;
	for (k = 0; k < 16; k++)
	{
		if (i > 3)
			break;
		sMask[k] = char(b[i] % 10 + 48);
		b[i] = b[i] / 10;
		if (b[i] == 0)
		{
			sMask[++k] = '.';
			i++;
		}
	}
	k--;
	sMask[k] = 0;

	char tmp;
	for (int j = 0; j <= (k - 1) / 2; j++)
	{
		tmp = sMask[j];
		sMask[j] = sMask[k - 1 - j];
		sMask[k - 1 - j] = tmp;
	}

	return CString(sMask);
}

// SockMsg member functions

void SockMsg::OnReceive(int nErrorCode)
{
	// TODO: Add your specialized code here and/or call the base class
	CLANTalkDlg * dlg = (CLANTalkDlg *)AfxGetMainWnd();
	UDP_Pack pack1;
	memset(&pack1, 0, sizeof(UDP_Pack));
	CString  sIP;
	UINT port = UDP_PORT;

	while (ReceiveFrom(&pack1, sizeof(pack1),sIP,port))
	{
		if (pack1.nCmd == SEND_ON)
		{
			StrInfo UserInfo;
			memcpy(&UserInfo, pack1.data, sizeof(UserInfo));
			CString sName(UserInfo.name);
			CString sPCName(UserInfo.pc);
			CString sMask(UserInfo.mask);
			CString sIP(UserInfo.ip);
			dlg->InsertUser(sName, sPCName, sIP, sMask);
			if (sIP != CString(theApp.sInfo.ip))
				theApp.ReplyHello(sIP);
			break;
		}
		else if (pack1.nCmd == SEND_REPLY)
		{
			StrInfo UserInfo;
			memcpy(&UserInfo, pack1.data, sizeof(UserInfo));
			CString sName(UserInfo.name);
			CString sPCName(UserInfo.pc);
			CString sMask(UserInfo.mask);
			CString sIP(UserInfo.ip);
			dlg->InsertUser(sName, sPCName, sIP, sMask);
			break;
		}
		else if (pack1.nCmd == SEND_MSG)
		{
			wchar_t * wMsg = (wchar_t *)pack1.data;
			//UINT16 MsgLen = (UINT16)wMsg[0];
			CString csMsg(wMsg + 1);
			//dlg->InserMessage(sIP,csMsg);
			dlg->GetChatDlg()->InsertRecMsg(sIP, csMsg);
			break;
		}
	}

	CAsyncSocket::OnReceive(nErrorCode);
}

<|endoftext|>