<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <typeinfo>
#include "compiler/normalizer/normalizer.h"
#include "context/static_context.h"
#include "compiler/expression/expr.h"
#include "functions/signature.h"
#include "types/root_typemanager.h"
#include "types/typeops.h"
#include "system/globalenv.h"
#include <iostream>

namespace zorba {

#define LOOKUP_FN( pfx, local, arity ) (sctx->lookup_fn (pfx, local, arity))
#define LOOKUP_OP1( local ) (m_sctx->lookup_builtin_fn (":" local, 1))

static inline expr::expr_t wrap_in_bev(static_context *sctx, expr::expr_t e)
{
  expr::expr_t fh(new fo_expr(e->get_loc(), LOOKUP_FN("fn", "boolean", 1)));
  fo_expr *fp = static_cast<fo_expr *>(fh.getp());
  fp->add(e);
  return fh;
}

static inline expr::expr_t wrap_in_atomization(static_context *sctx, expr::expr_t e)
{
  expr::expr_t fh(new fo_expr(e->get_loc(), LOOKUP_FN("fn", "data", 1)));
  fo_expr *fp = static_cast<fo_expr *>(fh.getp());
  fp->add(e);
  return fh;
}

static inline expr::expr_t wrap_in_typematch(expr::expr_t e, xqtref_t type)
{
  // treat_expr should be avoided for updating expressions too,
  // but in that case "type" will be item()* anyway
  return TypeOps::is_subtype(*GENV_TYPESYSTEM.ITEM_TYPE_STAR, *type)
    ? e
    : new treat_expr (e->get_loc (), e, type, XPTY0004);
}

static inline expr::expr_t wrap_in_type_conversion(expr::expr_t e, xqtref_t type)
{
  expr::expr_t ph = new promote_expr(e->get_loc(), e, type);
  // TODO : Need to add convert_simple_operand
  return ph;
}

static inline void checkNonUpdating(const expr* lExpr)
{
  if (lExpr != 0 && lExpr->is_updating())
    ZORBA_ERROR_LOC(XUST0001, lExpr->get_loc());
}

class normalizer : public expr_visitor
{
  static_context *m_sctx;

public:
  normalizer(CompilerCB* aCompilerCB) : m_sctx(aCompilerCB->m_sctx) { }
  ~normalizer() { }

#define DEF_VISIT_METHODS( e )                  \
  bool begin_visit (e& node) { return true; }   \
  void end_visit (e& node) {}

DEF_VISIT_METHODS (expr)

#ifdef ZORBA_DEBUGGER
DEF_VISIT_METHODS (debugger_expr)
#endif

DEF_VISIT_METHODS (wrapper_expr)

DEF_VISIT_METHODS (sequential_expr)

DEF_VISIT_METHODS (var_expr)

DEF_VISIT_METHODS (order_modifier)

DEF_VISIT_METHODS (eval_expr)

DEF_VISIT_METHODS (typeswitch_expr)

void wrap_flwor_wincond(static_context* sctx, flwor_wincond* c)
{
  if (c == NULL)
    return;

  c->set_cond(wrap_in_bev(sctx, c->get_cond()));
}

bool begin_visit(flwor_expr& node)
{
  return true;
}

void end_visit(flwor_expr& node)
{
  for (unsigned i = 0; i < node.num_clauses(); i++)
  {
    flwor_clause& c = *(node[i]);

    if (c.get_kind() == flwor_clause::where_clause)
    {
      where_clause* wc = static_cast<where_clause *>(&c);
      wc->set_where(wrap_in_bev(m_sctx, wc->get_where()));
    }
    else if (c.get_kind() == flwor_clause::order_clause)
    {
      orderby_clause* obc = static_cast<orderby_clause *>(&c);

      unsigned numColumns = obc->num_columns();

      for (unsigned i = 0; i < numColumns; ++i)
      {
        obc->set_column_expr(i, wrap_in_atomization(m_sctx, obc->get_column_expr(i)));
      }
    }
    else if (c.get_kind() == flwor_clause::for_clause)
    {
      for_clause* fc = static_cast<for_clause *> (&c);

      xqtref_t vartype = fc->get_var()->get_type();
      if (vartype != NULL)
      {
        if (TypeOps::is_empty(*vartype))
          ZORBA_ERROR_LOC_PARAM(XPTY0004, fc->get_loc(), "empty-sequence()", "");

        xqtref_t promote_type = m_sctx->get_typemanager()->
                                create_type(*vartype, TypeConstants::QUANT_STAR);

        expr_t e = fc->get_expr();
        fc->set_expr(wrap_in_typematch(e, promote_type));
      }
    }
    else if (c.get_kind() == flwor_clause::let_clause)
    {
      let_clause* lc = static_cast<let_clause *> (&c);

      xqtref_t vartype = lc->get_var()->get_type();
      if (vartype != NULL)
      {
        expr_t e = lc->get_expr();
        lc->set_expr(wrap_in_typematch(e, vartype));
      }
    }
    else if (c.get_kind() == flwor_clause::window_clause)
    {
      window_clause* wc = static_cast<window_clause *> (&c);

      xqtref_t vartype = wc->get_var()->get_type();
      if (vartype != NULL)
      {
        expr_t e = wc->get_expr();
        wc->set_expr(wrap_in_typematch(e, vartype));
      }

      wrap_flwor_wincond(m_sctx, wc->get_win_start());
      wrap_flwor_wincond(m_sctx, wc->get_win_stop());
    }
  }
}

void end_visit (case_clause&)
{
}

bool begin_visit (case_clause& node)
{
  checkNonUpdating(&*node.get_var_h());
  return true;
}

void end_visit (promote_expr&) {}
bool begin_visit (promote_expr& node)
{
  checkNonUpdating(&*node.get_input());
  return true;
}

void end_visit (trycatch_expr&) {}
bool begin_visit (trycatch_expr& node)
{
  checkNonUpdating(&*node.get_try_expr());
  return true;
}

void end_visit (if_expr&) {}
bool begin_visit (if_expr& node)
{
  node.set_cond_expr(wrap_in_bev(m_sctx, node.get_cond_expr()));
  return true;
}

void end_visit (fo_expr&) {}
bool begin_visit (fo_expr& node)
{
  const signature& sign = node.get_signature();

  int n = node.size();
  for(int i = 0; i < n; ++i) {
    expr::expr_t arg = node[i];
    if (!node.is_updating())
      checkNonUpdating(&*arg);
    const xqtref_t& arg_type = sign[i];
    xqtref_t arg_prime_type = TypeOps::prime_type(*arg_type);
    if (TypeOps::is_subtype(*arg_prime_type, *GENV_TYPESYSTEM.ANY_ATOMIC_TYPE_ONE)) {
      arg = wrap_in_atomization(m_sctx, arg);
      arg = wrap_in_type_conversion(arg, arg_type);
    }
    else arg = wrap_in_typematch(arg, arg_type);

    node[i] = arg;
  }

  return true;
}

void end_visit (instanceof_expr&) {}
bool begin_visit (instanceof_expr& node)
{
  checkNonUpdating(&*node.get_input());
  return true;
}

DEF_VISIT_METHODS (treat_expr)

void end_visit (castable_expr&)
{
}

bool begin_visit (castable_expr& node)
{
  checkNonUpdating(&*node.get_input());
  return true;
}

void end_visit (cast_expr&)
{
}

bool begin_visit (cast_expr& node)
{
  checkNonUpdating(&*node.get_input());

  expr_t arg = wrap_in_atomization(m_sctx, node.get_input());

  node.set_input(arg);

  return true;
}

void end_visit (name_cast_expr&)
{
}

bool begin_visit (name_cast_expr& node)
{
  checkNonUpdating(&*node.get_input());
  return true;
}

void end_visit (validate_expr&)
{
}

bool begin_visit (validate_expr& node)
{
  checkNonUpdating(&*node.get_expr());
  return true;
}

void end_visit (extension_expr&)
{
}

bool begin_visit (extension_expr& node)
{
  checkNonUpdating(&*node.get_expr());
  return true;
}

void end_visit (relpath_expr&)
{
}

bool begin_visit (relpath_expr& node)
{
  if (node.size() > 0) {
    expr_t ie = node[0];
    expr_t pe = new treat_expr(ie->get_loc(), ie, GENV_TYPESYSTEM.ANY_NODE_TYPE_STAR, XPTY0019);
    node[0] = pe;
  }
  return true;
}

DEF_VISIT_METHODS (axis_step_expr)
DEF_VISIT_METHODS (match_expr)
DEF_VISIT_METHODS (const_expr)
DEF_VISIT_METHODS (order_expr)

void end_visit (elem_expr&) {}
bool begin_visit (elem_expr& node)
{
  //node.setQNameExpr(wrap_in_atomization(m_sctx, node.getQNameExpr()));
  checkNonUpdating(&*node.getQNameExpr());
  checkNonUpdating(&*node.getAttrs());
  checkNonUpdating(&*node.getContent());
  return true;
}

void end_visit (doc_expr&) {}
bool begin_visit (doc_expr& node)
{
  checkNonUpdating(node.getContent());
  return true;
}

void end_visit (attr_expr&) {}
bool begin_visit (attr_expr& node)
{
  assert (node.getQNameExpr() != NULL);

  checkNonUpdating(&*node.getQNameExpr());
  checkNonUpdating(&*node.getValueExpr());

  // node.setQNameExpr(wrap_in_atomization(m_sctx, node.getQNameExpr()));

  if (node.getValueExpr() != NULL) {
    rchandle<expr> lExpr = node.getValueExpr();
    fo_expr* lFoExpr = 0;
    if ((lFoExpr = dynamic_cast<fo_expr*>(&*lExpr)))
    {
      function* lTestFunc = LOOKUP_OP1("enclosed-expr");
      if (lFoExpr->get_func() == lTestFunc)
      {
        (*lFoExpr)[0] = wrap_in_atomization(m_sctx, (*lFoExpr)[0].getp());
        return true;
      }
    }
    node.setValueExpr(wrap_in_atomization(m_sctx, node.getValueExpr()));
  }

  return true;
}

#undef LOOKUP_OP1

void end_visit (text_expr&) {}
bool begin_visit (text_expr& node)
{
  checkNonUpdating(&*node.get_text());
  return true;
}

void end_visit (pi_expr&) {}
bool begin_visit (pi_expr& node)
{
  checkNonUpdating(&*node.get_target_expr());
  return true;
}

DEF_VISIT_METHODS (insert_expr)
DEF_VISIT_METHODS (delete_expr)
DEF_VISIT_METHODS (rename_expr)
DEF_VISIT_METHODS (replace_expr)
DEF_VISIT_METHODS (transform_expr)
DEF_VISIT_METHODS (exit_expr)
DEF_VISIT_METHODS (while_expr)
DEF_VISIT_METHODS (flowctl_expr)
DEF_VISIT_METHODS (function_def_expr)

DEF_VISIT_METHODS (ft_select_expr)
DEF_VISIT_METHODS (ft_contains_expr)

};

void normalize_expr_tree(
    const char* norm_descr,
    CompilerCB* aCompilerCB,
    expr_t& root,
    const XQType* rType)
{
  normalizer n (aCompilerCB);

  root->accept(n);

  if (rType != NULL)
  {
    if (TypeOps::is_builtin_simple(*rType))
    {
      root = wrap_in_atomization(aCompilerCB->m_sctx, root);
      root = wrap_in_type_conversion(root, rType);
    }
    else
    {
      root = wrap_in_typematch (root, rType);
    }
  }

  if (aCompilerCB->m_config.normalize_cb)
    aCompilerCB->m_config.normalize_cb (&*root, norm_descr);
}

}

/* vim:set ts=2 sw=2: */
<commit_msg>cosmetic<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <typeinfo>
#include "compiler/normalizer/normalizer.h"
#include "context/static_context.h"
#include "compiler/expression/expr.h"
#include "functions/signature.h"
#include "types/root_typemanager.h"
#include "types/typeops.h"
#include "system/globalenv.h"
#include <iostream>

namespace zorba {

#define LOOKUP_FN( pfx, local, arity ) (sctx->lookup_fn (pfx, local, arity))
#define LOOKUP_OP1( local ) (m_sctx->lookup_builtin_fn (":" local, 1))

static inline expr::expr_t wrap_in_bev(static_context *sctx, expr::expr_t e)
{
  expr::expr_t fh(new fo_expr(e->get_loc(), LOOKUP_FN("fn", "boolean", 1)));
  fo_expr *fp = static_cast<fo_expr *>(fh.getp());
  fp->add(e);
  return fh;
}

static inline expr::expr_t wrap_in_atomization(static_context *sctx, expr::expr_t e)
{
  expr::expr_t fh(new fo_expr(e->get_loc(), LOOKUP_FN("fn", "data", 1)));
  fo_expr *fp = static_cast<fo_expr *>(fh.getp());
  fp->add(e);
  return fh;
}

static inline expr::expr_t wrap_in_typematch(expr::expr_t e, xqtref_t type)
{
  // treat_expr should be avoided for updating expressions too,
  // but in that case "type" will be item()* anyway
  return TypeOps::is_subtype(*GENV_TYPESYSTEM.ITEM_TYPE_STAR, *type)
    ? e
    : new treat_expr (e->get_loc (), e, type, XPTY0004);
}

static inline expr::expr_t wrap_in_type_conversion(expr::expr_t e, xqtref_t type)
{
  expr::expr_t ph = new promote_expr(e->get_loc(), e, type);
  // TODO : Need to add convert_simple_operand
  return ph;
}

static inline void checkNonUpdating(const expr* lExpr)
{
  if (lExpr != 0 && lExpr->is_updating())
    ZORBA_ERROR_LOC(XUST0001, lExpr->get_loc());
}

class normalizer : public expr_visitor
{
  static_context *m_sctx;

public:
  normalizer(CompilerCB* aCompilerCB) : m_sctx(aCompilerCB->m_sctx) { }
  ~normalizer() { }

#define DEF_VISIT_METHODS( e )                  \
  bool begin_visit (e& node) { return true; }   \
  void end_visit (e& node) {}

DEF_VISIT_METHODS (expr)

#ifdef ZORBA_DEBUGGER
DEF_VISIT_METHODS (debugger_expr)
#endif

DEF_VISIT_METHODS (wrapper_expr)

DEF_VISIT_METHODS (sequential_expr)

DEF_VISIT_METHODS (var_expr)

DEF_VISIT_METHODS (order_modifier)

DEF_VISIT_METHODS (eval_expr)

DEF_VISIT_METHODS (typeswitch_expr)

void wrap_flwor_wincond(static_context* sctx, flwor_wincond* c)
{
  if (c == NULL)
    return;

  c->set_cond(wrap_in_bev(sctx, c->get_cond()));
}

bool begin_visit(flwor_expr& node)
{
  return true;
}

void end_visit(flwor_expr& node)
{
  for (unsigned i = 0; i < node.num_clauses(); i++)
  {
    flwor_clause& c = *(node[i]);

    if (c.get_kind() == flwor_clause::where_clause)
    {
      where_clause* wc = static_cast<where_clause *>(&c);
      wc->set_where(wrap_in_bev(m_sctx, wc->get_where()));
    }
    else if (c.get_kind() == flwor_clause::order_clause)
    {
      orderby_clause* obc = static_cast<orderby_clause *>(&c);

      unsigned numColumns = obc->num_columns();

      for (unsigned i = 0; i < numColumns; ++i)
      {
        obc->set_column_expr(i, wrap_in_atomization(m_sctx, obc->get_column_expr(i)));
      }
    }
    else if (c.get_kind() == flwor_clause::for_clause)
    {
      for_clause* fc = static_cast<for_clause *> (&c);

      xqtref_t vartype = fc->get_var()->get_type();
      if (vartype != NULL)
      {
        if (TypeOps::is_empty(*vartype))
          ZORBA_ERROR_LOC_PARAM(XPTY0004, fc->get_loc(), "empty-sequence()", "");

        xqtref_t promote_type = m_sctx->get_typemanager()->
                                create_type(*vartype, TypeConstants::QUANT_STAR);

        expr_t e = fc->get_expr();
        fc->set_expr(wrap_in_typematch(e, promote_type));
      }
    }
    else if (c.get_kind() == flwor_clause::let_clause)
    {
      let_clause* lc = static_cast<let_clause *> (&c);

      xqtref_t vartype = lc->get_var()->get_type();
      if (vartype != NULL)
      {
        expr_t e = lc->get_expr();
        lc->set_expr(wrap_in_typematch(e, vartype));
      }
    }
    else if (c.get_kind() == flwor_clause::window_clause)
    {
      window_clause* wc = static_cast<window_clause *> (&c);

      xqtref_t vartype = wc->get_var()->get_type();
      if (vartype != NULL)
      {
        expr_t e = wc->get_expr();
        wc->set_expr(wrap_in_typematch(e, vartype));
      }

      wrap_flwor_wincond(m_sctx, wc->get_win_start());
      wrap_flwor_wincond(m_sctx, wc->get_win_stop());
    }
  }
}

void end_visit (case_clause&)
{
}

bool begin_visit (case_clause& node)
{
  checkNonUpdating(&*node.get_var_h());
  return true;
}

void end_visit (promote_expr&) {}
bool begin_visit (promote_expr& node)
{
  checkNonUpdating(&*node.get_input());
  return true;
}

void end_visit (trycatch_expr&) {}
bool begin_visit (trycatch_expr& node)
{
  checkNonUpdating(&*node.get_try_expr());
  return true;
}

void end_visit (if_expr&) {}
bool begin_visit (if_expr& node)
{
  node.set_cond_expr(wrap_in_bev(m_sctx, node.get_cond_expr()));
  return true;
}

void end_visit (fo_expr&)
{
}

bool begin_visit (fo_expr& node)
{
  const signature& sign = node.get_signature();

  int n = node.size();
  for(int i = 0; i < n; ++i)
  {
    expr::expr_t param = node[i];

    if (!node.is_updating())
      checkNonUpdating(&*param);

    const xqtref_t& param_type = sign[i];
    xqtref_t param_prime_type = TypeOps::prime_type(*param_type);

    if (TypeOps::is_subtype(*param_prime_type, *GENV_TYPESYSTEM.ANY_ATOMIC_TYPE_ONE))
    {
      param = wrap_in_atomization(m_sctx, param);
      param = wrap_in_type_conversion(param, param_type);
    }
    else
    {
      param = wrap_in_typematch(param, param_type);
    }

    node[i] = param;
  }

  return true;
}

void end_visit (instanceof_expr&) {}
bool begin_visit (instanceof_expr& node)
{
  checkNonUpdating(&*node.get_input());
  return true;
}

DEF_VISIT_METHODS (treat_expr)

void end_visit (castable_expr&)
{
}

bool begin_visit (castable_expr& node)
{
  checkNonUpdating(&*node.get_input());
  return true;
}

void end_visit (cast_expr&)
{
}

bool begin_visit (cast_expr& node)
{
  checkNonUpdating(&*node.get_input());

  expr_t arg = wrap_in_atomization(m_sctx, node.get_input());

  node.set_input(arg);

  return true;
}

void end_visit (name_cast_expr&)
{
}

bool begin_visit (name_cast_expr& node)
{
  checkNonUpdating(&*node.get_input());
  return true;
}

void end_visit (validate_expr&)
{
}

bool begin_visit (validate_expr& node)
{
  checkNonUpdating(&*node.get_expr());
  return true;
}

void end_visit (extension_expr&)
{
}

bool begin_visit (extension_expr& node)
{
  checkNonUpdating(&*node.get_expr());
  return true;
}

void end_visit (relpath_expr&)
{
}

bool begin_visit (relpath_expr& node)
{
  if (node.size() > 0) {
    expr_t ie = node[0];
    expr_t pe = new treat_expr(ie->get_loc(), ie, GENV_TYPESYSTEM.ANY_NODE_TYPE_STAR, XPTY0019);
    node[0] = pe;
  }
  return true;
}

DEF_VISIT_METHODS (axis_step_expr)
DEF_VISIT_METHODS (match_expr)
DEF_VISIT_METHODS (const_expr)
DEF_VISIT_METHODS (order_expr)

void end_visit (elem_expr&) {}
bool begin_visit (elem_expr& node)
{
  //node.setQNameExpr(wrap_in_atomization(m_sctx, node.getQNameExpr()));
  checkNonUpdating(&*node.getQNameExpr());
  checkNonUpdating(&*node.getAttrs());
  checkNonUpdating(&*node.getContent());
  return true;
}

void end_visit (doc_expr&) {}
bool begin_visit (doc_expr& node)
{
  checkNonUpdating(node.getContent());
  return true;
}

void end_visit (attr_expr&) {}
bool begin_visit (attr_expr& node)
{
  assert (node.getQNameExpr() != NULL);

  checkNonUpdating(&*node.getQNameExpr());
  checkNonUpdating(&*node.getValueExpr());

  // node.setQNameExpr(wrap_in_atomization(m_sctx, node.getQNameExpr()));

  if (node.getValueExpr() != NULL) {
    rchandle<expr> lExpr = node.getValueExpr();
    fo_expr* lFoExpr = 0;
    if ((lFoExpr = dynamic_cast<fo_expr*>(&*lExpr)))
    {
      function* lTestFunc = LOOKUP_OP1("enclosed-expr");
      if (lFoExpr->get_func() == lTestFunc)
      {
        (*lFoExpr)[0] = wrap_in_atomization(m_sctx, (*lFoExpr)[0].getp());
        return true;
      }
    }
    node.setValueExpr(wrap_in_atomization(m_sctx, node.getValueExpr()));
  }

  return true;
}

#undef LOOKUP_OP1

void end_visit (text_expr&) {}
bool begin_visit (text_expr& node)
{
  checkNonUpdating(&*node.get_text());
  return true;
}

void end_visit (pi_expr&) {}
bool begin_visit (pi_expr& node)
{
  checkNonUpdating(&*node.get_target_expr());
  return true;
}

DEF_VISIT_METHODS (insert_expr)
DEF_VISIT_METHODS (delete_expr)
DEF_VISIT_METHODS (rename_expr)
DEF_VISIT_METHODS (replace_expr)
DEF_VISIT_METHODS (transform_expr)
DEF_VISIT_METHODS (exit_expr)
DEF_VISIT_METHODS (while_expr)
DEF_VISIT_METHODS (flowctl_expr)
DEF_VISIT_METHODS (function_def_expr)

DEF_VISIT_METHODS (ft_select_expr)
DEF_VISIT_METHODS (ft_contains_expr)

};

void normalize_expr_tree(
    const char* norm_descr,
    CompilerCB* aCompilerCB,
    expr_t& root,
    const XQType* rType)
{
  normalizer n (aCompilerCB);

  root->accept(n);

  if (rType != NULL)
  {
    if (TypeOps::is_builtin_simple(*rType))
    {
      root = wrap_in_atomization(aCompilerCB->m_sctx, root);
      root = wrap_in_type_conversion(root, rType);
    }
    else
    {
      root = wrap_in_typematch (root, rType);
    }
  }

  if (aCompilerCB->m_config.normalize_cb)
    aCompilerCB->m_config.normalize_cb (&*root, norm_descr);
}

}

/* vim:set ts=2 sw=2: */
<|endoftext|>


<commit_before>#ifndef __CONCURRENCY_QUEUE_LIMITED_FIFO_HPP__
#define __CONCURRENCY_QUEUE_LIMITED_FIFO_HPP__

#include "concurrency/queue/passive_producer.hpp"
#include "concurrency/semaphore.hpp"
#include <list>

/* `limited_fifo_queue_t` is a first-in, first-out queue that has a limited depth. If the
consumer is not reading of the queue as fast as the producer is pushing things onto the
queue, then `push()` will start to block. */

template<class value_t, class queue_t = std::list<value_t> >
struct limited_fifo_queue_t : public passive_producer_t<value_t> {

    limited_fifo_queue_t(int depth) :
        passive_producer_t<value_t>(&available_var),
        semaphore(depth)
    {
        rassert(depth > 0);
    }

    void push(const value_t &value) {
        semaphore.co_lock();
        queue.push_back(value);
        available_var.set(!queue.empty());
    }

private:
    semaphore_t semaphore;
    watchable_var_t<bool> available_var;
    value_t produce_next_value() {
        value_t v = queue.front();
        queue.pop_front();
        semaphore.unlock();
        available_var.set(!queue.empty());
        return v;
    }
    queue_t queue;
    DISABLE_COPYING(limited_fifo_queue_t);
};

#endif /* __CONCURRENCY_QUEUE_LIMITED_FIFO_HPP__ */
<commit_msg>Fixed a thread bug in replication.<commit_after>#ifndef __CONCURRENCY_QUEUE_LIMITED_FIFO_HPP__
#define __CONCURRENCY_QUEUE_LIMITED_FIFO_HPP__

#include "concurrency/queue/passive_producer.hpp"
#include "concurrency/semaphore.hpp"
#include <list>

/* `limited_fifo_queue_t` is a first-in, first-out queue that has a limited depth. If the
consumer is not reading of the queue as fast as the producer is pushing things onto the
queue, then `push()` will start to block. */

template<class value_t, class queue_t = std::list<value_t> >
struct limited_fifo_queue_t :
    public home_thread_mixin_t,
    public passive_producer_t<value_t>
{
    limited_fifo_queue_t(int depth) :
        passive_producer_t<value_t>(&available_var),
        semaphore(depth)
    {
        rassert(depth > 0);
    }

    void push(const value_t &value) {
        on_thread_t thread_switcher(home_thread);
        semaphore.co_lock();
        queue.push_back(value);
        available_var.set(!queue.empty());
    }

private:
    semaphore_t semaphore;
    watchable_var_t<bool> available_var;
    value_t produce_next_value() {
        assert_thread();
        value_t v = queue.front();
        queue.pop_front();
        semaphore.unlock();
        available_var.set(!queue.empty());
        return v;
    }
    queue_t queue;
    DISABLE_COPYING(limited_fifo_queue_t);
};

#endif /* __CONCURRENCY_QUEUE_LIMITED_FIFO_HPP__ */
<|endoftext|>


<commit_before>/***************************************************************
 *
 * Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

/*
   This is some wisdom from Cygnus's web page.  If you just try to use
   the "stringify" operator on a preprocessor directive, you'd get
   "PLATFORM", not "Intel Linux" (or whatever the value of PLATFORM
   is).  That's because the stringify operator is a special case, and
   the preprocessor isn't allowed to expand things that are passed to
   it.  However, by defining two layers of macros, you get the right
   behavior, since the first pass converts:

   xstr(PLATFORM) -> str(Intel Linux)

   and the next pass gives:

   str(Intel Linux) -> "Intel Linux"

   This is exactly what we want, so we use it.  -Derek Wright and Jeff
   Ballard, 12/2/99

   Also, because the NT build system is totally different, we have to
   define the correct platform string right in here. :( -Derek 12/3/99
*/

#include "condor_common.h"

#define xstr(s) str(s)
#define str(s) #s

#if defined(WIN32)
#define PLATFORM INTEL-WINNT50
#endif

/* Via configure, one may have specified a particular buildid string to use
	in the version string. So honor that request here. */
#if defined(BUILDID)
#define BUILDIDSTR " BuildID: " xstr(BUILDID)
#else
#define BUILDIDSTR ""
#endif

/* Here is the version string - update before a public release */
/* --- IMPORTANT!  THE FORMAT OF THE VERSION STRING IS VERY STRICT
   BECAUSE IT IS PARSED AT RUNTIME AND COMPILE-TIME.  DO NOT ALTER THE
   FORMAT OR ENTER ANYTHING EXTRA BEFORE THE DATE.  IF YOU WISH TO ADD
   EXTRA INFORMATION, DO SO _AFTER_ THE BUILDIDSTR AND BEFORE THE TRAILING
   '$' CHARACTER.
   EXAMPLES:
       $CondorVersion: 6.9.5 " __DATE__ BUILDIDSTR " WinNTPreview $ [OK]
	   $CondorVersion: 6.9.5 WinNTPreview " __DATE__ BUILDIDSTR " $ [WRONG!!!]
   Any questions?  See Todd or Derek.  Note: if you mess it up, DaemonCore
   will EXCEPT at startup time.
*/

static char* CondorVersionString = "$CondorVersion: 7.4.3 " __DATE__ BUILDIDSTR " PRE-RELEASE $";

/* Here is the platform string.  You don't need to edit this */
static char* CondorPlatformString = "$CondorPlatform: " xstr(PLATFORM) " $";

extern "C" {

const char*
CondorVersion( void )
{
	return CondorVersionString;
}

const char*
CondorPlatform( void )
{
	return CondorPlatformString;
}

} /* extern "C" */

<commit_msg>set the version string for 7.4.2.<commit_after>/***************************************************************
 *
 * Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

/*
   This is some wisdom from Cygnus's web page.  If you just try to use
   the "stringify" operator on a preprocessor directive, you'd get
   "PLATFORM", not "Intel Linux" (or whatever the value of PLATFORM
   is).  That's because the stringify operator is a special case, and
   the preprocessor isn't allowed to expand things that are passed to
   it.  However, by defining two layers of macros, you get the right
   behavior, since the first pass converts:

   xstr(PLATFORM) -> str(Intel Linux)

   and the next pass gives:

   str(Intel Linux) -> "Intel Linux"

   This is exactly what we want, so we use it.  -Derek Wright and Jeff
   Ballard, 12/2/99

   Also, because the NT build system is totally different, we have to
   define the correct platform string right in here. :( -Derek 12/3/99
*/

#include "condor_common.h"

#define xstr(s) str(s)
#define str(s) #s

#if defined(WIN32)
#define PLATFORM INTEL-WINNT50
#endif

/* Via configure, one may have specified a particular buildid string to use
	in the version string. So honor that request here. */
#if defined(BUILDID)
#define BUILDIDSTR " BuildID: " xstr(BUILDID)
#else
#define BUILDIDSTR ""
#endif

/* Here is the version string - update before a public release */
/* --- IMPORTANT!  THE FORMAT OF THE VERSION STRING IS VERY STRICT
   BECAUSE IT IS PARSED AT RUNTIME AND COMPILE-TIME.  DO NOT ALTER THE
   FORMAT OR ENTER ANYTHING EXTRA BEFORE THE DATE.  IF YOU WISH TO ADD
   EXTRA INFORMATION, DO SO _AFTER_ THE BUILDIDSTR AND BEFORE THE TRAILING
   '$' CHARACTER.
   EXAMPLES:
       $CondorVersion: 6.9.5 " __DATE__ BUILDIDSTR " WinNTPreview $ [OK]
	   $CondorVersion: 6.9.5 WinNTPreview " __DATE__ BUILDIDSTR " $ [WRONG!!!]
   Any questions?  See Todd or Derek.  Note: if you mess it up, DaemonCore
   will EXCEPT at startup time.
*/

static char* CondorVersionString = "$CondorVersion: 7.4.2 " __DATE__ BUILDIDSTR " $";

/* Here is the platform string.  You don't need to edit this */
static char* CondorPlatformString = "$CondorPlatform: " xstr(PLATFORM) " $";

extern "C" {

const char*
CondorVersion( void )
{
	return CondorVersionString;
}

const char*
CondorPlatform( void )
{
	return CondorPlatformString;
}

} /* extern "C" */

<|endoftext|>


<commit_before>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <windows.h>
#include <shlwapi.h>

#include "base/command_line.h"
#include "base/debug/trace_event.h"
#include "base/environment.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/memory/scoped_ptr.h"
#include "base/win/registry.h"
#include "base/string_util.h"
#include "base/utf_string_conversions.h"
#include "base/version.h"
#include "chrome/app/breakpad_win.h"
#include "chrome/app/client_util.h"
#include "chrome/common/chrome_constants.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/installer/util/browser_distribution.h"
#include "chrome/installer/util/channel_info.h"
#include "chrome/installer/util/install_util.h"
#include "chrome/installer/util/google_update_constants.h"
#include "chrome/installer/util/google_update_settings.h"
#include "chrome/installer/util/util_constants.h"
#include "content/common/result_codes.h"

namespace {
// The entry point signature of chrome.dll.
typedef int (*DLL_MAIN)(HINSTANCE, sandbox::SandboxInterfaceInfo*, wchar_t*);

typedef void (*RelaunchChromeBrowserWithNewCommandLineIfNeededFunc)();

// Gets chrome version according to the load path. |exe_path| must be the
// backslash terminated directory of the current chrome.exe.
bool GetVersion(const wchar_t* exe_path, const wchar_t* key_path,
                std::wstring* version) {
  HKEY reg_root = InstallUtil::IsPerUserInstall(exe_path) ? HKEY_CURRENT_USER :
                                                            HKEY_LOCAL_MACHINE;
  bool success = false;

  base::win::RegKey key(reg_root, key_path, KEY_QUERY_VALUE);
  if (key.Valid()) {
    // If 'new_chrome.exe' is present it means chrome was auto-updated while
    // running. We need to consult the opv value so we can load the old dll.
    // TODO(cpu) : This is solving the same problem as the environment variable
    // so one of them will eventually be deprecated.
    std::wstring new_chrome_exe(exe_path);
    new_chrome_exe.append(installer::kChromeNewExe);
    if (::PathFileExistsW(new_chrome_exe.c_str()) &&
        key.ReadValue(google_update::kRegOldVersionField,
                      version) == ERROR_SUCCESS) {
      success = true;
    } else {
      success = (key.ReadValue(google_update::kRegVersionField,
                               version) == ERROR_SUCCESS);
    }
  }

  return success;
}

// Gets the path of the current exe with a trailing backslash.
std::wstring GetExecutablePath() {
  wchar_t path[MAX_PATH];
  ::GetModuleFileNameW(NULL, path, MAX_PATH);
  if (!::PathRemoveFileSpecW(path))
    return std::wstring();
  std::wstring exe_path(path);
  return exe_path.append(L"\\");
}

// Not generic, we only handle strings up to 128 chars.
bool EnvQueryStr(const wchar_t* key_name, std::wstring* value) {
  wchar_t out[128];
  DWORD count = sizeof(out)/sizeof(out[0]);
  DWORD rv = ::GetEnvironmentVariableW(key_name, out, count);
  if ((rv == 0) || (rv >= count))
    return false;
  *value = out;
  return true;
}

// Expects that |dir| has a trailing backslash. |dir| is modified so it
// contains the full path that was tried. Caller must check for the return
// value not being null to determine if this path contains a valid dll.
HMODULE LoadChromeWithDirectory(std::wstring* dir) {
  ::SetCurrentDirectoryW(dir->c_str());
  const CommandLine& cmd_line = *CommandLine::ForCurrentProcess();
#ifdef _WIN64
  if ((cmd_line.GetSwitchValueASCII(switches::kProcessType) ==
      switches::kNaClBrokerProcess) ||
      (cmd_line.GetSwitchValueASCII(switches::kProcessType) ==
      switches::kNaClLoaderProcess)) {
    // Load the 64-bit DLL when running in a 64-bit process.
    dir->append(installer::kChromeNaCl64Dll);
  } else {
    // Only NaCl broker and loader can be launched as Win64 processes.
    NOTREACHED();
    return NULL;
  }
#else
  dir->append(installer::kChromeDll);
#endif

#ifdef NDEBUG
  // Experimental pre-reading optimization
  // The idea is to pre read significant portion of chrome.dll in advance
  // so that subsequent hard page faults are avoided.
  if (!cmd_line.HasSwitch(switches::kProcessType)) {
    // The kernel brings in 8 pages for the code section at a time and 4 pages
    // for other sections. We can skip over these pages to avoid a soft page
    // fault which may not occur during code execution. However skipping 4K at
    // a time still has better performance over 32K and 16K according to data.
    // TODO(ananta)
    // Investigate this and tune.
    const size_t kStepSize = 4 * 1024;

    DWORD pre_read_size = 0;
    DWORD pre_read_step_size = kStepSize;
    DWORD pre_read = 1;

    base::win::RegKey key(HKEY_CURRENT_USER, L"Software\\Google\\ChromeFrame",
                          KEY_QUERY_VALUE);
    if (key.Valid()) {
      key.ReadValueDW(L"PreReadSize", &pre_read_size);
      key.ReadValueDW(L"PreReadStepSize", &pre_read_step_size);
      key.ReadValueDW(L"PreRead", &pre_read);
      key.Close();
    }
    if (pre_read) {
      TRACE_EVENT_BEGIN_ETW("PreReadImage", 0, "");
      file_util::PreReadImage(dir->c_str(), pre_read_size, pre_read_step_size);
      TRACE_EVENT_END_ETW("PreReadImage", 0, "");
    }
  }
#endif  // NDEBUG

  return ::LoadLibraryExW(dir->c_str(), NULL,
                          LOAD_WITH_ALTERED_SEARCH_PATH);
}

void RecordDidRun(const std::wstring& dll_path) {
  bool system_level = !InstallUtil::IsPerUserInstall(dll_path.c_str());
  GoogleUpdateSettings::UpdateDidRunState(true, system_level);
}

void ClearDidRun(const std::wstring& dll_path) {
  bool system_level = !InstallUtil::IsPerUserInstall(dll_path.c_str());
  GoogleUpdateSettings::UpdateDidRunState(false, system_level);
}

}
//=============================================================================

MainDllLoader::MainDllLoader() : dll_(NULL) {
}

MainDllLoader::~MainDllLoader() {
#ifdef PURIFY
  // We should never unload the dll. There is only risk and no gain from
  // doing so. The singleton dtors have been already run by AtExitManager.
  ::FreeLibrary(dll_);
#endif
}

// Loading chrome is an interesting affair. First we try loading from the
// current directory to support run-what-you-compile and other development
// scenarios.
// If that fails then we look at the --chrome-version command line flag followed
// by the 'CHROME_VERSION' env variable to determine if we should stick with an
// older dll version even if a new one is available to support upgrade-in-place
// scenarios.
// If that fails then finally we look at the registry which should point us
// to the latest version. This is the expected path for the first chrome.exe
// browser instance in an installed build.
HMODULE MainDllLoader::Load(std::wstring* out_version, std::wstring* out_file) {
  std::wstring dir(GetExecutablePath());
  *out_file = dir;
  HMODULE dll = LoadChromeWithDirectory(out_file);
  if (dll)
    return dll;

  std::wstring version_string;
  scoped_ptr<Version> version;
  const CommandLine& cmd_line = *CommandLine::ForCurrentProcess();
  if (cmd_line.HasSwitch(switches::kChromeVersion)) {
    version_string = cmd_line.GetSwitchValueNative(switches::kChromeVersion);
    version.reset(Version::GetVersionFromString(WideToASCII(version_string)));

    if (!version.get()) {
      // If a bogus command line flag was given, then abort.
      LOG(ERROR) << "Invalid version string received on command line: "
                 << version_string;
      return NULL;
    }
  }

  if (!version.get()) {
    if (EnvQueryStr(ASCIIToWide(chrome::kChromeVersionEnvVar).c_str(),
                    &version_string)) {
      version.reset(Version::GetVersionFromString(WideToASCII(version_string)));
    }
  }

  if (!version.get()) {
    std::wstring reg_path(GetRegistryPath());
    // Look into the registry to find the latest version. We don't validate
    // this by building a Version object to avoid harming normal case startup
    // time.
    version_string.clear();
    GetVersion(dir.c_str(), reg_path.c_str(), &version_string);
  }

  if (version.get() || !version_string.empty()) {
    *out_file = dir;
    *out_version = version_string;
    out_file->append(*out_version).append(L"\\");
    dll = LoadChromeWithDirectory(out_file);
    if (!dll) {
      LOG(ERROR) << "Failed to load Chrome DLL from " << out_file;
    }
    return dll;
  } else {
    LOG(ERROR) << "Could not get Chrome DLL version.";
    return NULL;
  }
}

// Launching is a matter of loading the right dll, setting the CHROME_VERSION
// environment variable and just calling the entry point. Derived classes can
// add custom code in the OnBeforeLaunch callback.
int MainDllLoader::Launch(HINSTANCE instance,
                          sandbox::SandboxInterfaceInfo* sbox_info) {
  std::wstring version;
  std::wstring file;
  dll_ = Load(&version, &file);
  if (!dll_)
    return ResultCodes::MISSING_DATA;

  scoped_ptr<base::Environment> env(base::Environment::Create());
  env->SetVar(chrome::kChromeVersionEnvVar, WideToUTF8(version));

  InitCrashReporterWithDllPath(file);
  OnBeforeLaunch(file);

  DLL_MAIN entry_point =
      reinterpret_cast<DLL_MAIN>(::GetProcAddress(dll_, "ChromeMain"));
  if (!entry_point)
    return ResultCodes::BAD_PROCESS_TYPE;

  int rc = entry_point(instance, sbox_info, ::GetCommandLineW());
  return OnBeforeExit(rc, file);
}

void MainDllLoader::RelaunchChromeBrowserWithNewCommandLineIfNeeded() {
  RelaunchChromeBrowserWithNewCommandLineIfNeededFunc relaunch_function =
      reinterpret_cast<RelaunchChromeBrowserWithNewCommandLineIfNeededFunc>(
          ::GetProcAddress(dll_,
                           "RelaunchChromeBrowserWithNewCommandLineIfNeeded"));
  if (!relaunch_function) {
    LOG(ERROR) << "Could not find exported function "
               << "RelaunchChromeBrowserWithNewCommandLineIfNeeded";
  } else {
    relaunch_function();
  }
}

//=============================================================================

class ChromeDllLoader : public MainDllLoader {
 public:
  virtual std::wstring GetRegistryPath() {
    std::wstring key(google_update::kRegPathClients);
    BrowserDistribution* dist = BrowserDistribution::GetDistribution();
    key.append(L"\\").append(dist->GetAppGuid());
    return key;
  }

  virtual void OnBeforeLaunch(const std::wstring& dll_path) {
    RecordDidRun(dll_path);
  }

  virtual int OnBeforeExit(int return_code, const std::wstring& dll_path) {
    // NORMAL_EXIT_CANCEL is used for experiments when the user cancels
    // so we need to reset the did_run signal so omaha does not count
    // this run as active usage.
    if (ResultCodes::NORMAL_EXIT_CANCEL == return_code) {
      ClearDidRun(dll_path);
    }
    return return_code;
  }
};

//=============================================================================

class ChromiumDllLoader : public MainDllLoader {
 public:
  virtual std::wstring GetRegistryPath() {
    BrowserDistribution* dist = BrowserDistribution::GetDistribution();
    return dist->GetVersionKey();
  }
};

MainDllLoader* MakeMainDllLoader() {
#if defined(GOOGLE_CHROME_BUILD)
  return new ChromeDllLoader();
#else
  return new ChromiumDllLoader();
#endif
}
<commit_msg>Applying http://codereview.chromium.org/7400027/ for Chris Hamilton (chrisha@chromium.org)<commit_after>// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <windows.h>
#include <shlwapi.h>

#include "base/command_line.h"
#include "base/debug/trace_event.h"
#include "base/environment.h"
#include "base/file_util.h"
#include "base/logging.h"
#include "base/memory/scoped_ptr.h"
#include "base/win/registry.h"
#include "base/string_util.h"
#include "base/utf_string_conversions.h"
#include "base/version.h"
#include "chrome/app/breakpad_win.h"
#include "chrome/app/client_util.h"
#include "chrome/common/chrome_constants.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/installer/util/browser_distribution.h"
#include "chrome/installer/util/channel_info.h"
#include "chrome/installer/util/install_util.h"
#include "chrome/installer/util/google_update_constants.h"
#include "chrome/installer/util/google_update_settings.h"
#include "chrome/installer/util/util_constants.h"
#include "content/common/result_codes.h"

namespace {
// The entry point signature of chrome.dll.
typedef int (*DLL_MAIN)(HINSTANCE, sandbox::SandboxInterfaceInfo*, wchar_t*);

typedef void (*RelaunchChromeBrowserWithNewCommandLineIfNeededFunc)();

// Gets chrome version according to the load path. |exe_path| must be the
// backslash terminated directory of the current chrome.exe.
bool GetVersion(const wchar_t* exe_path, const wchar_t* key_path,
                std::wstring* version) {
  HKEY reg_root = InstallUtil::IsPerUserInstall(exe_path) ? HKEY_CURRENT_USER :
                                                            HKEY_LOCAL_MACHINE;
  bool success = false;

  base::win::RegKey key(reg_root, key_path, KEY_QUERY_VALUE);
  if (key.Valid()) {
    // If 'new_chrome.exe' is present it means chrome was auto-updated while
    // running. We need to consult the opv value so we can load the old dll.
    // TODO(cpu) : This is solving the same problem as the environment variable
    // so one of them will eventually be deprecated.
    std::wstring new_chrome_exe(exe_path);
    new_chrome_exe.append(installer::kChromeNewExe);
    if (::PathFileExistsW(new_chrome_exe.c_str()) &&
        key.ReadValue(google_update::kRegOldVersionField,
                      version) == ERROR_SUCCESS) {
      success = true;
    } else {
      success = (key.ReadValue(google_update::kRegVersionField,
                               version) == ERROR_SUCCESS);
    }
  }

  return success;
}

// Gets the path of the current exe with a trailing backslash.
std::wstring GetExecutablePath() {
  wchar_t path[MAX_PATH];
  ::GetModuleFileNameW(NULL, path, MAX_PATH);
  if (!::PathRemoveFileSpecW(path))
    return std::wstring();
  std::wstring exe_path(path);
  return exe_path.append(L"\\");
}

// Not generic, we only handle strings up to 128 chars.
bool EnvQueryStr(const wchar_t* key_name, std::wstring* value) {
  wchar_t out[128];
  DWORD count = sizeof(out)/sizeof(out[0]);
  DWORD rv = ::GetEnvironmentVariableW(key_name, out, count);
  if ((rv == 0) || (rv >= count))
    return false;
  *value = out;
  return true;
}

// Expects that |dir| has a trailing backslash. |dir| is modified so it
// contains the full path that was tried. Caller must check for the return
// value not being null to determine if this path contains a valid dll.
HMODULE LoadChromeWithDirectory(std::wstring* dir) {
  ::SetCurrentDirectoryW(dir->c_str());
  const CommandLine& cmd_line = *CommandLine::ForCurrentProcess();
#ifdef _WIN64
  if ((cmd_line.GetSwitchValueASCII(switches::kProcessType) ==
      switches::kNaClBrokerProcess) ||
      (cmd_line.GetSwitchValueASCII(switches::kProcessType) ==
      switches::kNaClLoaderProcess)) {
    // Load the 64-bit DLL when running in a 64-bit process.
    dir->append(installer::kChromeNaCl64Dll);
  } else {
    // Only NaCl broker and loader can be launched as Win64 processes.
    NOTREACHED();
    return NULL;
  }
#else
  dir->append(installer::kChromeDll);
#endif

#ifndef WIN_DISABLE_PREREAD
#ifdef NDEBUG
  // Experimental pre-reading optimization
  // The idea is to pre-read a significant portion of chrome.dll in advance
  // so that subsequent hard page faults are avoided.
  //
  // Pre-read may be disabled at compile time by defining WIN_DISABLE_PREREAD,
  // but by default it is enabled in release builds. The ability to disable it
  // is useful for evaluating competing optimization techniques.
  if (!cmd_line.HasSwitch(switches::kProcessType)) {
    // The kernel brings in 8 pages for the code section at a time and 4 pages
    // for other sections. We can skip over these pages to avoid a soft page
    // fault which may not occur during code execution. However skipping 4K at
    // a time still has better performance over 32K and 16K according to data.
    // TODO(ananta): Investigate this and tune.
    const size_t kStepSize = 4 * 1024;

    DWORD pre_read_size = 0;
    DWORD pre_read_step_size = kStepSize;
    DWORD pre_read = 1;

    base::win::RegKey key(HKEY_CURRENT_USER, L"Software\\Google\\ChromeFrame",
                          KEY_QUERY_VALUE);
    if (key.Valid()) {
      key.ReadValueDW(L"PreReadSize", &pre_read_size);
      key.ReadValueDW(L"PreReadStepSize", &pre_read_step_size);
      key.ReadValueDW(L"PreRead", &pre_read);
      key.Close();
    }
    if (pre_read) {
      TRACE_EVENT_BEGIN_ETW("PreReadImage", 0, "");
      file_util::PreReadImage(dir->c_str(), pre_read_size, pre_read_step_size);
      TRACE_EVENT_END_ETW("PreReadImage", 0, "");
    }
  }
#endif  // NDEBUG
#endif  // WIN_DISABLE_PREREAD

  return ::LoadLibraryExW(dir->c_str(), NULL,
                          LOAD_WITH_ALTERED_SEARCH_PATH);
}

void RecordDidRun(const std::wstring& dll_path) {
  bool system_level = !InstallUtil::IsPerUserInstall(dll_path.c_str());
  GoogleUpdateSettings::UpdateDidRunState(true, system_level);
}

void ClearDidRun(const std::wstring& dll_path) {
  bool system_level = !InstallUtil::IsPerUserInstall(dll_path.c_str());
  GoogleUpdateSettings::UpdateDidRunState(false, system_level);
}

}
//=============================================================================

MainDllLoader::MainDllLoader() : dll_(NULL) {
}

MainDllLoader::~MainDllLoader() {
#ifdef PURIFY
  // We should never unload the dll. There is only risk and no gain from
  // doing so. The singleton dtors have been already run by AtExitManager.
  ::FreeLibrary(dll_);
#endif
}

// Loading chrome is an interesting affair. First we try loading from the
// current directory to support run-what-you-compile and other development
// scenarios.
// If that fails then we look at the --chrome-version command line flag followed
// by the 'CHROME_VERSION' env variable to determine if we should stick with an
// older dll version even if a new one is available to support upgrade-in-place
// scenarios.
// If that fails then finally we look at the registry which should point us
// to the latest version. This is the expected path for the first chrome.exe
// browser instance in an installed build.
HMODULE MainDllLoader::Load(std::wstring* out_version, std::wstring* out_file) {
  std::wstring dir(GetExecutablePath());
  *out_file = dir;
  HMODULE dll = LoadChromeWithDirectory(out_file);
  if (dll)
    return dll;

  std::wstring version_string;
  scoped_ptr<Version> version;
  const CommandLine& cmd_line = *CommandLine::ForCurrentProcess();
  if (cmd_line.HasSwitch(switches::kChromeVersion)) {
    version_string = cmd_line.GetSwitchValueNative(switches::kChromeVersion);
    version.reset(Version::GetVersionFromString(WideToASCII(version_string)));

    if (!version.get()) {
      // If a bogus command line flag was given, then abort.
      LOG(ERROR) << "Invalid version string received on command line: "
                 << version_string;
      return NULL;
    }
  }

  if (!version.get()) {
    if (EnvQueryStr(ASCIIToWide(chrome::kChromeVersionEnvVar).c_str(),
                    &version_string)) {
      version.reset(Version::GetVersionFromString(WideToASCII(version_string)));
    }
  }

  if (!version.get()) {
    std::wstring reg_path(GetRegistryPath());
    // Look into the registry to find the latest version. We don't validate
    // this by building a Version object to avoid harming normal case startup
    // time.
    version_string.clear();
    GetVersion(dir.c_str(), reg_path.c_str(), &version_string);
  }

  if (version.get() || !version_string.empty()) {
    *out_file = dir;
    *out_version = version_string;
    out_file->append(*out_version).append(L"\\");
    dll = LoadChromeWithDirectory(out_file);
    if (!dll) {
      LOG(ERROR) << "Failed to load Chrome DLL from " << out_file;
    }
    return dll;
  } else {
    LOG(ERROR) << "Could not get Chrome DLL version.";
    return NULL;
  }
}

// Launching is a matter of loading the right dll, setting the CHROME_VERSION
// environment variable and just calling the entry point. Derived classes can
// add custom code in the OnBeforeLaunch callback.
int MainDllLoader::Launch(HINSTANCE instance,
                          sandbox::SandboxInterfaceInfo* sbox_info) {
  std::wstring version;
  std::wstring file;
  dll_ = Load(&version, &file);
  if (!dll_)
    return ResultCodes::MISSING_DATA;

  scoped_ptr<base::Environment> env(base::Environment::Create());
  env->SetVar(chrome::kChromeVersionEnvVar, WideToUTF8(version));

  InitCrashReporterWithDllPath(file);
  OnBeforeLaunch(file);

  DLL_MAIN entry_point =
      reinterpret_cast<DLL_MAIN>(::GetProcAddress(dll_, "ChromeMain"));
  if (!entry_point)
    return ResultCodes::BAD_PROCESS_TYPE;

  int rc = entry_point(instance, sbox_info, ::GetCommandLineW());
  return OnBeforeExit(rc, file);
}

void MainDllLoader::RelaunchChromeBrowserWithNewCommandLineIfNeeded() {
  RelaunchChromeBrowserWithNewCommandLineIfNeededFunc relaunch_function =
      reinterpret_cast<RelaunchChromeBrowserWithNewCommandLineIfNeededFunc>(
          ::GetProcAddress(dll_,
                           "RelaunchChromeBrowserWithNewCommandLineIfNeeded"));
  if (!relaunch_function) {
    LOG(ERROR) << "Could not find exported function "
               << "RelaunchChromeBrowserWithNewCommandLineIfNeeded";
  } else {
    relaunch_function();
  }
}

//=============================================================================

class ChromeDllLoader : public MainDllLoader {
 public:
  virtual std::wstring GetRegistryPath() {
    std::wstring key(google_update::kRegPathClients);
    BrowserDistribution* dist = BrowserDistribution::GetDistribution();
    key.append(L"\\").append(dist->GetAppGuid());
    return key;
  }

  virtual void OnBeforeLaunch(const std::wstring& dll_path) {
    RecordDidRun(dll_path);
  }

  virtual int OnBeforeExit(int return_code, const std::wstring& dll_path) {
    // NORMAL_EXIT_CANCEL is used for experiments when the user cancels
    // so we need to reset the did_run signal so omaha does not count
    // this run as active usage.
    if (ResultCodes::NORMAL_EXIT_CANCEL == return_code) {
      ClearDidRun(dll_path);
    }
    return return_code;
  }
};

//=============================================================================

class ChromiumDllLoader : public MainDllLoader {
 public:
  virtual std::wstring GetRegistryPath() {
    BrowserDistribution* dist = BrowserDistribution::GetDistribution();
    return dist->GetVersionKey();
  }
};

MainDllLoader* MakeMainDllLoader() {
#if defined(GOOGLE_CHROME_BUILD)
  return new ChromeDllLoader();
#else
  return new ChromiumDllLoader();
#endif
}
<|endoftext|>


<commit_before>/*
 * RecordSerializerUtil.cpp
 *
 *  Created on: Feb 28, 2014
 *      Author: srch2
 */

#include "RecordSerializerUtil.h"
#include <set>
#include <sstream>
#include <algorithm>
#include "thirdparty/snappy-1.0.4/snappy.h"
#include "util/DateAndTimeHandler.h"

using namespace srch2::instantsearch;

namespace srch2 {
namespace util {

void RecordSerializerUtil::populateStoredSchema(Schema* storedSchema, const Schema *schema) {

	const string* primaryKey = schema->getPrimaryKey();
	storedSchema->setPrimaryKey(*primaryKey);
	bool pk_found = false;
	std::set<string> visitedAttr;
	std::map<std::string, unsigned>::const_iterator searchableAttributeIter =
			schema->getSearchableAttribute().begin();
	for ( ; searchableAttributeIter != schema->getSearchableAttribute().end();
			searchableAttributeIter++)
	{
		bool isMultiValued = schema->isSearchableAttributeMultiValued(searchableAttributeIter->second);
		bool isHighLight = schema->isHighlightEnabled(searchableAttributeIter->second);
		storedSchema->setSearchableAttribute(searchableAttributeIter->first, 1, isMultiValued, isHighLight);
		visitedAttr.insert(searchableAttributeIter->first);
	}

	map<string, unsigned>::const_iterator refiningAttributeIter =
			schema->getRefiningAttributes()->begin();
	for ( ; refiningAttributeIter != schema->getRefiningAttributes()->end();
			refiningAttributeIter++)
	{
		if (visitedAttr.count(refiningAttributeIter->first) > 0){
			continue;
		}
		bool isMultiValued = schema->isRefiningAttributeMultiValued(refiningAttributeIter->second);
		if (isMultiValued) {
			storedSchema->setSearchableAttribute(refiningAttributeIter->first,
								1, true, false);
			continue;
		}
		srch2is::FilterType type = schema->getTypeOfRefiningAttribute(refiningAttributeIter->second);
		switch (type) {
		case srch2is::ATTRIBUTE_TYPE_UNSIGNED:
		case srch2is::ATTRIBUTE_TYPE_FLOAT:
			storedSchema->setRefiningAttribute(refiningAttributeIter->first,
					type, *schema->getDefaultValueOfRefiningAttribute(refiningAttributeIter->second),
					false);
			break;
		default:
			storedSchema->setSearchableAttribute(refiningAttributeIter->first,
					1, false, false);
		}
	}
}

void RecordSerializerUtil::convertCompactToJSONString(Schema * storedAttrSchema,
		StoredRecordBuffer buffer, const string& externalRecordId, string& jsonBuffer) {
	convertCompactToJSONString(storedAttrSchema, buffer, externalRecordId, jsonBuffer, NULL);
}
void RecordSerializerUtil::convertCompactToJSONString(Schema * storedAttrSchema, StoredRecordBuffer buffer,
		const string& externalRecordId, string& jsonBuffer, const vector<string>* attrToReturn) {

		RecordSerializer compactRecDeserializer = RecordSerializer(*storedAttrSchema);
		std::map<std::string, unsigned>::const_iterator iter =
				storedAttrSchema->getSearchableAttribute().begin();
		jsonBuffer.reserve(1.25 * buffer.length);
		jsonBuffer.append("{") ;
		jsonBuffer+='"'; jsonBuffer+=*(storedAttrSchema->getPrimaryKey()); jsonBuffer+='"';
		jsonBuffer+=":\""; jsonBuffer+=externalRecordId; jsonBuffer+="\",";
		for ( ; iter != storedAttrSchema->getSearchableAttribute().end(); iter++)
		{
			if (attrToReturn &&
			    std::find(attrToReturn->begin(), attrToReturn->end(), iter->first) == attrToReturn->end()) {
				continue;
			}
			unsigned id = storedAttrSchema->getSearchableAttributeId(iter->first);
			unsigned lenOffset = compactRecDeserializer.getSearchableOffset(id);
			const char *attrdata = buffer.start.get() + *((unsigned *)(buffer.start.get() + lenOffset));
			unsigned len = *(((unsigned *)(buffer.start.get() + lenOffset)) + 1) -
					*((unsigned *)(buffer.start.get() + lenOffset));

			std::string uncompressedInMemoryRecordString;
			snappy::Uncompress(attrdata,len, &uncompressedInMemoryRecordString);
			jsonBuffer+='"'; jsonBuffer+=iter->first; jsonBuffer+='"';
			jsonBuffer+=':';
			if (storedAttrSchema->isSearchableAttributeMultiValued(id)) {
				jsonBuffer+='[';
				size_t lastpos = 0;
				while(1) {
					size_t pos = uncompressedInMemoryRecordString.find(" $$ ", lastpos) ;
					if (pos == string::npos)
						break;
					string result =  uncompressedInMemoryRecordString.substr(lastpos, pos - lastpos);
					jsonBuffer+='"';
					cleanAndAppendToBuffer(result, jsonBuffer);
					jsonBuffer+='"';
					jsonBuffer+=',';
					lastpos = pos + 4;
				}
				string result =  uncompressedInMemoryRecordString.substr(lastpos,
															uncompressedInMemoryRecordString.length());
				jsonBuffer+='"';
				cleanAndAppendToBuffer(result, jsonBuffer);
				jsonBuffer+='"';
				jsonBuffer+=']';
				jsonBuffer+=',';
			} else {
				jsonBuffer+='"';
				cleanAndAppendToBuffer(uncompressedInMemoryRecordString, jsonBuffer);
				jsonBuffer+='"';
				jsonBuffer+=',';
			}
		}
		iter = storedAttrSchema->getRefiningAttributes()->begin();
		for ( ; iter != storedAttrSchema->getRefiningAttributes()->end(); iter++)
		{
			if (attrToReturn &&
					std::find(attrToReturn->begin(), attrToReturn->end(), iter->first) == attrToReturn->end()) {
				continue;
			}
			unsigned id = storedAttrSchema->getRefiningAttributeId(iter->first);
			unsigned lenOffset = compactRecDeserializer.getRefiningOffset(id);
			jsonBuffer+='"'; jsonBuffer+=iter->first; jsonBuffer+='"';
			jsonBuffer+=':';
			jsonBuffer+='"';
			switch(storedAttrSchema->getTypeOfRefiningAttribute(id)){
			case srch2is::ATTRIBUTE_TYPE_FLOAT:
			{
				float attrdata = *((float *)(buffer.start.get() + lenOffset));
				stringstream ss;
				ss << attrdata;
				jsonBuffer += ss.str();
				break;
			}
			case srch2is::ATTRIBUTE_TYPE_UNSIGNED:
			{
				unsigned attrdata = *((unsigned *)(buffer.start.get() + lenOffset));
				stringstream ss;
				ss << attrdata;
				jsonBuffer += ss.str();
				break;
			}
			default: break;
				// should not come here.
			}
			jsonBuffer+='"';
			jsonBuffer+=',';
		}
		if (storedAttrSchema->getRefiningAttributes()->size() > 0 ||
				storedAttrSchema->getSearchableAttribute().size() > 0)
			jsonBuffer.erase(jsonBuffer.length()-1);
		jsonBuffer.append("}");
}

void RecordSerializerUtil::cleanAndAppendToBuffer(const string& in, string& out) {
	unsigned inLen = in.length();
	unsigned inIdx = 0;
	while (inIdx < inLen) {
		// remove non printable characters
		if (in[inIdx] < 32) {
			++inIdx; continue;
		}
		switch(in[inIdx]) {
		case '"':
		{
			// because we have reached here, there was no '\' before this '"'
			out +='\\'; out +='"';
			break;
		}
		case '\\':
		{
			if (inIdx != inLen - 1 and in[inIdx + 1] == '"') {  // looking for '\"'
				out += in[inIdx++];      // push them in one go...
				out += in[inIdx];
			} else {
				out +='\\'; out +='\\';  // escape the lonesome '\'
			}
			break;
		}
		default:
			out += in[inIdx];
		}
		++inIdx;
	}
}

/*
 *  The function returns the list of values for the list of refining attributes from
 *  the in-memory representation in forward Index.
 */
void RecordSerializerUtil::getBatchOfAttributes(
        const std::vector<string> & refiningAttributes, const Schema * schema, const Byte* data,
        std::vector<TypedValue> * typedValuesArg)  {

    std::vector<TypedValue>& typedValues = (*typedValuesArg);
    Schema *storedSchema = Schema::create();
    RecordSerializerUtil::populateStoredSchema(storedSchema, schema);
    RecordSerializer recSerializer(*storedSchema);
    // now extract the scores
    unsigned startOffset = 0;
    for(unsigned i = 0 ; i < refiningAttributes.size(); ++i){
    	const string& name = refiningAttributes[i];
    	FilterType type = getAttributeType(name, schema);
    	bool multiVal = schema->isRefiningAttributeMultiValued(schema->getRefiningAttributeId(name));
        TypedValue attributeValue;
        convertByteArrayToTypedValue(name , multiVal, type, recSerializer, data , &attributeValue);
        typedValues.push_back(attributeValue);
    }
    delete storedSchema;
}

FilterType RecordSerializerUtil::getAttributeType(const string& name,
        const Schema * schema) {
	unsigned id = schema->getRefiningAttributeId(name);
	return schema->getTypeOfRefiningAttribute(id);
}

/*
 *  read an unsigned int value at an offset (= startOffset) from the data pointer.
 */
unsigned RecordSerializerUtil::convertByteArrayToUnsigned(
        unsigned startOffset , const Byte * data) {

    const Byte * bytePointer = data + startOffset;
    unsigned * unsignedPointer = (unsigned *) bytePointer;
    return *unsignedPointer;
}

/*
 *  read a float value at an offset (= startOffset) from the data pointer.
 */
float RecordSerializerUtil::convertByteArrayToFloat(unsigned startOffset, const Byte * data) {
    const Byte * bytePointer = data + startOffset;
    float * floatPointer = (float *) bytePointer;
    return *floatPointer;
}

/*
 *  read a long value at an offset (= startOffset) from the data pointer.
 */
long RecordSerializerUtil::convertByteArrayToLong(
        unsigned startOffset, const Byte * data) {
    const Byte * bytePointer = data + startOffset;
    long * longPointer = (long *) bytePointer;
    return *longPointer;
}

/*
 *   Given a refining attribute name and type, fetch its value from in-memory compact representation.
 *   - Single value refining attribute of type int , float and long are stored as it is in the byte
 *     array.
 *   - Multivalue refining attributes are stored as single compressed string where each values is
 *     separated by a delimiter (MULTI_VAL_ATTR_DELIMITER defined in Constants.h).
 *   - Single Value refining attributes of type text and time are stored as compressed string.
 *     Note: time string is converted to long format.
 */
void RecordSerializerUtil::convertByteArrayToTypedValue(const string& name,
		bool isMultiValued, const FilterType& type, RecordSerializer& recSerializer, const Byte * data,
		TypedValue * result) {

	if(isMultiValued == false){ // case of single value
		unsigned intValue = 0;
		float floatValue = 0;
		long longValue = 0;
		unsigned sizeOfString = 0;
		string stringValue = "";
		switch (type) {
		case ATTRIBUTE_TYPE_UNSIGNED:
		{
			unsigned startOffset = 0;
			if (recSerializer.getStorageSchema().getRefiningAttributeId(name) != -1) {
				startOffset = recSerializer.getRefiningOffset(name);
				intValue = convertByteArrayToUnsigned(startOffset,data);
			} else {
				ASSERT(false);  // for Debug mode
			}
			result->setTypedValue(intValue);
			break;
		}
		case ATTRIBUTE_TYPE_FLOAT:
		{
			unsigned startOffset = 0;
			if (recSerializer.getStorageSchema().getRefiningAttributeId(name) != -1) {
				startOffset = recSerializer.getRefiningOffset(name);
				floatValue = convertByteArrayToFloat(startOffset,data);
			} else {
				ASSERT(false);  // for Debug mode
			}
			result->setTypedValue(floatValue);
			break;
		}
		case ATTRIBUTE_TYPE_TEXT:
		{
			unsigned lenOffset = 0;
			if (recSerializer.getStorageSchema().getSearchableAttributeId(name) != -1) {
				lenOffset = recSerializer.getSearchableOffset(name);
				const char *attrdata = data + *((unsigned *)(data + lenOffset));
				unsigned len = *(((unsigned *)(data + lenOffset)) + 1) -
						*((unsigned *)(data + lenOffset));
				snappy::Uncompress(attrdata,len, &stringValue);
				std::transform(stringValue.begin(), stringValue.end(), stringValue.begin(), ::tolower);
			} else {
				ASSERT(false);  // for Debug mode
			}
			result->setTypedValue(stringValue);
			break;
		}
		case ATTRIBUTE_TYPE_TIME:
		{
			unsigned lenOffset = 0;
			if (recSerializer.getStorageSchema().getSearchableAttributeId(name) != -1) {
				lenOffset = recSerializer.getSearchableOffset(name);
				const char *attrdata = data + *((unsigned *)(data + lenOffset));
				unsigned len = *(((unsigned *)(data + lenOffset)) + 1) -
						*((unsigned *)(data + lenOffset));
				snappy::Uncompress(attrdata,len, &stringValue);
				longValue = DateAndTimeHandler::convertDateTimeStringToSecondsFromEpoch(stringValue);
			} else {
				ASSERT(false);  // for Debug mode
			}
			result->setTypedValue(longValue);
			break;
		}

		default:
			ASSERT(false);
			break;
		}
	}else{ // case of multi value

		unsigned lenOffset = 0;
		string stringValue = "";
		vector<string> multiValues;
		if (recSerializer.getStorageSchema().getSearchableAttributeId(name) != -1) {
			lenOffset = recSerializer.getSearchableOffset(name);
			const char *attrdata = data + *((unsigned *)(data + lenOffset));
			unsigned len = *(((unsigned *)(data + lenOffset)) + 1) -
					*((unsigned *)(data + lenOffset));
			snappy::Uncompress(attrdata,len, &stringValue);
			size_t lastpos = 0;
			while(1) {
				size_t pos = stringValue.find(MULTI_VAL_ATTR_DELIMITER, lastpos) ;
				if (pos == string::npos)
					break;
				string result =  stringValue.substr(lastpos, pos - lastpos);
				std::transform(result.begin(), result.end(), result.begin(), ::tolower);
				multiValues.push_back(result);
				lastpos = pos + 4;
			}
			string lastVal = stringValue.substr(lastpos, stringValue.size());
			std::transform(lastVal.begin(), lastVal.end(), lastVal.begin(), ::tolower);
			multiValues.push_back(lastVal);
		} else {
			ASSERT(false);  // for debug ..else we return empty values
		}

		vector<unsigned> intValues;
		vector<float> floatValues;
		vector<long> timeValues;

		switch (type) {
		case ATTRIBUTE_TYPE_UNSIGNED:
			for(int i=0;i<multiValues.size(); i++){
				intValues.push_back(atol(multiValues[i].c_str()));
			}
			result->setTypedValue(intValues);
			break;
		case ATTRIBUTE_TYPE_FLOAT:
			for(int i=0;i<multiValues.size(); i++){
				floatValues.push_back(atof(multiValues[i].c_str()));
			}
			result->setTypedValue(floatValues);
			break;
		case ATTRIBUTE_TYPE_TEXT:
			result->setTypedValue(multiValues);
			break;
		case ATTRIBUTE_TYPE_TIME:
			for(int i=0;i<multiValues.size(); i++){
				long lTime = DateAndTimeHandler::convertDateTimeStringToSecondsFromEpoch(multiValues[i]);
				timeValues.push_back(lTime);
			}
			result->setTypedValue(timeValues);
			break;
		case ATTRIBUTE_TYPE_DURATION:
			ASSERT(false);
			break;
		default:
			ASSERT(false);
			break;
		}
	}
}

RecordSerializerUtil::RecordSerializerUtil() {
}

RecordSerializerUtil::~RecordSerializerUtil() {
}

} /* namespace util */
} /* namespace srch2 */
<commit_msg>cast the char value to unsigned char to skip the control logic for the UTF8 chars<commit_after>/*
 * RecordSerializerUtil.cpp
 *
 *  Created on: Feb 28, 2014
 *      Author: srch2
 */

#include "RecordSerializerUtil.h"
#include <set>
#include <sstream>
#include <algorithm>
#include "thirdparty/snappy-1.0.4/snappy.h"
#include "util/DateAndTimeHandler.h"

using namespace srch2::instantsearch;

namespace srch2 {
namespace util {

void RecordSerializerUtil::populateStoredSchema(Schema* storedSchema, const Schema *schema) {

	const string* primaryKey = schema->getPrimaryKey();
	storedSchema->setPrimaryKey(*primaryKey);
	bool pk_found = false;
	std::set<string> visitedAttr;
	std::map<std::string, unsigned>::const_iterator searchableAttributeIter =
			schema->getSearchableAttribute().begin();
	for ( ; searchableAttributeIter != schema->getSearchableAttribute().end();
			searchableAttributeIter++)
	{
		bool isMultiValued = schema->isSearchableAttributeMultiValued(searchableAttributeIter->second);
		bool isHighLight = schema->isHighlightEnabled(searchableAttributeIter->second);
		storedSchema->setSearchableAttribute(searchableAttributeIter->first, 1, isMultiValued, isHighLight);
		visitedAttr.insert(searchableAttributeIter->first);
	}

	map<string, unsigned>::const_iterator refiningAttributeIter =
			schema->getRefiningAttributes()->begin();
	for ( ; refiningAttributeIter != schema->getRefiningAttributes()->end();
			refiningAttributeIter++)
	{
		if (visitedAttr.count(refiningAttributeIter->first) > 0){
			continue;
		}
		bool isMultiValued = schema->isRefiningAttributeMultiValued(refiningAttributeIter->second);
		if (isMultiValued) {
			storedSchema->setSearchableAttribute(refiningAttributeIter->first,
								1, true, false);
			continue;
		}
		srch2is::FilterType type = schema->getTypeOfRefiningAttribute(refiningAttributeIter->second);
		switch (type) {
		case srch2is::ATTRIBUTE_TYPE_UNSIGNED:
		case srch2is::ATTRIBUTE_TYPE_FLOAT:
			storedSchema->setRefiningAttribute(refiningAttributeIter->first,
					type, *schema->getDefaultValueOfRefiningAttribute(refiningAttributeIter->second),
					false);
			break;
		default:
			storedSchema->setSearchableAttribute(refiningAttributeIter->first,
					1, false, false);
		}
	}
}

void RecordSerializerUtil::convertCompactToJSONString(Schema * storedAttrSchema,
		StoredRecordBuffer buffer, const string& externalRecordId, string& jsonBuffer) {
	convertCompactToJSONString(storedAttrSchema, buffer, externalRecordId, jsonBuffer, NULL);
}
void RecordSerializerUtil::convertCompactToJSONString(Schema * storedAttrSchema, StoredRecordBuffer buffer,
		const string& externalRecordId, string& jsonBuffer, const vector<string>* attrToReturn) {

		RecordSerializer compactRecDeserializer = RecordSerializer(*storedAttrSchema);
		std::map<std::string, unsigned>::const_iterator iter =
				storedAttrSchema->getSearchableAttribute().begin();
		jsonBuffer.reserve(1.25 * buffer.length);
		jsonBuffer.append("{") ;
		jsonBuffer+='"'; jsonBuffer+=*(storedAttrSchema->getPrimaryKey()); jsonBuffer+='"';
		jsonBuffer+=":\""; jsonBuffer+=externalRecordId; jsonBuffer+="\",";
		for ( ; iter != storedAttrSchema->getSearchableAttribute().end(); iter++)
		{
			if (attrToReturn &&
			    std::find(attrToReturn->begin(), attrToReturn->end(), iter->first) == attrToReturn->end()) {
				continue;
			}
			unsigned id = storedAttrSchema->getSearchableAttributeId(iter->first);
			unsigned lenOffset = compactRecDeserializer.getSearchableOffset(id);
			const char *attrdata = buffer.start.get() + *((unsigned *)(buffer.start.get() + lenOffset));
			unsigned len = *(((unsigned *)(buffer.start.get() + lenOffset)) + 1) -
					*((unsigned *)(buffer.start.get() + lenOffset));

			std::string uncompressedInMemoryRecordString;
			snappy::Uncompress(attrdata,len, &uncompressedInMemoryRecordString);
			jsonBuffer+='"'; jsonBuffer+=iter->first; jsonBuffer+='"';
			jsonBuffer+=':';
			if (storedAttrSchema->isSearchableAttributeMultiValued(id)) {
				jsonBuffer+='[';
				size_t lastpos = 0;
				while(1) {
					size_t pos = uncompressedInMemoryRecordString.find(" $$ ", lastpos) ;
					if (pos == string::npos)
						break;
					string result =  uncompressedInMemoryRecordString.substr(lastpos, pos - lastpos);
					jsonBuffer+='"';
					cleanAndAppendToBuffer(result, jsonBuffer);
					jsonBuffer+='"';
					jsonBuffer+=',';
					lastpos = pos + 4;
				}
				string result =  uncompressedInMemoryRecordString.substr(lastpos,
															uncompressedInMemoryRecordString.length());
				jsonBuffer+='"';
				cleanAndAppendToBuffer(result, jsonBuffer);
				jsonBuffer+='"';
				jsonBuffer+=']';
				jsonBuffer+=',';
			} else {
				jsonBuffer+='"';
				cleanAndAppendToBuffer(uncompressedInMemoryRecordString, jsonBuffer);
				jsonBuffer+='"';
				jsonBuffer+=',';
			}
		}
		iter = storedAttrSchema->getRefiningAttributes()->begin();
		for ( ; iter != storedAttrSchema->getRefiningAttributes()->end(); iter++)
		{
			if (attrToReturn &&
					std::find(attrToReturn->begin(), attrToReturn->end(), iter->first) == attrToReturn->end()) {
				continue;
			}
			unsigned id = storedAttrSchema->getRefiningAttributeId(iter->first);
			unsigned lenOffset = compactRecDeserializer.getRefiningOffset(id);
			jsonBuffer+='"'; jsonBuffer+=iter->first; jsonBuffer+='"';
			jsonBuffer+=':';
			jsonBuffer+='"';
			switch(storedAttrSchema->getTypeOfRefiningAttribute(id)){
			case srch2is::ATTRIBUTE_TYPE_FLOAT:
			{
				float attrdata = *((float *)(buffer.start.get() + lenOffset));
				stringstream ss;
				ss << attrdata;
				jsonBuffer += ss.str();
				break;
			}
			case srch2is::ATTRIBUTE_TYPE_UNSIGNED:
			{
				unsigned attrdata = *((unsigned *)(buffer.start.get() + lenOffset));
				stringstream ss;
				ss << attrdata;
				jsonBuffer += ss.str();
				break;
			}
			default: break;
				// should not come here.
			}
			jsonBuffer+='"';
			jsonBuffer+=',';
		}
		if (storedAttrSchema->getRefiningAttributes()->size() > 0 ||
				storedAttrSchema->getSearchableAttribute().size() > 0)
			jsonBuffer.erase(jsonBuffer.length()-1);
		jsonBuffer.append("}");
}

void RecordSerializerUtil::cleanAndAppendToBuffer(const string& in, string& out) {
	unsigned inLen = in.length();
	unsigned inIdx = 0;
	while (inIdx < inLen) {
		// remove non printable characters
		if ( static_cast<unsigned> (in[inIdx]) < 32) {
			++inIdx; continue;
		}
		switch(in[inIdx]) {
		case '"':
		{
			// because we have reached here, there was no '\' before this '"'
			out +='\\'; out +='"';
			break;
		}
		case '\\':
		{
			if (inIdx != inLen - 1 and in[inIdx + 1] == '"') {  // looking for '\"'
				out += in[inIdx++];      // push them in one go...
				out += in[inIdx];
			} else {
				out +='\\'; out +='\\';  // escape the lonesome '\'
			}
			break;
		}
		default:
			out += in[inIdx];
		}
		++inIdx;
	}
}

/*
 *  The function returns the list of values for the list of refining attributes from
 *  the in-memory representation in forward Index.
 */
void RecordSerializerUtil::getBatchOfAttributes(
        const std::vector<string> & refiningAttributes, const Schema * schema, const Byte* data,
        std::vector<TypedValue> * typedValuesArg)  {

    std::vector<TypedValue>& typedValues = (*typedValuesArg);
    Schema *storedSchema = Schema::create();
    RecordSerializerUtil::populateStoredSchema(storedSchema, schema);
    RecordSerializer recSerializer(*storedSchema);
    // now extract the scores
    unsigned startOffset = 0;
    for(unsigned i = 0 ; i < refiningAttributes.size(); ++i){
    	const string& name = refiningAttributes[i];
    	FilterType type = getAttributeType(name, schema);
    	bool multiVal = schema->isRefiningAttributeMultiValued(schema->getRefiningAttributeId(name));
        TypedValue attributeValue;
        convertByteArrayToTypedValue(name , multiVal, type, recSerializer, data , &attributeValue);
        typedValues.push_back(attributeValue);
    }
    delete storedSchema;
}

FilterType RecordSerializerUtil::getAttributeType(const string& name,
        const Schema * schema) {
	unsigned id = schema->getRefiningAttributeId(name);
	return schema->getTypeOfRefiningAttribute(id);
}

/*
 *  read an unsigned int value at an offset (= startOffset) from the data pointer.
 */
unsigned RecordSerializerUtil::convertByteArrayToUnsigned(
        unsigned startOffset , const Byte * data) {

    const Byte * bytePointer = data + startOffset;
    unsigned * unsignedPointer = (unsigned *) bytePointer;
    return *unsignedPointer;
}

/*
 *  read a float value at an offset (= startOffset) from the data pointer.
 */
float RecordSerializerUtil::convertByteArrayToFloat(unsigned startOffset, const Byte * data) {
    const Byte * bytePointer = data + startOffset;
    float * floatPointer = (float *) bytePointer;
    return *floatPointer;
}

/*
 *  read a long value at an offset (= startOffset) from the data pointer.
 */
long RecordSerializerUtil::convertByteArrayToLong(
        unsigned startOffset, const Byte * data) {
    const Byte * bytePointer = data + startOffset;
    long * longPointer = (long *) bytePointer;
    return *longPointer;
}

/*
 *   Given a refining attribute name and type, fetch its value from in-memory compact representation.
 *   - Single value refining attribute of type int , float and long are stored as it is in the byte
 *     array.
 *   - Multivalue refining attributes are stored as single compressed string where each values is
 *     separated by a delimiter (MULTI_VAL_ATTR_DELIMITER defined in Constants.h).
 *   - Single Value refining attributes of type text and time are stored as compressed string.
 *     Note: time string is converted to long format.
 */
void RecordSerializerUtil::convertByteArrayToTypedValue(const string& name,
		bool isMultiValued, const FilterType& type, RecordSerializer& recSerializer, const Byte * data,
		TypedValue * result) {

	if(isMultiValued == false){ // case of single value
		unsigned intValue = 0;
		float floatValue = 0;
		long longValue = 0;
		unsigned sizeOfString = 0;
		string stringValue = "";
		switch (type) {
		case ATTRIBUTE_TYPE_UNSIGNED:
		{
			unsigned startOffset = 0;
			if (recSerializer.getStorageSchema().getRefiningAttributeId(name) != -1) {
				startOffset = recSerializer.getRefiningOffset(name);
				intValue = convertByteArrayToUnsigned(startOffset,data);
			} else {
				ASSERT(false);  // for Debug mode
			}
			result->setTypedValue(intValue);
			break;
		}
		case ATTRIBUTE_TYPE_FLOAT:
		{
			unsigned startOffset = 0;
			if (recSerializer.getStorageSchema().getRefiningAttributeId(name) != -1) {
				startOffset = recSerializer.getRefiningOffset(name);
				floatValue = convertByteArrayToFloat(startOffset,data);
			} else {
				ASSERT(false);  // for Debug mode
			}
			result->setTypedValue(floatValue);
			break;
		}
		case ATTRIBUTE_TYPE_TEXT:
		{
			unsigned lenOffset = 0;
			if (recSerializer.getStorageSchema().getSearchableAttributeId(name) != -1) {
				lenOffset = recSerializer.getSearchableOffset(name);
				const char *attrdata = data + *((unsigned *)(data + lenOffset));
				unsigned len = *(((unsigned *)(data + lenOffset)) + 1) -
						*((unsigned *)(data + lenOffset));
				snappy::Uncompress(attrdata,len, &stringValue);
				std::transform(stringValue.begin(), stringValue.end(), stringValue.begin(), ::tolower);
			} else {
				ASSERT(false);  // for Debug mode
			}
			result->setTypedValue(stringValue);
			break;
		}
		case ATTRIBUTE_TYPE_TIME:
		{
			unsigned lenOffset = 0;
			if (recSerializer.getStorageSchema().getSearchableAttributeId(name) != -1) {
				lenOffset = recSerializer.getSearchableOffset(name);
				const char *attrdata = data + *((unsigned *)(data + lenOffset));
				unsigned len = *(((unsigned *)(data + lenOffset)) + 1) -
						*((unsigned *)(data + lenOffset));
				snappy::Uncompress(attrdata,len, &stringValue);
				longValue = DateAndTimeHandler::convertDateTimeStringToSecondsFromEpoch(stringValue);
			} else {
				ASSERT(false);  // for Debug mode
			}
			result->setTypedValue(longValue);
			break;
		}

		default:
			ASSERT(false);
			break;
		}
	}else{ // case of multi value

		unsigned lenOffset = 0;
		string stringValue = "";
		vector<string> multiValues;
		if (recSerializer.getStorageSchema().getSearchableAttributeId(name) != -1) {
			lenOffset = recSerializer.getSearchableOffset(name);
			const char *attrdata = data + *((unsigned *)(data + lenOffset));
			unsigned len = *(((unsigned *)(data + lenOffset)) + 1) -
					*((unsigned *)(data + lenOffset));
			snappy::Uncompress(attrdata,len, &stringValue);
			size_t lastpos = 0;
			while(1) {
				size_t pos = stringValue.find(MULTI_VAL_ATTR_DELIMITER, lastpos) ;
				if (pos == string::npos)
					break;
				string result =  stringValue.substr(lastpos, pos - lastpos);
				std::transform(result.begin(), result.end(), result.begin(), ::tolower);
				multiValues.push_back(result);
				lastpos = pos + 4;
			}
			string lastVal = stringValue.substr(lastpos, stringValue.size());
			std::transform(lastVal.begin(), lastVal.end(), lastVal.begin(), ::tolower);
			multiValues.push_back(lastVal);
		} else {
			ASSERT(false);  // for debug ..else we return empty values
		}

		vector<unsigned> intValues;
		vector<float> floatValues;
		vector<long> timeValues;

		switch (type) {
		case ATTRIBUTE_TYPE_UNSIGNED:
			for(int i=0;i<multiValues.size(); i++){
				intValues.push_back(atol(multiValues[i].c_str()));
			}
			result->setTypedValue(intValues);
			break;
		case ATTRIBUTE_TYPE_FLOAT:
			for(int i=0;i<multiValues.size(); i++){
				floatValues.push_back(atof(multiValues[i].c_str()));
			}
			result->setTypedValue(floatValues);
			break;
		case ATTRIBUTE_TYPE_TEXT:
			result->setTypedValue(multiValues);
			break;
		case ATTRIBUTE_TYPE_TIME:
			for(int i=0;i<multiValues.size(); i++){
				long lTime = DateAndTimeHandler::convertDateTimeStringToSecondsFromEpoch(multiValues[i]);
				timeValues.push_back(lTime);
			}
			result->setTypedValue(timeValues);
			break;
		case ATTRIBUTE_TYPE_DURATION:
			ASSERT(false);
			break;
		default:
			ASSERT(false);
			break;
		}
	}
}

RecordSerializerUtil::RecordSerializerUtil() {
}

RecordSerializerUtil::~RecordSerializerUtil() {
}

} /* namespace util */
} /* namespace srch2 */
<|endoftext|>


<commit_before>#pragma once

#include "numerics/ulp_distance.hpp"

#include <cmath>
#include <cstdint>
#include <cstring>
#include <limits>

namespace principia {
namespace numerics {

inline std::int64_t ULPDistance(double const x, double const y) {
  if (x == y) {
    return 0;
  }
  double const x_sign = std::copysign(1, x);
  double const y_sign = std::copysign(1, y);
  if (x_sign != y_sign) {
    double const positive = x_sign == 1 ? x : y;
    double const negative = x_sign == 1 ? y : x;
    std::int64_t const positive_distance = ULPDistance(positive, +0.0);
    std::int64_t const negative_distance = ULPDistance(negative, -0.0);
    if (positive_distance >
            std::numeric_limits<std::int64_t>::max() - negative_distance) {
      return std::numeric_limits<std::int64_t>::max();
    } else {
      return positive_distance + negative_distance;
    }
  }
  std::int64_t x_bits;
  std::int64_t y_bits;
  static_assert(sizeof(x_bits) == sizeof(x), "");
  std::memcpy(&x_bits, &x, sizeof(x));
  std::memcpy(&y_bits, &y, sizeof(y));
  return std::abs(x_bits - y_bits);
}

}  // namespace numerics
}  // namespace principia
<commit_msg>after pleroy's review<commit_after>#pragma once

#include "numerics/ulp_distance.hpp"

#include <cmath>
#include <cstdint>
#include <cstring>
#include <limits>

namespace principia {
namespace numerics {

inline std::int64_t ULPDistance(double const x, double const y) {
  if (x == y) {
    return 0;
  }
  double const x_sign = std::copysign(1, x);
  double const y_sign = std::copysign(1, y);
  if (x_sign != y_sign) {
    double const positive = x_sign == 1 ? x : y;
    double const negative = x_sign == 1 ? y : x;
    std::int64_t const positive_distance = ULPDistance(positive, +0.0);
    std::int64_t const negative_distance = ULPDistance(negative, -0.0);
    if (positive_distance >
            std::numeric_limits<std::int64_t>::max() - negative_distance) {
      return std::numeric_limits<std::int64_t>::max();
    } else {
      return positive_distance + negative_distance;
    }
  }
  std::int64_t x_bits;
  std::int64_t y_bits;
  static_assert(sizeof(x_bits) == sizeof(x),
                "conversion between types of different sizes");
  std::memcpy(&x_bits, &x, sizeof(x));
  static_assert(sizeof(y_bits) == sizeof(y),
                "conversion between types of different sizes");
  std::memcpy(&y_bits, &y, sizeof(y));
  return std::abs(x_bits - y_bits);
}

}  // namespace numerics
}  // namespace principia
<|endoftext|>


<commit_before>#define CATCH_CONFIG_MAIN
#include "catch/catch.hpp"
#include "tangram.h"
#include "platform.h"
#include "debug.h"
#include "scene/scene.h"
#include "style/style.h"
#include "style/textStyle.h"
#include "labels/labels.h"

#include "view/view.h"
#include "tile/tile.h"

namespace Tangram {

glm::vec2 screenSize(256.f, 256.f);
TextBuffer dummy(nullptr);

std::unique_ptr<TextLabel> makeLabel(Label::Transform _transform, Label::Type _type, std::string id) {
    Label::Options options;
    options.color = 0xff;
    options.offset = {0.0f, 0.0f};
    options.properties = std::make_shared<Properties>();
    options.properties->add("id", id);
    options.interactive = true;

    return std::unique_ptr<TextLabel>(new TextLabel("label",
                                                    _transform, _type, {0, 0},
                                                    dummy, {0, 0}, options));
}

TEST_CASE("Test getFeaturesAtPoint", "[Labels][FeaturePicking]") {
    std::unique_ptr<Labels> labels(new Labels());

    View view(256, 256);
    view.setPosition(0, 0);
    view.setZoom(0);
    view.update();

    auto labelMesh = std::unique_ptr<LabelMesh>(new LabelMesh(nullptr, 0));
    auto textStyle = std::unique_ptr<TextStyle>(new TextStyle("test", nullptr));

    labelMesh->addLabel(makeLabel(glm::vec2{0,0}, Label::Type::point, "0"));
    labelMesh->addLabel(makeLabel(glm::vec2{1,-1}, Label::Type::point, "1"));
    labelMesh->addLabel(makeLabel(glm::vec2{1,1}, Label::Type::point, "2"));

    std::shared_ptr<Tile> tile(new Tile({0,0,0}, view.getMapProjection()));
    tile->getMesh(*textStyle.get()) = std::move(labelMesh);
    tile->update(0, view);

    std::vector<std::unique_ptr<Style>> styles;
    styles.push_back(std::move(textStyle));

    std::vector<std::shared_ptr<Tile>> tiles;

    tiles.push_back(tile);
    {
        auto& items = labels->getFeaturesAtPoint(view, 0, styles, tiles, 128, 128, false);
        REQUIRE(items.size() == 1);
        REQUIRE(items[0]->getString("id") == "0");
    }
    {
        auto& items = labels->getFeaturesAtPoint(view, 0, styles, tiles, 256, 256, false);
        REQUIRE(items.size() == 1);
        REQUIRE(items[0]->getString("id") == "1");
    }

    {
        auto& items = labels->getFeaturesAtPoint(view, 0, styles, tiles, 256, 0, false);
        REQUIRE(items.size() == 1);
        REQUIRE(items[0]->getString("id") == "2");
    }
}

}
<commit_msg>fix testcase<commit_after>#define CATCH_CONFIG_MAIN
#include "catch/catch.hpp"
#include "tangram.h"
#include "platform.h"
#include "debug.h"
#include "scene/scene.h"
#include "style/style.h"
#include "style/textStyle.h"
#include "labels/labels.h"

#include "view/view.h"
#include "tile/tile.h"

namespace Tangram {

glm::vec2 screenSize(256.f, 256.f);
TextBuffer dummy(nullptr);

std::unique_ptr<TextLabel> makeLabel(Label::Transform _transform, Label::Type _type, std::string id) {
    Label::Options options;
    options.color = 0xff;
    options.offset = {0.0f, 0.0f};
    options.properties = std::make_shared<Properties>();
    options.properties->add("id", id);
    options.interactive = true;

    return std::unique_ptr<TextLabel>(new TextLabel("label",
                                                    _transform, _type, {10, 10},
                                                    dummy, {0, 0}, options));
}

TEST_CASE("Test getFeaturesAtPoint", "[Labels][FeaturePicking]") {
    std::unique_ptr<Labels> labels(new Labels());

    View view(256, 256);
    view.setPosition(0, 0);
    view.setZoom(0);
    view.update();

    auto labelMesh = std::unique_ptr<LabelMesh>(new LabelMesh(nullptr, 0));
    auto textStyle = std::unique_ptr<TextStyle>(new TextStyle("test", nullptr));

    labelMesh->addLabel(makeLabel(glm::vec2{0,0}, Label::Type::point, "0"));
    labelMesh->addLabel(makeLabel(glm::vec2{1,-1}, Label::Type::point, "1"));
    labelMesh->addLabel(makeLabel(glm::vec2{1,1}, Label::Type::point, "2"));

    std::shared_ptr<Tile> tile(new Tile({0,0,0}, view.getMapProjection()));
    tile->getMesh(*textStyle.get()) = std::move(labelMesh);
    tile->update(0, view);

    std::vector<std::unique_ptr<Style>> styles;
    styles.push_back(std::move(textStyle));

    std::vector<std::shared_ptr<Tile>> tiles;

    tiles.push_back(tile);
    {
        auto& items = labels->getFeaturesAtPoint(view, 0, styles, tiles, 128, 128, false);
        REQUIRE(items.size() == 1);
        REQUIRE(items[0]->getString("id") == "0");
    }
    {
        auto& items = labels->getFeaturesAtPoint(view, 0, styles, tiles, 256, 256, false);
        REQUIRE(items.size() == 1);
        REQUIRE(items[0]->getString("id") == "1");
    }

    {
        auto& items = labels->getFeaturesAtPoint(view, 0, styles, tiles, 256, 0, false);
        REQUIRE(items.size() == 1);
        REQUIRE(items[0]->getString("id") == "2");
    }
}

}
<|endoftext|>