<commit_before>/*
 *
 * Copyright 2015, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <condition_variable>
#include <memory>
#include <mutex>
#include <sstream>

#include <signal.h>
#include <unistd.h>

#include <gflags/gflags.h>
#include <grpc/grpc.h>
#include <grpc/support/log.h>
#include <grpc++/config.h>
#include <grpc++/server.h>
#include <grpc++/server_builder.h>
#include <grpc++/server_context.h>
#include <grpc++/server_credentials.h>
#include <grpc++/status.h>
#include "test/core/util/reconnect_server.h"
#include "test/cpp/util/test_config.h"
#include "test/proto/test.grpc.pb.h"
#include "test/proto/empty.grpc.pb.h"
#include "test/proto/messages.grpc.pb.h"

DEFINE_int32(control_port, 0, "Server port for controlling the server.");
DEFINE_int32(retry_port, 0,
             "Server port for raw tcp connections. All incoming "
             "connections will be closed immediately.");

using grpc::Server;
using grpc::ServerBuilder;
using grpc::ServerContext;
using grpc::ServerCredentials;
using grpc::ServerReader;
using grpc::ServerReaderWriter;
using grpc::ServerWriter;
using grpc::SslServerCredentialsOptions;
using grpc::Status;
using grpc::testing::Empty;
using grpc::testing::ReconnectService;
using grpc::testing::ReconnectInfo;

static bool got_sigint = false;

class ReconnectServiceImpl : public ReconnectService::Service {
 public:
  explicit ReconnectServiceImpl(int retry_port)
      : retry_port_(retry_port), serving_(false), shutdown_(false) {
    reconnect_server_init(&tcp_server_);
  }

  ~ReconnectServiceImpl() {
    if (tcp_server_.tcp_server) {
      reconnect_server_destroy(&tcp_server_);
    }
  }

  void Poll(int seconds) { reconnect_server_poll(&tcp_server_, seconds); }

  Status Start(ServerContext* context, const Empty* request, Empty* response) {
    std::unique_lock<std::mutex> lock(mu_);
    while (serving_ && !shutdown_) {
      cv_.wait(lock);
    }
    if (shutdown_) {
      return Status(grpc::StatusCode::UNAVAILABLE, "shutting down");
    }
    serving_ = true;
    lock.unlock();

    if (!tcp_server_.tcp_server) {
      reconnect_server_start(&tcp_server_, retry_port_);
    } else {
      reconnect_server_clear_timestamps(&tcp_server_);
    }
    return Status::OK;
  }

  Status Stop(ServerContext* context, const Empty* request,
              ReconnectInfo* response) {
    // extract timestamps and set response
    Verify(response);
    reconnect_server_clear_timestamps(&tcp_server_);
    {
      std::lock_guard<std::mutex> lock(mu_);
      serving_ = false;
    }
    cv_.notify_one();
    return Status::OK;
  }

  void Verify(ReconnectInfo* response) {
    double expected_backoff = 1000.0;
    const double kTransmissionDelay = 100.0;
    const double kBackoffMultiplier = 1.6;
    const double kJitterFactor = 0.2;
    const int kMaxBackoffMs = 120 * 1000;
    bool passed = true;
    for (timestamp_list* cur = tcp_server_.head; cur && cur->next;
         cur = cur->next) {
      double backoff = gpr_time_to_millis(
          gpr_time_sub(cur->next->timestamp, cur->timestamp));
      double min_backoff = expected_backoff * (1 - kJitterFactor);
      double max_backoff = expected_backoff * (1 + kJitterFactor);
      if (backoff < min_backoff - kTransmissionDelay ||
          backoff > max_backoff + kTransmissionDelay) {
        passed = false;
      }
      response->add_backoff_ms(static_cast<gpr_int32>(backoff));
      expected_backoff *= kBackoffMultiplier;
      expected_backoff =
          expected_backoff > kMaxBackoffMs ? kMaxBackoffMs : expected_backoff;
    }
    response->set_passed(passed);
  }

  void Shutdown() {
    {
      std::lock_guard<std::mutex> lock(mu_);
      shutdown_ = true;
    }
    cv_.notify_all();
  }

 private:
  int retry_port_;
  reconnect_server tcp_server_;
  bool serving_;
  bool shutdown_;
  std::mutex mu_;
  std::condition_variable cv_;
};

void RunServer() {
  std::ostringstream server_address;
  server_address << "0.0.0.0:" << FLAGS_control_port;
  ReconnectServiceImpl service(FLAGS_retry_port);

  ServerBuilder builder;
  builder.RegisterService(&service);
  builder.AddListeningPort(server_address.str(),
                           grpc::InsecureServerCredentials());
  std::unique_ptr<Server> server(builder.BuildAndStart());
  gpr_log(GPR_INFO, "Server listening on %s", server_address.str().c_str());
  while (!got_sigint) {
    service.Poll(5);
  }
  service.Shutdown();
}

static void sigint_handler(int x) { got_sigint = true; }

int main(int argc, char** argv) {
  grpc::testing::InitTest(&argc, &argv, true);
  signal(SIGINT, sigint_handler);

  GPR_ASSERT(FLAGS_control_port != 0);
  GPR_ASSERT(FLAGS_retry_port != 0);
  RunServer();

  return 0;
}
<commit_msg>notify inside lock<commit_after>/*
 *
 * Copyright 2015, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <condition_variable>
#include <memory>
#include <mutex>
#include <sstream>

#include <signal.h>
#include <unistd.h>

#include <gflags/gflags.h>
#include <grpc/grpc.h>
#include <grpc/support/log.h>
#include <grpc++/config.h>
#include <grpc++/server.h>
#include <grpc++/server_builder.h>
#include <grpc++/server_context.h>
#include <grpc++/server_credentials.h>
#include <grpc++/status.h>
#include "test/core/util/reconnect_server.h"
#include "test/cpp/util/test_config.h"
#include "test/proto/test.grpc.pb.h"
#include "test/proto/empty.grpc.pb.h"
#include "test/proto/messages.grpc.pb.h"

DEFINE_int32(control_port, 0, "Server port for controlling the server.");
DEFINE_int32(retry_port, 0,
             "Server port for raw tcp connections. All incoming "
             "connections will be closed immediately.");

using grpc::Server;
using grpc::ServerBuilder;
using grpc::ServerContext;
using grpc::ServerCredentials;
using grpc::ServerReader;
using grpc::ServerReaderWriter;
using grpc::ServerWriter;
using grpc::SslServerCredentialsOptions;
using grpc::Status;
using grpc::testing::Empty;
using grpc::testing::ReconnectService;
using grpc::testing::ReconnectInfo;

static bool got_sigint = false;

class ReconnectServiceImpl : public ReconnectService::Service {
 public:
  explicit ReconnectServiceImpl(int retry_port)
      : retry_port_(retry_port), serving_(false), shutdown_(false) {
    reconnect_server_init(&tcp_server_);
  }

  ~ReconnectServiceImpl() {
    if (tcp_server_.tcp_server) {
      reconnect_server_destroy(&tcp_server_);
    }
  }

  void Poll(int seconds) { reconnect_server_poll(&tcp_server_, seconds); }

  Status Start(ServerContext* context, const Empty* request, Empty* response) {
    std::unique_lock<std::mutex> lock(mu_);
    while (serving_ && !shutdown_) {
      cv_.wait(lock);
    }
    if (shutdown_) {
      return Status(grpc::StatusCode::UNAVAILABLE, "shutting down");
    }
    serving_ = true;
    lock.unlock();

    if (!tcp_server_.tcp_server) {
      reconnect_server_start(&tcp_server_, retry_port_);
    } else {
      reconnect_server_clear_timestamps(&tcp_server_);
    }
    return Status::OK;
  }

  Status Stop(ServerContext* context, const Empty* request,
              ReconnectInfo* response) {
    // extract timestamps and set response
    Verify(response);
    reconnect_server_clear_timestamps(&tcp_server_);
    {
      std::lock_guard<std::mutex> lock(mu_);
      serving_ = false;
      cv_.notify_one();
    }
    return Status::OK;
  }

  void Verify(ReconnectInfo* response) {
    double expected_backoff = 1000.0;
    const double kTransmissionDelay = 100.0;
    const double kBackoffMultiplier = 1.6;
    const double kJitterFactor = 0.2;
    const int kMaxBackoffMs = 120 * 1000;
    bool passed = true;
    for (timestamp_list* cur = tcp_server_.head; cur && cur->next;
         cur = cur->next) {
      double backoff = gpr_time_to_millis(
          gpr_time_sub(cur->next->timestamp, cur->timestamp));
      double min_backoff = expected_backoff * (1 - kJitterFactor);
      double max_backoff = expected_backoff * (1 + kJitterFactor);
      if (backoff < min_backoff - kTransmissionDelay ||
          backoff > max_backoff + kTransmissionDelay) {
        passed = false;
      }
      response->add_backoff_ms(static_cast<gpr_int32>(backoff));
      expected_backoff *= kBackoffMultiplier;
      expected_backoff =
          expected_backoff > kMaxBackoffMs ? kMaxBackoffMs : expected_backoff;
    }
    response->set_passed(passed);
  }

  void Shutdown() {
    std::lock_guard<std::mutex> lock(mu_);
    shutdown_ = true;
    cv_.notify_all();
  }

 private:
  int retry_port_;
  reconnect_server tcp_server_;
  bool serving_;
  bool shutdown_;
  std::mutex mu_;
  std::condition_variable cv_;
};

void RunServer() {
  std::ostringstream server_address;
  server_address << "0.0.0.0:" << FLAGS_control_port;
  ReconnectServiceImpl service(FLAGS_retry_port);

  ServerBuilder builder;
  builder.RegisterService(&service);
  builder.AddListeningPort(server_address.str(),
                           grpc::InsecureServerCredentials());
  std::unique_ptr<Server> server(builder.BuildAndStart());
  gpr_log(GPR_INFO, "Server listening on %s", server_address.str().c_str());
  while (!got_sigint) {
    service.Poll(5);
  }
  service.Shutdown();
}

static void sigint_handler(int x) { got_sigint = true; }

int main(int argc, char** argv) {
  grpc::testing::InitTest(&argc, &argv, true);
  signal(SIGINT, sigint_handler);

  GPR_ASSERT(FLAGS_control_port != 0);
  GPR_ASSERT(FLAGS_retry_port != 0);
  RunServer();

  return 0;
}
<|endoftext|>


<commit_before>#include <iostream>
#include <seqan/graph_msa.h>
#include <seqan/index.h>

using namespace seqan;
using namespace std;

int main(int argc, const char *argv[])
{
	typedef String<Dna> TSequence;
	typedef StringSet<TSequence> TSequenceSet;

	TSequenceSet seqs;
	for(int i = 1; i < argc; ++i)
		appendValue(seqs, String<Dna, FileReader<Fasta> >(argv[i]));

	typedef Fragment<> TMatch;
	String<TMatch> matches;

	typedef Index<TSequenceSet> TIndex;
	TIndex index(seqs);

	Iterator<TIndex,MUMs>::Type mumIt(index, 5);
	String<SAValue<TIndex>::Type> occs;

	while (!atEnd(mumIt))
	{
		occs = getOccurrences(mumIt);
		for(unsigned i = 0; i < length(occs); ++i)
		{
			for(unsigned j = i+1; j < length(occs); ++j)
			{
				TMatch m(getValueI1(occs[i]),getValueI2(occs[i]),getValueI1(occs[j]), getValueI2(occs[j]),repLength(mumIt));
				appendValue(matches,m);
			}
		}
		++mumIt;
	}

	typedef StringSet<TSequence,Dependent<> > TDepSequenceSet;
	typedef Graph<Alignment<TDepSequenceSet> > TAlignmentGraph;
	TAlignmentGraph g(seqs);
	matchRefinement(matches, seqs, g);
	clear(matches);

	tripletLibraryExtension(g);

	typedef String<double> TDistanceMatrix;
	TDistanceMatrix distanceMatrix;
	getDistanceMatrix(g, distanceMatrix,KmerDistance());

	typedef Graph<Tree<double> > TGuideTree;
	TGuideTree  guideTree;
	upgmaTree(distanceMatrix, guideTree);

	TAlignmentGraph gOut(seqs);
	progressiveAlignment(g, guideTree, gOut);
	cout << gOut;

	return 0;
}
<commit_msg><commit_after>#include <iostream>
#include <seqan/graph_msa.h>
#include <seqan/index.h>

using namespace seqan;
using namespace std;

int main(int argc, const char *argv[])
{
	typedef String<Dna> TSequence;
	typedef StringSet<TSequence> TSequenceSet;

	if(argc < 3)
	{
		::std::cout << "\nAt least two Fasta sequence files need to be specified.\n\n";
		::std::cout << "Usage: ./segmentalignment <seq1.fa> <seq2.fa> ...\n\n";
		return 1;
	}

	TSequenceSet seqs;
	for(int i = 1; i < argc; ++i)
		appendValue(seqs, String<Dna, FileReader<Fasta> >(argv[i]));

	typedef Fragment<> TMatch;
	String<TMatch> matches;

	typedef Index<TSequenceSet> TIndex;
	TIndex index(seqs);

	Iterator<TIndex,MUMs>::Type mumIt(index, 5);
	String<SAValue<TIndex>::Type> occs;

	while (!atEnd(mumIt))
	{
		occs = getOccurrences(mumIt);
		for(unsigned i = 0; i < length(occs); ++i)
		{
			for(unsigned j = i+1; j < length(occs); ++j)
			{
				TMatch m(getValueI1(occs[i]),getValueI2(occs[i]),getValueI1(occs[j]), getValueI2(occs[j]),repLength(mumIt));
				appendValue(matches,m);
			}
		}
		++mumIt;
	}

	typedef StringSet<TSequence,Dependent<> > TDepSequenceSet;
	typedef Graph<Alignment<TDepSequenceSet> > TAlignmentGraph;
	TAlignmentGraph g(seqs);
	matchRefinement(matches, seqs, g);
	clear(matches);

	tripletLibraryExtension(g);

	typedef String<double> TDistanceMatrix;
	TDistanceMatrix distanceMatrix;
	getDistanceMatrix(g, distanceMatrix,KmerDistance());

	typedef Graph<Tree<double> > TGuideTree;
	TGuideTree  guideTree;
	upgmaTree(distanceMatrix, guideTree);

	TAlignmentGraph gOut(seqs);
	progressiveAlignment(g, guideTree, gOut);
	cout << gOut;

	return 0;
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "rpc/rpc-mgr.inline.h"

#include "common/init.h"
#include "exec/kudu-util.h"
#include "kudu/rpc/rpc_context.h"
#include "kudu/rpc/rpc_controller.h"
#include "kudu/rpc/rpc_header.pb.h"
#include "kudu/rpc/rpc_sidecar.h"
#include "kudu/util/monotime.h"
#include "kudu/util/status.h"
#include "rpc/auth-provider.h"
#include "testutil/gtest-util.h"
#include "testutil/mini-kdc-wrapper.h"
#include "util/counting-barrier.h"
#include "util/network-util.h"
#include "util/test-info.h"

#include "gen-cpp/rpc_test.proxy.h"
#include "gen-cpp/rpc_test.service.h"

#include "common/names.h"

using kudu::rpc::ErrorStatusPB;
using kudu::rpc::ServiceIf;
using kudu::rpc::RpcController;
using kudu::rpc::RpcContext;
using kudu::rpc::RpcSidecar;
using kudu::MonoDelta;
using kudu::Slice;

using namespace std;

DECLARE_int32(num_reactor_threads);
DECLARE_int32(num_acceptor_threads);
DECLARE_string(hostname);

// The path of the current executable file that is required for passing into the SASL
// library as the 'application name'.
static string current_executable_path;

namespace impala {

static int32_t SERVICE_PORT = FindUnusedEphemeralPort(nullptr);

int GetServerPort() {
  int port = FindUnusedEphemeralPort(nullptr);
  EXPECT_FALSE(port == -1);
  return port;
}

static int kdc_port = GetServerPort();

#define PAYLOAD_SIZE (4096)

template <class T> class RpcMgrTestBase : public T {
 protected:
  TNetworkAddress krpc_address_;
  RpcMgr rpc_mgr_;

  virtual void SetUp() {
    IpAddr ip;
    ASSERT_OK(HostnameToIpAddr(FLAGS_hostname, &ip));
    krpc_address_ = MakeNetworkAddress(ip, SERVICE_PORT);
    ASSERT_OK(rpc_mgr_.Init());
  }

  virtual void TearDown() {
    rpc_mgr_.Shutdown();
  }

  // Utility function to initialize the parameter for ScanMem RPC.
  // Picks a random value and fills 'payload_' with it. Adds 'payload_' as a sidecar
  // to 'controller'. Also sets up 'request' with the random value and index of the
  // sidecar.
  void SetupScanMemRequest(ScanMemRequestPB* request, RpcController* controller) {
    int32_t pattern = random();
    for (int i = 0; i < PAYLOAD_SIZE / sizeof(int32_t); ++i) payload_[i] = pattern;
    int idx;
    Slice slice(reinterpret_cast<const uint8_t*>(payload_), PAYLOAD_SIZE);
    controller->AddOutboundSidecar(RpcSidecar::FromSlice(slice), &idx);
    request->set_pattern(pattern);
    request->set_sidecar_idx(idx);
  }

 private:
  int32_t payload_[PAYLOAD_SIZE];
};

// For tests that do not require kerberized testing, we use RpcTest.
class RpcMgrTest : public RpcMgrTestBase<testing::Test> {
  virtual void SetUp() {
    RpcMgrTestBase::SetUp();
  }

  virtual void TearDown() {
    RpcMgrTestBase::TearDown();
  }
};

class RpcMgrKerberizedTest :
    public RpcMgrTestBase<testing::TestWithParam<KerberosSwitch> > {
  virtual void SetUp() {
    IpAddr ip;
    ASSERT_OK(HostnameToIpAddr(FLAGS_hostname, &ip));
    string spn = Substitute("impala-test/$0", ip);

    kdc_wrapper_.reset(new MiniKdcWrapper(
        std::move(spn), "KRBTEST.COM", "24h", "7d", kdc_port));
    DCHECK(kdc_wrapper_.get() != nullptr);

    ASSERT_OK(kdc_wrapper_->SetupAndStartMiniKDC(GetParam()));
    ASSERT_OK(InitAuth(current_executable_path));

    RpcMgrTestBase::SetUp();
  }

  virtual void TearDown() {
    ASSERT_OK(kdc_wrapper_->TearDownMiniKDC(GetParam()));
    RpcMgrTestBase::TearDown();
  }

 private:
  boost::scoped_ptr<MiniKdcWrapper> kdc_wrapper_;
};

typedef std::function<void(RpcContext*)> ServiceCB;

class PingServiceImpl : public PingServiceIf {
 public:
  // 'cb' is a callback used by tests to inject custom behaviour into the RPC handler.
  PingServiceImpl(const scoped_refptr<kudu::MetricEntity>& entity,
      const scoped_refptr<kudu::rpc::ResultTracker> tracker,
      ServiceCB cb = [](RpcContext* ctx) { ctx->RespondSuccess(); })
    : PingServiceIf(entity, tracker), cb_(cb) {}

  virtual void Ping(
      const PingRequestPB* request, PingResponsePB* response, RpcContext* context) {
    response->set_int_response(42);
    cb_(context);
  }

 private:
  ServiceCB cb_;
};

class ScanMemServiceImpl : public ScanMemServiceIf {
 public:
  ScanMemServiceImpl(const scoped_refptr<kudu::MetricEntity>& entity,
      const scoped_refptr<kudu::rpc::ResultTracker> tracker)
    : ScanMemServiceIf(entity, tracker) {
  }

  // The request comes with an int 'pattern' and a payload of int array sent with
  // sidecar. Scan the array to make sure every element matches 'pattern'.
  virtual void ScanMem(const ScanMemRequestPB* request, ScanMemResponsePB* response,
      RpcContext* context) {
    int32_t pattern = request->pattern();
    Slice payload;
    ASSERT_OK(
        FromKuduStatus(context->GetInboundSidecar(request->sidecar_idx(), &payload)));
    ASSERT_EQ(payload.size() % sizeof(int32_t), 0);

    const int32_t* v = reinterpret_cast<const int32_t*>(payload.data());
    for (int i = 0; i < payload.size() / sizeof(int32_t); ++i) {
      int32_t val = v[i];
      if (val != pattern) {
        context->RespondFailure(kudu::Status::Corruption(
            Substitute("Expecting $1; Found $2", pattern, val)));
        return;
      }
    }
    context->RespondSuccess();
  }
};

INSTANTIATE_TEST_CASE_P(KerberosOnAndOff,
                        RpcMgrKerberizedTest,
                        ::testing::Values(KERBEROS_OFF,
                                          USE_KUDU_KERBEROS,
                                          USE_IMPALA_KERBEROS));

TEST_P(RpcMgrKerberizedTest, MultipleServices) {
  // Test that a service can be started, and will respond to requests.
  unique_ptr<ServiceIf> ping_impl(
      new PingServiceImpl(rpc_mgr_.metric_entity(), rpc_mgr_.result_tracker()));
  ASSERT_OK(rpc_mgr_.RegisterService(10, 10, move(ping_impl)));

  // Test that a second service, that verifies the RPC payload is not corrupted,
  // can be started.
  unique_ptr<ServiceIf> scan_mem_impl(
      new ScanMemServiceImpl(rpc_mgr_.metric_entity(), rpc_mgr_.result_tracker()));
  ASSERT_OK(rpc_mgr_.RegisterService(10, 10, move(scan_mem_impl)));

  FLAGS_num_acceptor_threads = 2;
  FLAGS_num_reactor_threads = 10;
  ASSERT_OK(rpc_mgr_.StartServices(krpc_address_));

  unique_ptr<PingServiceProxy> ping_proxy;
  ASSERT_OK(rpc_mgr_.GetProxy<PingServiceProxy>(krpc_address_, &ping_proxy));

  unique_ptr<ScanMemServiceProxy> scan_mem_proxy;
  ASSERT_OK(rpc_mgr_.GetProxy<ScanMemServiceProxy>(krpc_address_, &scan_mem_proxy));

  RpcController controller;
  srand(0);
  // Randomly invoke either services to make sure a RpcMgr can host multiple
  // services at the same time.
  for (int i = 0; i < 100; ++i) {
    controller.Reset();
    if (random() % 2 == 0) {
      PingRequestPB request;
      PingResponsePB response;
      kudu::Status status = ping_proxy->Ping(request, &response, &controller);
      ASSERT_TRUE(status.ok());
      ASSERT_EQ(response.int_response(), 42);
    } else {
      ScanMemRequestPB request;
      ScanMemResponsePB response;
      SetupScanMemRequest(&request, &controller);
      kudu::Status status = scan_mem_proxy->ScanMem(request, &response, &controller);
      ASSERT_TRUE(status.ok());
    }
  }
}

TEST_F(RpcMgrTest, SlowCallback) {

  // Use a callback which is slow to respond.
  auto slow_cb = [](RpcContext* ctx) {
    SleepForMs(300);
    ctx->RespondSuccess();
  };

  // Test a service which is slow to respond and has a short queue.
  // Set a timeout on the client side. Expect either a client timeout
  // or the service queue filling up.
  unique_ptr<ServiceIf> impl(
      new PingServiceImpl(rpc_mgr_.metric_entity(), rpc_mgr_.result_tracker(), slow_cb));
  const int num_service_threads = 1;
  const int queue_size = 3;
  ASSERT_OK(rpc_mgr_.RegisterService(num_service_threads, queue_size, move(impl)));

  FLAGS_num_acceptor_threads = 2;
  FLAGS_num_reactor_threads = 10;
  ASSERT_OK(rpc_mgr_.StartServices(krpc_address_));

  unique_ptr<PingServiceProxy> proxy;
  ASSERT_OK(rpc_mgr_.GetProxy<PingServiceProxy>(krpc_address_, &proxy));

  PingRequestPB request;
  PingResponsePB response;
  RpcController controller;
  for (int i = 0; i < 100; ++i) {
    controller.Reset();
    controller.set_timeout(MonoDelta::FromMilliseconds(50));
    kudu::Status status = proxy->Ping(request, &response, &controller);
    ASSERT_TRUE(status.IsTimedOut() || RpcMgr::IsServerTooBusy(controller));
  }
}

TEST_F(RpcMgrTest, AsyncCall) {
  unique_ptr<ServiceIf> scan_mem_impl(
      new ScanMemServiceImpl(rpc_mgr_.metric_entity(), rpc_mgr_.result_tracker()));
  ASSERT_OK(rpc_mgr_.RegisterService(10, 10, move(scan_mem_impl)));

  unique_ptr<ScanMemServiceProxy> scan_mem_proxy;
  ASSERT_OK(rpc_mgr_.GetProxy<ScanMemServiceProxy>(krpc_address_, &scan_mem_proxy));

  FLAGS_num_acceptor_threads = 2;
  FLAGS_num_reactor_threads = 10;
  ASSERT_OK(rpc_mgr_.StartServices(krpc_address_));

  RpcController controller;
  srand(0);
  for (int i = 0; i < 10; ++i) {
    controller.Reset();
    ScanMemRequestPB request;
    ScanMemResponsePB response;
    SetupScanMemRequest(&request, &controller);
    CountingBarrier barrier(1);
    scan_mem_proxy->ScanMemAsync(request, &response, &controller,
        [barrier_ptr = &barrier]() { barrier_ptr->Notify(); });
    // TODO: Inject random cancellation here.
    barrier.Wait();
    ASSERT_TRUE(controller.status().ok()) << controller.status().ToString();
  }
}

} // namespace impala

int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
  impala::InitCommonRuntime(argc, argv, false, impala::TestInfo::BE_TEST);

  // Fill in the path of the current binary for use by the tests.
  current_executable_path = argv[0];
  return RUN_ALL_TESTS();
}
<commit_msg>IMPALA-6268: KerberosOnAndOff/RpcMgrKerberizedTest.MultipleServices failing<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "rpc/rpc-mgr.inline.h"

#include "common/init.h"
#include "exec/kudu-util.h"
#include "kudu/rpc/rpc_context.h"
#include "kudu/rpc/rpc_controller.h"
#include "kudu/rpc/rpc_header.pb.h"
#include "kudu/rpc/rpc_sidecar.h"
#include "kudu/util/monotime.h"
#include "kudu/util/status.h"
#include "rpc/auth-provider.h"
#include "testutil/gtest-util.h"
#include "testutil/mini-kdc-wrapper.h"
#include "util/counting-barrier.h"
#include "util/network-util.h"
#include "util/test-info.h"

#include "gen-cpp/rpc_test.proxy.h"
#include "gen-cpp/rpc_test.service.h"

#include "common/names.h"

using kudu::rpc::ErrorStatusPB;
using kudu::rpc::ServiceIf;
using kudu::rpc::RpcController;
using kudu::rpc::RpcContext;
using kudu::rpc::RpcSidecar;
using kudu::MonoDelta;
using kudu::Slice;

using namespace std;

DECLARE_int32(num_reactor_threads);
DECLARE_int32(num_acceptor_threads);
DECLARE_string(hostname);

// The path of the current executable file that is required for passing into the SASL
// library as the 'application name'.
static string current_executable_path;

namespace impala {

static int32_t SERVICE_PORT = FindUnusedEphemeralPort(nullptr);

int GetServerPort() {
  int port = FindUnusedEphemeralPort(nullptr);
  EXPECT_FALSE(port == -1);
  return port;
}

static int kdc_port = GetServerPort();

#define PAYLOAD_SIZE (4096)

template <class T> class RpcMgrTestBase : public T {
 protected:
  TNetworkAddress krpc_address_;
  RpcMgr rpc_mgr_;

  virtual void SetUp() {
    IpAddr ip;
    ASSERT_OK(HostnameToIpAddr(FLAGS_hostname, &ip));
    krpc_address_ = MakeNetworkAddress(ip, SERVICE_PORT);
    ASSERT_OK(rpc_mgr_.Init());
  }

  virtual void TearDown() {
    rpc_mgr_.Shutdown();
  }

  // Utility function to initialize the parameter for ScanMem RPC.
  // Picks a random value and fills 'payload_' with it. Adds 'payload_' as a sidecar
  // to 'controller'. Also sets up 'request' with the random value and index of the
  // sidecar.
  void SetupScanMemRequest(ScanMemRequestPB* request, RpcController* controller) {
    int32_t pattern = random();
    for (int i = 0; i < PAYLOAD_SIZE / sizeof(int32_t); ++i) payload_[i] = pattern;
    int idx;
    Slice slice(reinterpret_cast<const uint8_t*>(payload_), PAYLOAD_SIZE);
    controller->AddOutboundSidecar(RpcSidecar::FromSlice(slice), &idx);
    request->set_pattern(pattern);
    request->set_sidecar_idx(idx);
  }

 private:
  int32_t payload_[PAYLOAD_SIZE];
};

// For tests that do not require kerberized testing, we use RpcTest.
class RpcMgrTest : public RpcMgrTestBase<testing::Test> {
  virtual void SetUp() {
    RpcMgrTestBase::SetUp();
  }

  virtual void TearDown() {
    RpcMgrTestBase::TearDown();
  }
};

class RpcMgrKerberizedTest :
    public RpcMgrTestBase<testing::TestWithParam<KerberosSwitch> > {
  virtual void SetUp() {
    IpAddr ip;
    ASSERT_OK(HostnameToIpAddr(FLAGS_hostname, &ip));
    string spn = Substitute("impala-test/$0", ip);

    kdc_wrapper_.reset(new MiniKdcWrapper(
        std::move(spn), "KRBTEST.COM", "24h", "7d", kdc_port));
    DCHECK(kdc_wrapper_.get() != nullptr);

    ASSERT_OK(kdc_wrapper_->SetupAndStartMiniKDC(GetParam()));
    ASSERT_OK(InitAuth(current_executable_path));

    RpcMgrTestBase::SetUp();
  }

  virtual void TearDown() {
    ASSERT_OK(kdc_wrapper_->TearDownMiniKDC(GetParam()));
    RpcMgrTestBase::TearDown();
  }

 private:
  boost::scoped_ptr<MiniKdcWrapper> kdc_wrapper_;
};

typedef std::function<void(RpcContext*)> ServiceCB;

class PingServiceImpl : public PingServiceIf {
 public:
  // 'cb' is a callback used by tests to inject custom behaviour into the RPC handler.
  PingServiceImpl(const scoped_refptr<kudu::MetricEntity>& entity,
      const scoped_refptr<kudu::rpc::ResultTracker> tracker,
      ServiceCB cb = [](RpcContext* ctx) { ctx->RespondSuccess(); })
    : PingServiceIf(entity, tracker), cb_(cb) {}

  virtual void Ping(
      const PingRequestPB* request, PingResponsePB* response, RpcContext* context) {
    response->set_int_response(42);
    cb_(context);
  }

 private:
  ServiceCB cb_;
};

class ScanMemServiceImpl : public ScanMemServiceIf {
 public:
  ScanMemServiceImpl(const scoped_refptr<kudu::MetricEntity>& entity,
      const scoped_refptr<kudu::rpc::ResultTracker> tracker)
    : ScanMemServiceIf(entity, tracker) {
  }

  // The request comes with an int 'pattern' and a payload of int array sent with
  // sidecar. Scan the array to make sure every element matches 'pattern'.
  virtual void ScanMem(const ScanMemRequestPB* request, ScanMemResponsePB* response,
      RpcContext* context) {
    int32_t pattern = request->pattern();
    Slice payload;
    ASSERT_OK(
        FromKuduStatus(context->GetInboundSidecar(request->sidecar_idx(), &payload)));
    ASSERT_EQ(payload.size() % sizeof(int32_t), 0);

    const int32_t* v = reinterpret_cast<const int32_t*>(payload.data());
    for (int i = 0; i < payload.size() / sizeof(int32_t); ++i) {
      int32_t val = v[i];
      if (val != pattern) {
        context->RespondFailure(kudu::Status::Corruption(
            Substitute("Expecting $1; Found $2", pattern, val)));
        return;
      }
    }
    context->RespondSuccess();
  }
};

// TODO: Disabled 'USE_KUDU_KERBEROS' and 'USE_IMPALA_KERBEROS' due to IMPALA-6268.
// Reenable after fixing.
INSTANTIATE_TEST_CASE_P(KerberosOnAndOff,
                        RpcMgrKerberizedTest,
                        ::testing::Values(KERBEROS_OFF));

TEST_P(RpcMgrKerberizedTest, MultipleServices) {
  // Test that a service can be started, and will respond to requests.
  unique_ptr<ServiceIf> ping_impl(
      new PingServiceImpl(rpc_mgr_.metric_entity(), rpc_mgr_.result_tracker()));
  ASSERT_OK(rpc_mgr_.RegisterService(10, 10, move(ping_impl)));

  // Test that a second service, that verifies the RPC payload is not corrupted,
  // can be started.
  unique_ptr<ServiceIf> scan_mem_impl(
      new ScanMemServiceImpl(rpc_mgr_.metric_entity(), rpc_mgr_.result_tracker()));
  ASSERT_OK(rpc_mgr_.RegisterService(10, 10, move(scan_mem_impl)));

  FLAGS_num_acceptor_threads = 2;
  FLAGS_num_reactor_threads = 10;
  ASSERT_OK(rpc_mgr_.StartServices(krpc_address_));

  unique_ptr<PingServiceProxy> ping_proxy;
  ASSERT_OK(rpc_mgr_.GetProxy<PingServiceProxy>(krpc_address_, &ping_proxy));

  unique_ptr<ScanMemServiceProxy> scan_mem_proxy;
  ASSERT_OK(rpc_mgr_.GetProxy<ScanMemServiceProxy>(krpc_address_, &scan_mem_proxy));

  RpcController controller;
  srand(0);
  // Randomly invoke either services to make sure a RpcMgr can host multiple
  // services at the same time.
  for (int i = 0; i < 100; ++i) {
    controller.Reset();
    if (random() % 2 == 0) {
      PingRequestPB request;
      PingResponsePB response;
      kudu::Status status = ping_proxy->Ping(request, &response, &controller);
      ASSERT_TRUE(status.ok());
      ASSERT_EQ(response.int_response(), 42);
    } else {
      ScanMemRequestPB request;
      ScanMemResponsePB response;
      SetupScanMemRequest(&request, &controller);
      kudu::Status status = scan_mem_proxy->ScanMem(request, &response, &controller);
      ASSERT_TRUE(status.ok());
    }
  }
}

TEST_F(RpcMgrTest, SlowCallback) {

  // Use a callback which is slow to respond.
  auto slow_cb = [](RpcContext* ctx) {
    SleepForMs(300);
    ctx->RespondSuccess();
  };

  // Test a service which is slow to respond and has a short queue.
  // Set a timeout on the client side. Expect either a client timeout
  // or the service queue filling up.
  unique_ptr<ServiceIf> impl(
      new PingServiceImpl(rpc_mgr_.metric_entity(), rpc_mgr_.result_tracker(), slow_cb));
  const int num_service_threads = 1;
  const int queue_size = 3;
  ASSERT_OK(rpc_mgr_.RegisterService(num_service_threads, queue_size, move(impl)));

  FLAGS_num_acceptor_threads = 2;
  FLAGS_num_reactor_threads = 10;
  ASSERT_OK(rpc_mgr_.StartServices(krpc_address_));

  unique_ptr<PingServiceProxy> proxy;
  ASSERT_OK(rpc_mgr_.GetProxy<PingServiceProxy>(krpc_address_, &proxy));

  PingRequestPB request;
  PingResponsePB response;
  RpcController controller;
  for (int i = 0; i < 100; ++i) {
    controller.Reset();
    controller.set_timeout(MonoDelta::FromMilliseconds(50));
    kudu::Status status = proxy->Ping(request, &response, &controller);
    ASSERT_TRUE(status.IsTimedOut() || RpcMgr::IsServerTooBusy(controller));
  }
}

TEST_F(RpcMgrTest, AsyncCall) {
  unique_ptr<ServiceIf> scan_mem_impl(
      new ScanMemServiceImpl(rpc_mgr_.metric_entity(), rpc_mgr_.result_tracker()));
  ASSERT_OK(rpc_mgr_.RegisterService(10, 10, move(scan_mem_impl)));

  unique_ptr<ScanMemServiceProxy> scan_mem_proxy;
  ASSERT_OK(rpc_mgr_.GetProxy<ScanMemServiceProxy>(krpc_address_, &scan_mem_proxy));

  FLAGS_num_acceptor_threads = 2;
  FLAGS_num_reactor_threads = 10;
  ASSERT_OK(rpc_mgr_.StartServices(krpc_address_));

  RpcController controller;
  srand(0);
  for (int i = 0; i < 10; ++i) {
    controller.Reset();
    ScanMemRequestPB request;
    ScanMemResponsePB response;
    SetupScanMemRequest(&request, &controller);
    CountingBarrier barrier(1);
    scan_mem_proxy->ScanMemAsync(request, &response, &controller,
        [barrier_ptr = &barrier]() { barrier_ptr->Notify(); });
    // TODO: Inject random cancellation here.
    barrier.Wait();
    ASSERT_TRUE(controller.status().ok()) << controller.status().ToString();
  }
}

} // namespace impala

int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
  impala::InitCommonRuntime(argc, argv, false, impala::TestInfo::BE_TEST);

  // Fill in the path of the current binary for use by the tests.
  current_executable_path = argv[0];
  return RUN_ALL_TESTS();
}
<|endoftext|>


<commit_before>// Copyright (c) 2021 ASMlover. All rights reserved.
//
//  _____         _             _
// |_   _|_ _  __| |_ __   ___ | | ___
//   | |/ _` |/ _` | '_ \ / _ \| |/ _ \
//   | | (_| | (_| | |_) | (_) | |  __/
//   |_|\__,_|\__,_| .__/ \___/|_|\___|
//                 |_|
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include <iostream>
#include "../lex/lexer.hh"
#include "../object/string_object.hh"
#include "vm.hh"
#include "parser.hh"

namespace tadpole {

GlobalParser::GlobalParser(VM& vm, Lexer& lex) noexcept
  : vm_(vm), lex_(lex) {
}

void GlobalParser::iter_objects(ObjectVisitor&& visitor) {
  for (FunCompiler* c = curr_compiler_; c != nullptr; c = c->enclosing())
    visitor(c->fn());
}

FunctionObject* GlobalParser::compile() {
  FunCompiler compiler;
  init_compiler(&compiler, 0, FunType::TOPLEVEL);

  advance();
  while (!check(TokenKind::TK_EOF))
    declaration();
  FunctionObject* fn = finish_compiler();

  return had_error_ ? nullptr : fn;
}

const ParseRule& GlobalParser::get_rule(TokenKind kind) const noexcept {
#define _RULE(fn) [](GlobalParser& p, bool b) { p.fn(b); }

  static const ParseRule _rules[] = {
    {_RULE(grouping), _RULE(call), Precedence::CALL}, // PUNCTUATOR(LPAREN, "(")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(RPAREN, ")")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(LBRACE, "{")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(RBRACE, "}")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(COMMA, ",")
    {nullptr, _RULE(binary), Precedence::TERM},       // PUNCTUATOR(MINUS, "-")
    {nullptr, _RULE(binary), Precedence::TERM},       // PUNCTUATOR(PLUS, "+")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(SEMI, ";")
    {nullptr, _RULE(binary), Precedence::FACTOR},     // PUNCTUATOR(SLASH, "/")
    {nullptr, _RULE(binary), Precedence::FACTOR},     // PUNCTUATOR(STAR, "*")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(EQ, "=")

    {_RULE(variable), nullptr, Precedence::NONE},     // TOKEN(IDENTIFIER, "Identifier")
    {_RULE(numeric), nullptr, Precedence::NONE},      // TOKEN(NUMERIC, "Numeric")
    {_RULE(string), nullptr, Precedence::NONE},       // TOKEN(STRING, "String")

    {_RULE(literal), nullptr, Precedence::NONE},      // KEYWORD(FALSE, "false")
    {nullptr, nullptr, Precedence::NONE},             // KEYWORD(FN, "fn")
    {_RULE(literal), nullptr, Precedence::NONE},      // KEYWORD(NIL, "nil")
    {_RULE(literal), nullptr, Precedence::NONE},      // KEYWORD(TRUE, "true")
    {nullptr, nullptr, Precedence::NONE},             // KEYWORD(VAR, "var")

    {nullptr, nullptr, Precedence::NONE},             // TOKEN(EOF, "Eof")
    {nullptr, nullptr, Precedence::NONE},             // TOKEN(ERR, "Error")
  };
#undef _RULE

  return _rules[as_type<int>(kind)];
}

void GlobalParser::error_at(const Token& tok, const str_t& msg) noexcept {
  if (panic_mode_)
    return;
  panic_mode_ = true;

  std::cerr
    << "SyntaxError:" << std::endl
    << "  [LINE: " << tok.lineno() << "] ERROR ";
  if (tok.kind() == TokenKind::TK_EOF)
    std::cerr << "at end ";
  else if (tok.kind() == TokenKind::TK_ERR)
    TADPOLE_UNUSED(0);
  else
    std::cerr << "at `" << tok.literal() << "` ";
  std::cerr << ": " << msg << std::endl;
}

void GlobalParser::advance() {
  prev_ = curr_;

  for (;;) {
    curr_ = lex_.next_token();
    if (!check(TokenKind::TK_ERR))
      break;

    error_at_current(curr_.as_string());
  }
}

void GlobalParser::consume(TokenKind kind, const str_t& msg) {
  if (check(kind))
    advance();
  else
    error_at_current(msg);
}

bool GlobalParser::match(TokenKind kind) {
  if (check(kind)) {
    advance();
    return true;
  }
  return false;
}

void GlobalParser::init_compiler(FunCompiler* compiler, int scope_depth, FunType fn_type) {
  StringObject* fn_name{};
  if (fn_type == FunType::FUNCTION)
    fn_name = StringObject::create(prev_.as_string());

  compiler->set_compiler(
      curr_compiler_,
      FunctionObject::create(fn_name),
      fn_type,
      scope_depth);
  curr_compiler_ = compiler;

  curr_compiler_->append_local({Token::make(""), curr_compiler_->scope_depth(), false});
}

FunctionObject* GlobalParser::finish_compiler() {
  emit_return();

  FunctionObject* fn = curr_compiler_->fn();

#if defined(_TADPOLE_DEBUG_VM)
  if (!had_error_)
    curr_chunk()->dis(fn->name_asstr());
#endif

  curr_compiler_ = curr_compiler_->enclosing();
  return fn;
}

void GlobalParser::enter_scope() {
  curr_compiler_->enter_scope();
}

void GlobalParser::leave_scope() {
  curr_compiler_->leave_scope([this](const LocalVar& var) {
        emit_byte(var.is_upvalue ? Code::CLOSE_UPVALUE : Code::POP);
      });
}

u8_t GlobalParser::identifier_constant(const Token& name) noexcept {
  return curr_chunk()->add_constant(StringObject::create(name.as_string()));
}

u8_t GlobalParser::parse_variable(const str_t& msg) {
  consume(TokenKind::TK_IDENTIFIER, msg);

  curr_compiler_->declare_localvar(prev_, [this](const str_t& m) { error(m); });
  if (curr_compiler_->scope_depth() > 0)
    return 0;
  return identifier_constant(prev_);
}

void GlobalParser::mark_initialized() {
  if (curr_compiler_->scope_depth() == 0)
    return;
  curr_compiler_->peek_local().depth = curr_compiler_->scope_depth();
}

void GlobalParser::define_global(u8_t global) {
  if (curr_compiler_->scope_depth() > 0) {
    mark_initialized();
    return;
  }
  emit_bytes(Code::DEF_GLOBAL, global);
}

u8_t GlobalParser::arguments() {
  u8_t nargs = 0;
  if (!check(TokenKind::TK_RPAREN)) {
    do {
      expression();
      ++nargs;

      if (nargs > kMaxArgs)
        error(from_fmt("cannot have more than `%d` arguments", kMaxArgs));
    } while (match(TokenKind::TK_COMMA));
  }
  consume(TokenKind::TK_RPAREN, "expect `;` after function arguments");

  return nargs;
}

void GlobalParser::named_variable(const Token& name, bool can_assign) {
  auto errfn = [this](const str_t& msg) { error(msg); };

  Code getop, setop;
  int arg = curr_compiler_->resolve_local(name, errfn);
  if (arg != -1) {
    getop = Code::GET_LOCAL;
    setop = Code::SET_LOCAL;
  }
  else if (arg = curr_compiler_->resolve_upvalue(name, errfn); arg != -1) {
    getop = Code::GET_UPVALUE;
    setop = Code::SET_UPVALUE;
  }
  else {
    arg = identifier_constant(name);
    getop = Code::GET_GLOBAL;
    setop = Code::SET_GLOBAL;
  }

  if (can_assign && match(TokenKind::TK_EQ)) {
    expression();
    emit_bytes(setop, arg);
  }
  else {
    emit_bytes(getop, arg);
  }
}

void GlobalParser::parse_precedence(Precedence precedence) {
  advance();
  auto& prefix_fn = get_rule(prev_.kind()).prefix;
  if (!prefix_fn) {
    error("expect expression");
    return;
  }

  bool can_assign = precedence <= Precedence::ASSIGN;
  prefix_fn(*this, can_assign);

  while (precedence <= get_rule(curr_.kind()).precedence) {
    advance();
    auto& infix_fn = get_rule(prev_.kind()).infix;

    if (infix_fn)
      infix_fn(*this, can_assign);
  }

  if (can_assign && match(TokenKind::TK_EQ)) {
    error("invalid assignment target");
    expression();
  }
}

void GlobalParser::binary(bool can_assign) {
  TokenKind op = prev_.kind();

  parse_precedence(get_rule(op).precedence + 1);
  switch (op) {
  case TokenKind::TK_PLUS: emit_byte(Code::ADD); break;
  case TokenKind::TK_MINUS: emit_byte(Code::SUB); break;
  case TokenKind::TK_STAR: emit_byte(Code::MUL); break;
  case TokenKind::TK_SLASH: emit_byte(Code::DIV); break;
  }
}

void GlobalParser::call(bool can_assign) {
  emit_byte(Code::CALL_0 + arguments());
}

void GlobalParser::grouping(bool can_assign) {
  expression();
  consume(TokenKind::TK_RPAREN, "expect `)` after grouping expression");
}

void GlobalParser::literal(bool can_assign) {
  switch (prev_.kind()) {
  case TokenKind::KW_NIL: emit_byte(Code::NIL); break;
  case TokenKind::KW_TRUE: emit_byte(Code::TRUE); break;
  case TokenKind::KW_FALSE: emit_byte(Code::FALSE); break;
  }
}

void GlobalParser::variable(bool can_assign) {
  named_variable(prev_, can_assign);
}

void GlobalParser::numeric(bool can_assign) {
  emit_constant(prev_.as_numeric());
}

void GlobalParser::string(bool can_assign) {
  emit_constant(StringObject::create(prev_.as_string()));
}

void GlobalParser::block() {
  while (!check(TokenKind::TK_EOF) && !check(TokenKind::TK_RBRACE))
    declaration();
  consume(TokenKind::TK_RBRACE, "expect `}` after block body");
}

void GlobalParser::function(FunType fn_type) {
  FunCompiler fn_compiler;
  init_compiler(&fn_compiler, 1, fn_type);

  consume(TokenKind::TK_LPAREN, "expect `(` after function name");
  if (!check(TokenKind::TK_RPAREN)) {
    do {
      u8_t param_constant = parse_variable("expect function parameter's name");
      define_global(param_constant);

      curr_compiler_->fn()->inc_arity();
      if (curr_compiler_->fn()->arity() > kMaxArgs)
        error(from_fmt("cannot have more than `%s` parameters", kMaxArgs));
    } while (match(TokenKind::TK_COMMA));
  }
  consume(TokenKind::TK_RPAREN, "expect `)` after function parameters");

  consume(TokenKind::TK_LBRACE, "expect `{` before function body");
  block();

  leave_scope();
  FunctionObject* fn = finish_compiler();

  emit_bytes(Code::CLOSURE, curr_chunk()->add_constant(fn));
  for (sz_t i = 0; i < fn->upvalues_count(); ++i) {
    auto& upvalue = fn_compiler.get_upvalue(i);
    emit_bytes(upvalue.is_local ? 1 : 0, upvalue.index);
  }
}

void GlobalParser::synchronize() {
  panic_mode_ = false;

  while (!check(TokenKind::TK_EOF)) {
    if (prev_.kind() == TokenKind::TK_SEMI)
      break;

    switch (curr_.kind()) {
    case TokenKind::KW_FN:
    case TokenKind::KW_VAR:
      return;
    default: break;
    }
    advance();
  }
}

void GlobalParser::expression() {
  parse_precedence(Precedence::ASSIGN);
}

void GlobalParser::declaration() {
  if (match(TokenKind::KW_FN))
    fn_decl();
  else if (match(TokenKind::KW_VAR))
    var_decl();
  else
    statement();

  if (panic_mode_)
    synchronize();
}

void GlobalParser::statement() {
  if (match(TokenKind::TK_LPAREN)) {
    enter_scope();
    block();
    leave_scope();
  }
  else {
    expr_stmt();
  }
}

void GlobalParser::fn_decl() {
  u8_t fn_constant = parse_variable("expect function name");
  mark_initialized();
  function(FunType::FUNCTION);

  define_global(fn_constant);
}

void GlobalParser::var_decl() {
  u8_t var_constant = parse_variable("expect variable name");

  if (match(TokenKind::TK_EQ))
    expression();
  else
    emit_byte(Code::NIL);
  consume(TokenKind::TK_SEMI, "expect `;` after variable declaration");

  define_global(var_constant);
}

void GlobalParser::expr_stmt() {
  expression();
  consume(TokenKind::TK_SEMI, "expect `;` after expression");
  emit_byte(Code::POP);
}

}
<commit_msg>:construction: chore(parser): updated the parser<commit_after>// Copyright (c) 2021 ASMlover. All rights reserved.
//
//  _____         _             _
// |_   _|_ _  __| |_ __   ___ | | ___
//   | |/ _` |/ _` | '_ \ / _ \| |/ _ \
//   | | (_| | (_| | |_) | (_) | |  __/
//   |_|\__,_|\__,_| .__/ \___/|_|\___|
//                 |_|
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include <iostream>
#include "../common/colorful.hh"
#include "../lex/lexer.hh"
#include "../object/string_object.hh"
#include "vm.hh"
#include "parser.hh"

namespace tadpole {

GlobalParser::GlobalParser(VM& vm, Lexer& lex) noexcept
  : vm_(vm), lex_(lex) {
}

void GlobalParser::iter_objects(ObjectVisitor&& visitor) {
  for (FunCompiler* c = curr_compiler_; c != nullptr; c = c->enclosing())
    visitor(c->fn());
}

FunctionObject* GlobalParser::compile() {
  FunCompiler compiler;
  init_compiler(&compiler, 0, FunType::TOPLEVEL);

  advance();
  while (!check(TokenKind::TK_EOF))
    declaration();
  FunctionObject* fn = finish_compiler();

  return had_error_ ? nullptr : fn;
}

const ParseRule& GlobalParser::get_rule(TokenKind kind) const noexcept {
#define _RULE(fn) [](GlobalParser& p, bool b) { p.fn(b); }

  static const ParseRule _rules[] = {
    {_RULE(grouping), _RULE(call), Precedence::CALL}, // PUNCTUATOR(LPAREN, "(")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(RPAREN, ")")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(LBRACE, "{")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(RBRACE, "}")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(COMMA, ",")
    {nullptr, _RULE(binary), Precedence::TERM},       // PUNCTUATOR(MINUS, "-")
    {nullptr, _RULE(binary), Precedence::TERM},       // PUNCTUATOR(PLUS, "+")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(SEMI, ";")
    {nullptr, _RULE(binary), Precedence::FACTOR},     // PUNCTUATOR(SLASH, "/")
    {nullptr, _RULE(binary), Precedence::FACTOR},     // PUNCTUATOR(STAR, "*")
    {nullptr, nullptr, Precedence::NONE},             // PUNCTUATOR(EQ, "=")

    {_RULE(variable), nullptr, Precedence::NONE},     // TOKEN(IDENTIFIER, "Identifier")
    {_RULE(numeric), nullptr, Precedence::NONE},      // TOKEN(NUMERIC, "Numeric")
    {_RULE(string), nullptr, Precedence::NONE},       // TOKEN(STRING, "String")

    {_RULE(literal), nullptr, Precedence::NONE},      // KEYWORD(FALSE, "false")
    {nullptr, nullptr, Precedence::NONE},             // KEYWORD(FN, "fn")
    {_RULE(literal), nullptr, Precedence::NONE},      // KEYWORD(NIL, "nil")
    {_RULE(literal), nullptr, Precedence::NONE},      // KEYWORD(TRUE, "true")
    {nullptr, nullptr, Precedence::NONE},             // KEYWORD(VAR, "var")

    {nullptr, nullptr, Precedence::NONE},             // TOKEN(EOF, "Eof")
    {nullptr, nullptr, Precedence::NONE},             // TOKEN(ERR, "Error")
  };
#undef _RULE

  return _rules[as_type<int>(kind)];
}

void GlobalParser::error_at(const Token& tok, const str_t& msg) noexcept {
  if (panic_mode_)
    return;
  panic_mode_ = true;

  std::cerr
    << colorful::red
    << "SyntaxError:" << std::endl
    << "  [LINE: " << tok.lineno() << "] ERROR ";
  if (tok.kind() == TokenKind::TK_EOF)
    std::cerr << "at end ";
  else if (tok.kind() == TokenKind::TK_ERR)
    TADPOLE_UNUSED(0);
  else
    std::cerr << "at `" << tok.literal() << "` ";
  std::cerr << ": " << msg << colorful::reset << std::endl;
}

void GlobalParser::advance() {
  prev_ = curr_;

  for (;;) {
    curr_ = lex_.next_token();
    if (!check(TokenKind::TK_ERR))
      break;

    error_at_current(curr_.as_string());
  }
}

void GlobalParser::consume(TokenKind kind, const str_t& msg) {
  if (check(kind))
    advance();
  else
    error_at_current(msg);
}

bool GlobalParser::match(TokenKind kind) {
  if (check(kind)) {
    advance();
    return true;
  }
  return false;
}

void GlobalParser::init_compiler(FunCompiler* compiler, int scope_depth, FunType fn_type) {
  StringObject* fn_name{};
  if (fn_type == FunType::FUNCTION)
    fn_name = StringObject::create(prev_.as_string());

  compiler->set_compiler(
      curr_compiler_,
      FunctionObject::create(fn_name),
      fn_type,
      scope_depth);
  curr_compiler_ = compiler;

  curr_compiler_->append_local({Token::make(""), curr_compiler_->scope_depth(), false});
}

FunctionObject* GlobalParser::finish_compiler() {
  emit_return();

  FunctionObject* fn = curr_compiler_->fn();

#if defined(_TADPOLE_DEBUG_VM)
  if (!had_error_)
    curr_chunk()->dis(fn->name_asstr());
#endif

  curr_compiler_ = curr_compiler_->enclosing();
  return fn;
}

void GlobalParser::enter_scope() {
  curr_compiler_->enter_scope();
}

void GlobalParser::leave_scope() {
  curr_compiler_->leave_scope([this](const LocalVar& var) {
        emit_byte(var.is_upvalue ? Code::CLOSE_UPVALUE : Code::POP);
      });
}

u8_t GlobalParser::identifier_constant(const Token& name) noexcept {
  return curr_chunk()->add_constant(StringObject::create(name.as_string()));
}

u8_t GlobalParser::parse_variable(const str_t& msg) {
  consume(TokenKind::TK_IDENTIFIER, msg);

  curr_compiler_->declare_localvar(prev_, [this](const str_t& m) { error(m); });
  if (curr_compiler_->scope_depth() > 0)
    return 0;
  return identifier_constant(prev_);
}

void GlobalParser::mark_initialized() {
  if (curr_compiler_->scope_depth() == 0)
    return;
  curr_compiler_->peek_local().depth = curr_compiler_->scope_depth();
}

void GlobalParser::define_global(u8_t global) {
  if (curr_compiler_->scope_depth() > 0) {
    mark_initialized();
    return;
  }
  emit_bytes(Code::DEF_GLOBAL, global);
}

u8_t GlobalParser::arguments() {
  u8_t nargs = 0;
  if (!check(TokenKind::TK_RPAREN)) {
    do {
      expression();
      ++nargs;

      if (nargs > kMaxArgs)
        error(from_fmt("cannot have more than `%d` arguments", kMaxArgs));
    } while (match(TokenKind::TK_COMMA));
  }
  consume(TokenKind::TK_RPAREN, "expect `;` after function arguments");

  return nargs;
}

void GlobalParser::named_variable(const Token& name, bool can_assign) {
  auto errfn = [this](const str_t& msg) { error(msg); };

  Code getop, setop;
  int arg = curr_compiler_->resolve_local(name, errfn);
  if (arg != -1) {
    getop = Code::GET_LOCAL;
    setop = Code::SET_LOCAL;
  }
  else if (arg = curr_compiler_->resolve_upvalue(name, errfn); arg != -1) {
    getop = Code::GET_UPVALUE;
    setop = Code::SET_UPVALUE;
  }
  else {
    arg = identifier_constant(name);
    getop = Code::GET_GLOBAL;
    setop = Code::SET_GLOBAL;
  }

  if (can_assign && match(TokenKind::TK_EQ)) {
    expression();
    emit_bytes(setop, arg);
  }
  else {
    emit_bytes(getop, arg);
  }
}

void GlobalParser::parse_precedence(Precedence precedence) {
  advance();
  auto& prefix_fn = get_rule(prev_.kind()).prefix;
  if (!prefix_fn) {
    error("expect expression");
    return;
  }

  bool can_assign = precedence <= Precedence::ASSIGN;
  prefix_fn(*this, can_assign);

  while (precedence <= get_rule(curr_.kind()).precedence) {
    advance();
    auto& infix_fn = get_rule(prev_.kind()).infix;

    if (infix_fn)
      infix_fn(*this, can_assign);
  }

  if (can_assign && match(TokenKind::TK_EQ)) {
    error("invalid assignment target");
    expression();
  }
}

void GlobalParser::binary(bool can_assign) {
  TokenKind op = prev_.kind();

  parse_precedence(get_rule(op).precedence + 1);
  switch (op) {
  case TokenKind::TK_PLUS: emit_byte(Code::ADD); break;
  case TokenKind::TK_MINUS: emit_byte(Code::SUB); break;
  case TokenKind::TK_STAR: emit_byte(Code::MUL); break;
  case TokenKind::TK_SLASH: emit_byte(Code::DIV); break;
  }
}

void GlobalParser::call(bool can_assign) {
  emit_byte(Code::CALL_0 + arguments());
}

void GlobalParser::grouping(bool can_assign) {
  expression();
  consume(TokenKind::TK_RPAREN, "expect `)` after grouping expression");
}

void GlobalParser::literal(bool can_assign) {
  switch (prev_.kind()) {
  case TokenKind::KW_NIL: emit_byte(Code::NIL); break;
  case TokenKind::KW_TRUE: emit_byte(Code::TRUE); break;
  case TokenKind::KW_FALSE: emit_byte(Code::FALSE); break;
  }
}

void GlobalParser::variable(bool can_assign) {
  named_variable(prev_, can_assign);
}

void GlobalParser::numeric(bool can_assign) {
  emit_constant(prev_.as_numeric());
}

void GlobalParser::string(bool can_assign) {
  emit_constant(StringObject::create(prev_.as_string()));
}

void GlobalParser::block() {
  while (!check(TokenKind::TK_EOF) && !check(TokenKind::TK_RBRACE))
    declaration();
  consume(TokenKind::TK_RBRACE, "expect `}` after block body");
}

void GlobalParser::function(FunType fn_type) {
  FunCompiler fn_compiler;
  init_compiler(&fn_compiler, 1, fn_type);

  consume(TokenKind::TK_LPAREN, "expect `(` after function name");
  if (!check(TokenKind::TK_RPAREN)) {
    do {
      u8_t param_constant = parse_variable("expect function parameter's name");
      define_global(param_constant);

      curr_compiler_->fn()->inc_arity();
      if (curr_compiler_->fn()->arity() > kMaxArgs)
        error(from_fmt("cannot have more than `%s` parameters", kMaxArgs));
    } while (match(TokenKind::TK_COMMA));
  }
  consume(TokenKind::TK_RPAREN, "expect `)` after function parameters");

  consume(TokenKind::TK_LBRACE, "expect `{` before function body");
  block();

  leave_scope();
  FunctionObject* fn = finish_compiler();

  emit_bytes(Code::CLOSURE, curr_chunk()->add_constant(fn));
  for (sz_t i = 0; i < fn->upvalues_count(); ++i) {
    auto& upvalue = fn_compiler.get_upvalue(i);
    emit_bytes(upvalue.is_local ? 1 : 0, upvalue.index);
  }
}

void GlobalParser::synchronize() {
  panic_mode_ = false;

  while (!check(TokenKind::TK_EOF)) {
    if (prev_.kind() == TokenKind::TK_SEMI)
      break;

    switch (curr_.kind()) {
    case TokenKind::KW_FN:
    case TokenKind::KW_VAR:
      return;
    default: break;
    }
    advance();
  }
}

void GlobalParser::expression() {
  parse_precedence(Precedence::ASSIGN);
}

void GlobalParser::declaration() {
  if (match(TokenKind::KW_FN))
    fn_decl();
  else if (match(TokenKind::KW_VAR))
    var_decl();
  else
    statement();

  if (panic_mode_)
    synchronize();
}

void GlobalParser::statement() {
  if (match(TokenKind::TK_LPAREN)) {
    enter_scope();
    block();
    leave_scope();
  }
  else {
    expr_stmt();
  }
}

void GlobalParser::fn_decl() {
  u8_t fn_constant = parse_variable("expect function name");
  mark_initialized();
  function(FunType::FUNCTION);

  define_global(fn_constant);
}

void GlobalParser::var_decl() {
  u8_t var_constant = parse_variable("expect variable name");

  if (match(TokenKind::TK_EQ))
    expression();
  else
    emit_byte(Code::NIL);
  consume(TokenKind::TK_SEMI, "expect `;` after variable declaration");

  define_global(var_constant);
}

void GlobalParser::expr_stmt() {
  expression();
  consume(TokenKind::TK_SEMI, "expect `;` after expression");
  emit_byte(Code::POP);
}

}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014-2016, imec
 * All rights reserved.
 */

#include "error.h"
#include "bpmf.h"

#include <random>
#include <memory>
#include <cstdio>
#include <iostream>
#include <climits>
#include <stdexcept>

typedef Eigen::PermutationMatrix<Eigen::Dynamic, Eigen::Dynamic> PermMatrix;

void Sys::permuteCols(const PermMatrix &perm, Sys &other)
{
    // permute local matrices
    T = T * perm;
    Pavg = Pavg * perm;
    Pm2 = Pm2 * perm;
    M = M * perm;

    if (has_prop_posterior())
    {
        propMu *= perm;
        propLambda *= perm;
    }

    // permute other matrices
    other.T = T.transpose();
    other.Pavg = Pavg.transpose();
    other.Pm2 = Pm2.transpose();
    other.M = M.transpose();
}

//
// Distributes users/movies accros several nodes
// takes into account load balance and communication cost
//
void Sys::assign(Sys &other)
{
    if (nprocs == 1) {
        dom[0] = 0;
        dom[1] = num();
        return;
    }

    if (!permute) {
        int p = num() / nprocs;
        int i=0; for(; i<nprocs; ++i) dom[i] = i*p;
        dom[i] = num();
        return;
    }

    std::vector<std::vector<double>> comm_cost(num());
    if (other.assigned) {
        // comm_cost[i][j] == communication cost if item i is assigned to processor j
        for(int i=0; i<num(); ++i) {
            std::vector<unsigned> comm_per_proc(nprocs);
            const int count = M.innerVector(i).nonZeros();
            for (SparseMatrixD::InnerIterator it(M,i); it; ++it) comm_per_proc.at(other.proc(it.row()))++;
            for(int j=0; j<nprocs; j++) comm_cost.at(i).push_back(count - comm_per_proc.at(j));
        }
    }

    std::vector<unsigned> nnz_per_proc(nprocs);
    std::vector<unsigned> items_per_proc(nprocs);
    std::vector<double>   work_per_proc(nprocs);

    std::vector<int> item_to_proc(num(), -1);

    unsigned total_nnz   = 1;
    unsigned total_items = 1;
    double   total_work  = 0.01;
    unsigned total_comm  = 0;

    // computes best node to assign movie/user idx
    auto best = [&](int idx, double r1, double r2) {
        double min_cost = 1e9;
        int best_proc = -1;
        for(int i=0; i<nprocs; ++i) {
            //double nnz_unbalance = (double)nnz_per_proc[i] / total_nnz;
            //double items_unbalance =  (double)items_per_proc[i] / total_items;
            //double work_unbalance = std::max(nnz_unbalance, items_unbalance);
            double work_unbalance = work_per_proc[i] / total_work;

            double comm = other.assigned ? comm_cost.at(idx).at(i) : 0.0;
            double total_cost = r1 * work_unbalance + r2 * comm;
            if (total_cost > min_cost) continue;
            best_proc = i;
            min_cost = total_cost;
        }
        return best_proc;
    };

    // update cost function when item is assigned to proc
    auto assign = [&](int item, int proc) {
        const int nnz = M.innerVector(item).nonZeros();
        double work = 10.0 + nnz; // one item is as expensive as  NZs
        item_to_proc[item] = proc;
        nnz_per_proc  [proc] += nnz;
        items_per_proc[proc]++;
        work_per_proc [proc] += work;
        total_nnz += nnz;
        total_items++;
        total_work+= work;
        total_comm += (other.assigned ? comm_cost.at(item).at(proc) : 0);
    };

    // update cost function when item is removed from proc
    auto unassign = [&](int item) {
        int proc = item_to_proc[item];
        if (proc < 0) return;
        const int nnz = M.innerVector(item).nonZeros();
        double work = 7.1 + nnz;
        item_to_proc[item] = -1;
        nnz_per_proc  [proc] -= nnz;
        items_per_proc[proc]--;
        work_per_proc [proc] -= work;
        total_nnz -= nnz;
        total_items--;
        total_work -= work;
        total_comm -= (other.assigned ? comm_cost.at(item).at(proc) : 0);

    };

    // print cost after iterating once
    auto print = [&](int iter) {
        Sys::cout() << name << " -- iter " << iter << " -- \n";
        if (Sys::procid == 0) {
            int max_nnz = *std::max_element(nnz_per_proc.begin(), nnz_per_proc.end());
            int min_nnz = *std::min_element(nnz_per_proc.begin(), nnz_per_proc.end());
            int avg_nnz = nnz() / nprocs;

            int max_items = *std::max_element(items_per_proc.begin(), items_per_proc.end());
            int min_items = *std::min_element(items_per_proc.begin(), items_per_proc.end());
            int avg_items = num() / nprocs;

            double max_work = *std::max_element(work_per_proc.begin(), work_per_proc.end());
            double min_work = *std::min_element(work_per_proc.begin(), work_per_proc.end());
            double avg_work = total_work / nprocs;

            Sys::cout() << name << ": comm cost " << 100.0 * total_comm / nnz() / nprocs << "%\n";
            Sys::cout() << name << ": nnz unbalance: " << (int)(100.0 * Sys::nprocs * (max_nnz - min_nnz) / nnz()) << "%"
                << "\t(" << max_nnz << " <-> " << avg_nnz << " <-> " << min_nnz << ")\n";
            Sys::cout() << name << ": items unbalance: " << (int)(100.0 * Sys::nprocs * (max_items - min_items) / num()) << "%"
                << "\t(" << max_items << " <-> " << avg_items << " <-> " << min_items << ")\n";
            Sys::cout() << name << ": work unbalance: " << (int)(100.0 * Sys::nprocs * (max_work - min_work) / total_work) << "%"
                << "\t(" << max_work << " <-> " << avg_work << " <-> " << min_work << ")\n\n";
        }

        Sys::cout() << name << ": nnz:\t" << nnz_per_proc[procid] << " / " << nnz() << "\n";
        Sys::cout() << name << ": items:\t" << items_per_proc[procid] << " / " << num() << "\n";
        Sys::cout() << name << ": work:\t" << work_per_proc[procid] << " / " << total_work << "\n";
    };

    for(int j=0; j<3; ++j) {
        for(int i=0; i<num(); ++i) {
            unassign(i);
            assign(i, best(i, 10000, 0));
        }
        print(j);
    }

    std::vector<std::vector<unsigned>> proc_to_item(nprocs);
    for(int i=0; i<num(); ++i) proc_to_item[item_to_proc[i]].push_back(i);

    // permute T, P  based on assignment done before
    unsigned pos = 0;
    Eigen::PermutationMatrix<Eigen::Dynamic, Eigen::Dynamic> perm(num());
    for(auto p: proc_to_item) for(auto i: p) perm.indices()(pos++) = i;
    auto oldT = T;

    this->permuteCols(perm, other);

    int i = 0;
    int n = 0;
    dom[0] = 0;
    for(auto p : items_per_proc) dom[++i] = (n += p);

#ifndef NDEBUG
    int j = 0;
    for(auto i : proc_to_item.at(0)) assert(T.col(j++).nonZeros() == oldT.col(i).nonZeros());
#endif

    Sys::cout() << name << " domain:" << std::endl;
    print_dom(Sys::cout());
    Sys::cout() << std::endl;

    assigned = true;
}

//
// Update connectivity map (where to send certain items)
// based on assignment to nodes
//
void Sys::update_conn(Sys& other)
{
    unsigned tot = 0;
    conn_map.clear();
    conn_count_map.clear();
    assert(nprocs <= (int)max_procs);

    conn_map.resize(num());
    for (int k=0; k<num(); ++k) {
        std::bitset<max_procs> &bm = conn_map[k];
        for (SparseMatrixD::InnerIterator it(M,k); it; ++it) bm.set(other.proc(it.row()));
        for (SparseMatrixD::InnerIterator it(Pavg,k); it; ++it) bm.set(other.proc(it.row()));
        bm.reset(proc(k)); // not to self
        tot += bm.count();

        // keep track of how may proc to proc sends
        auto from_proc = proc(k);
        for(int to_proc=0; to_proc<Sys::nprocs; to_proc++) {
            if (!bm.test(to_proc)) continue;
            conn_count_map[std::make_pair(from_proc, to_proc)]++;
        }
    }

    if (Sys::procid == 0) {
        Sys::cout() << name << ": avg items to send per iter: " << tot << " / " << num() << " = " << (double)tot / (double)num() << std::endl;

        Sys::cout() << name << ": messages from -> to proc\n";
        for(int i=0; i<Sys::nprocs; ++i) Sys::cout() << "\t" << i;
        Sys::cout() << "\n";
        for(int i=0; i<Sys::nprocs; ++i) {
            Sys::cout() << i;
            for(int j=0; j<Sys::nprocs; ++j) Sys::cout() << "\t" << conn_count(i,j);
            Sys::cout() << "\n";
        }

    }
}

//
// try to keep items that have to be sent to the same node next to eachothe
//
void Sys::opt_conn(Sys& other)
{
    // sort internally according to hamming distance
    PermMatrix perm(num());
    perm.setIdentity();

    std::vector<std::string> s(num());

    auto v = perm.indices().data();
#pragma omp parallel for
    for (auto p = 0; p < nprocs; ++p)
    {
        for (int i = from(p); i < to(p); ++i)
        {
            s[i] = conn(i).to_string();
        }
        std::sort(v + from(p), v + to(p), [&](const int &a, const int &b) { return (s[a] < s[b]); });
    }

    permuteCols(perm, other);
}

void Sys::build_conn(Sys& other)
{
    if (nprocs == 1) return;

    update_conn(other);
    //opt_conn(other);
    //update_conn(other);
}<commit_msg>ENH: compute and print test items per proc<commit_after>/*
 * Copyright (c) 2014-2016, imec
 * All rights reserved.
 */

#include "error.h"
#include "bpmf.h"

#include <random>
#include <memory>
#include <cstdio>
#include <iostream>
#include <climits>
#include <stdexcept>

typedef Eigen::PermutationMatrix<Eigen::Dynamic, Eigen::Dynamic> PermMatrix;

void Sys::permuteCols(const PermMatrix &perm, Sys &other)
{
    // permute local matrices
    T = T * perm;
    Pavg = Pavg * perm;
    Pm2 = Pm2 * perm;
    M = M * perm;

    if (has_prop_posterior())
    {
        propMu *= perm;
        propLambda *= perm;
    }

    // permute other matrices
    other.T = T.transpose();
    other.Pavg = Pavg.transpose();
    other.Pm2 = Pm2.transpose();
    other.M = M.transpose();
}

//
// Distributes users/movies accros several nodes
// takes into account load balance and communication cost
//
void Sys::assign(Sys &other)
{
    if (nprocs == 1) {
        dom[0] = 0;
        dom[1] = num();
        return;
    }

    if (!permute) {
        int p = num() / nprocs;
        int i=0; for(; i<nprocs; ++i) dom[i] = i*p;
        dom[i] = num();
        return;
    }

    std::vector<std::vector<double>> comm_cost(num());
    if (other.assigned) {
        // comm_cost[i][j] == communication cost if item i is assigned to processor j
        for(int i=0; i<num(); ++i) {
            std::vector<unsigned> comm_per_proc(nprocs);
            const int count = M.innerVector(i).nonZeros();
            for (SparseMatrixD::InnerIterator it(M,i); it; ++it) comm_per_proc.at(other.proc(it.row()))++;
            for(int j=0; j<nprocs; j++) comm_cost.at(i).push_back(count - comm_per_proc.at(j));
        }
    }

    std::vector<unsigned> nnz_per_proc(nprocs);
    std::vector<unsigned> items_per_proc(nprocs);
    std::vector<double>   work_per_proc(nprocs);

    std::vector<int> item_to_proc(num(), -1);

    unsigned total_nnz   = 1;
    unsigned total_items = 1;
    double   total_work  = 0.01;
    unsigned total_comm  = 0;

    // computes best node to assign movie/user idx
    auto best = [&](int idx, double r1, double r2) {
        double min_cost = 1e9;
        int best_proc = -1;
        for(int i=0; i<nprocs; ++i) {
            //double nnz_unbalance = (double)nnz_per_proc[i] / total_nnz;
            //double items_unbalance =  (double)items_per_proc[i] / total_items;
            //double work_unbalance = std::max(nnz_unbalance, items_unbalance);
            double work_unbalance = work_per_proc[i] / total_work;

            double comm = other.assigned ? comm_cost.at(idx).at(i) : 0.0;
            double total_cost = r1 * work_unbalance + r2 * comm;
            if (total_cost > min_cost) continue;
            best_proc = i;
            min_cost = total_cost;
        }
        return best_proc;
    };

    // update cost function when item is assigned to proc
    auto assign = [&](int item, int proc) {
        const int nnz = M.innerVector(item).nonZeros();
        double work = 10.0 + nnz; // one item is as expensive as  NZs
        item_to_proc[item] = proc;
        nnz_per_proc  [proc] += nnz;
        items_per_proc[proc]++;
        work_per_proc [proc] += work;
        total_nnz += nnz;
        total_items++;
        total_work+= work;
        total_comm += (other.assigned ? comm_cost.at(item).at(proc) : 0);
    };

    // update cost function when item is removed from proc
    auto unassign = [&](int item) {
        int proc = item_to_proc[item];
        if (proc < 0) return;
        const int nnz = M.innerVector(item).nonZeros();
        double work = 7.1 + nnz;
        item_to_proc[item] = -1;
        nnz_per_proc  [proc] -= nnz;
        items_per_proc[proc]--;
        work_per_proc [proc] -= work;
        total_nnz -= nnz;
        total_items--;
        total_work -= work;
        total_comm -= (other.assigned ? comm_cost.at(item).at(proc) : 0);

    };

    // print cost after iterating once
    auto print = [&](int iter) {
        Sys::cout() << name << " -- iter " << iter << " -- \n";
        std::vector<unsigned> test_ratings_per_proc(nprocs);
        for (int i = 0; i < num(); ++i)
        {
            int proc = item_to_proc[i];
            test_ratings_per_proc[proc] += T.col(i).nonZeros();
        }

        int max_nnz = *std::max_element(nnz_per_proc.begin(), nnz_per_proc.end());
        int min_nnz = *std::min_element(nnz_per_proc.begin(), nnz_per_proc.end());
        int avg_nnz = nnz() / nprocs;

        int max_items = *std::max_element(items_per_proc.begin(), items_per_proc.end());
        int min_items = *std::min_element(items_per_proc.begin(), items_per_proc.end());
        int avg_items = num() / nprocs;

        double max_work = *std::max_element(work_per_proc.begin(), work_per_proc.end());
        double min_work = *std::min_element(work_per_proc.begin(), work_per_proc.end());
        double avg_work = total_work / nprocs;

        Sys::cout() << name << ": comm cost " << 100.0 * total_comm / nnz() / nprocs << "%\n";
        Sys::cout() << name << ": nnz unbalance: " << (int)(100.0 * Sys::nprocs * (max_nnz - min_nnz) / nnz()) << "%"
                    << "\t(" << max_nnz << " <-> " << avg_nnz << " <-> " << min_nnz << ")\n";
        Sys::cout() << name << ": items unbalance: " << (int)(100.0 * Sys::nprocs * (max_items - min_items) / num()) << "%"
                    << "\t(" << max_items << " <-> " << avg_items << " <-> " << min_items << ")\n";
        Sys::cout() << name << ": work unbalance: " << (int)(100.0 * Sys::nprocs * (max_work - min_work) / total_work) << "%"
                    << "\t(" << max_work << " <-> " << avg_work << " <-> " << min_work << ")\n\n";

        Sys::cout() << name << ": train nnz:\t" << nnz_per_proc[procid] << " / " << nnz() << "\n";
        Sys::cout() << name << ": test nnz:\t" << test_ratings_per_proc[procid] << " / " << nnz() << "\n";
        Sys::cout() << name << ": items:\t" << items_per_proc[procid] << " / " << num() << "\n";
        Sys::cout() << name << ": work:\t" << work_per_proc[procid] << " / " << total_work << "\n";
    };

    for(int j=0; j<3; ++j) {
        for(int i=0; i<num(); ++i) {
            unassign(i);
            assign(i, best(i, 10000, 0));
        }
        print(j);
    }

    std::vector<std::vector<unsigned>> proc_to_item(nprocs);
    for(int i=0; i<num(); ++i) proc_to_item[item_to_proc[i]].push_back(i);

    // permute T, P  based on assignment done before
    unsigned pos = 0;
    Eigen::PermutationMatrix<Eigen::Dynamic, Eigen::Dynamic> perm(num());
    for(auto p: proc_to_item) for(auto i: p) perm.indices()(pos++) = i;
    auto oldT = T;

    this->permuteCols(perm, other);

    int i = 0;
    int n = 0;
    dom[0] = 0;
    for(auto p : items_per_proc) dom[++i] = (n += p);

#ifndef NDEBUG
    int j = 0;
    for(auto i : proc_to_item.at(0)) assert(T.col(j++).nonZeros() == oldT.col(i).nonZeros());
#endif

    Sys::cout() << name << " domain:" << std::endl;
    print_dom(Sys::cout());
    Sys::cout() << std::endl;

    assigned = true;
}

//
// Update connectivity map (where to send certain items)
// based on assignment to nodes
//
void Sys::update_conn(Sys& other)
{
    unsigned tot = 0;
    conn_map.clear();
    conn_count_map.clear();
    assert(nprocs <= (int)max_procs);

    conn_map.resize(num());
    for (int k=0; k<num(); ++k) {
        std::bitset<max_procs> &bm = conn_map[k];
        for (SparseMatrixD::InnerIterator it(M,k); it; ++it) bm.set(other.proc(it.row()));
        for (SparseMatrixD::InnerIterator it(Pavg,k); it; ++it) bm.set(other.proc(it.row()));
        bm.reset(proc(k)); // not to self
        tot += bm.count();

        // keep track of how may proc to proc sends
        auto from_proc = proc(k);
        for(int to_proc=0; to_proc<Sys::nprocs; to_proc++) {
            if (!bm.test(to_proc)) continue;
            conn_count_map[std::make_pair(from_proc, to_proc)]++;
        }
    }

    if (Sys::procid == 0) {
        Sys::cout() << name << ": avg items to send per iter: " << tot << " / " << num() << " = " << (double)tot / (double)num() << std::endl;

        Sys::cout() << name << ": messages from -> to proc\n";
        for(int i=0; i<Sys::nprocs; ++i) Sys::cout() << "\t" << i;
        Sys::cout() << "\n";
        for(int i=0; i<Sys::nprocs; ++i) {
            Sys::cout() << i;
            for(int j=0; j<Sys::nprocs; ++j) Sys::cout() << "\t" << conn_count(i,j);
            Sys::cout() << "\n";
        }

    }
}

//
// try to keep items that have to be sent to the same node next to eachothe
//
void Sys::opt_conn(Sys& other)
{
    // sort internally according to hamming distance
    PermMatrix perm(num());
    perm.setIdentity();

    std::vector<std::string> s(num());

    auto v = perm.indices().data();
#pragma omp parallel for
    for (auto p = 0; p < nprocs; ++p)
    {
        for (int i = from(p); i < to(p); ++i)
        {
            s[i] = conn(i).to_string();
        }
        std::sort(v + from(p), v + to(p), [&](const int &a, const int &b) { return (s[a] < s[b]); });
    }

    permuteCols(perm, other);
}

void Sys::build_conn(Sys& other)
{
    if (nprocs == 1) return;

    update_conn(other);
    //opt_conn(other);
    //update_conn(other);
}<|endoftext|>