<commit_before>/*****************************************************************************
*  Copyright 2011 Sergey Shekyan
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* *****************************************************************************/

/*****
 * Author: Sergey Shekyan shekyan@gmail.com
 *
 * Slow HTTP attack  vulnerability test tool
 *  http://code.google.com/p/slowhttptest/
 *****/

#include <errno.h>
#include <fcntl.h>
#include <netdb.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

#include <algorithm>
#include <string>

#include <openssl/ssl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>

#include "slowlog.h"
#include "slowsocket.h"
#include "slowurl.h"

namespace slowhttptest {
SlowSocket::SlowSocket()
    : sockfd_(-1),
      requests_to_send_(0),
      followups_to_send_(0),
      last_followup_timing_(0),
      offset_(0),
      ssl_(0),
      ssl_ctx_(0),
      buf_(0),
      start_in_millisecs_(0),
      connected_in_millisecs_(0),
      stop_in_millisecs_(0),
      state_(eInit),
      last_read_in_msec_(0),
      read_interval_(0),
      window_size_(-1) {
}

SlowSocket::~SlowSocket() {
  close();
}

bool SlowSocket::set_window_size(int wnd_size) {
  int actual_wnd_size = 0;
  socklen_t actual_wnd_size_len = sizeof(actual_wnd_size);
	bool ret = setsockopt(sockfd_, SOL_SOCKET, SO_RCVBUF, &wnd_size, sizeof(wnd_size));
  if(ret) {
    slowlog(LOG_ERROR, "error setting socket send buffer size to %d: %s\n", wnd_size, strerror(errno));
  } else {
    getsockopt(sockfd_, SOL_SOCKET, SO_RCVBUF, &actual_wnd_size, &actual_wnd_size_len);
    slowlog(LOG_DEBUG, "set socket %d send buffer size to %d bytes(requested %d)\n", sockfd_, actual_wnd_size, wnd_size);
  }
  return ret;
}

int SlowSocket::set_nonblocking() {
  int flags;

  if(-1 == (flags = fcntl(sockfd_, F_GETFL, 0))) {
    flags = 0;
  }
  return fcntl(sockfd_, F_SETFL, flags | O_NONBLOCK);
}

bool SlowSocket::init(addrinfo* addr, const Url* url, int& maxfd,
                      int followups_to_send, int read_interval, int wnd_size_max) {
 	read_interval_ = read_interval * 1000;
  if(read_interval_) { // slow read test
    window_size_ = rand() % wnd_size_max;
    if(!window_size_) // null is not a good choice
      window_size_ = 1;
  }
	addrinfo* res;
  bool connect_initiated_ = false;
  for (res = addr; !connect_initiated_ && res; res = res->ai_next) {
    sockfd_ = socket(res->ai_family, res->ai_socktype,
                     res->ai_protocol);
    if(-1 == sockfd_) {
      slowlog(LOG_ERROR, "failed to create socket: %s\n", strerror(errno));
      return false;
    }

    if(-1 == set_nonblocking()) {
      slowlog(LOG_ERROR, "failed to set socket %d to non-blocking \n", sockfd_);
      return false;
    }
    if(read_interval_) {
      set_window_size(window_size_);
    }
    slowlog(LOG_DEBUG, "non-blocking socket %d created \n", sockfd_);
    if(connect_initiated_ = url->isSSL() ? connect_ssl(addr) : connect_plain(addr)) {
      break; //found right addrinfo
    }
  }

  followups_to_send_ = followups_to_send;
  requests_to_send_ = 1;

  maxfd = std::max(sockfd_, maxfd);
  return true;
}

bool SlowSocket::connect_plain(addrinfo* addr) {
  errno = 0;

  if (connect(sockfd_, addr->ai_addr, addr->ai_addrlen) < 0
      && EINPROGRESS != errno) {
    slowlog(LOG_ERROR, "cannot connect socket %d: %s\n", sockfd_,
            strerror(errno));
    close();
    return false;
  }
  return true;
}

bool SlowSocket::connect_ssl(addrinfo* addr) {
  // Establish regular connection.
  if(!connect_plain(addr)) return false;

  // Init SSL related stuff.
  static bool ssl_is_initialized = false;
  if (!ssl_is_initialized) {
    SSL_library_init();
    ssl_is_initialized = true;
  }
  SSL_METHOD* method = NULL;
  method = (SSL_METHOD*)SSLv23_client_method();
  ssl_ctx_ = SSL_CTX_new(method);
  if(!ssl_ctx_) {
    slowlog(LOG_ERROR, "cannot create new SSL context\n");
    close();
    return false;
  }
  ssl_ = SSL_new(ssl_ctx_);
  if(!ssl_) {
    SSL_CTX_free(ssl_ctx_);
    slowlog(LOG_ERROR, "cannot create SSL structure for a connection\n");
    close();
    return false;
  }
  SSL_set_fd(ssl_, sockfd_);
  int ret = SSL_connect(ssl_);
  if(ret <= 0) {
    int err = SSL_get_error(ssl_, ret);
    //slowlog(LOG_ERROR, "socket %d: SSL connect error: %d\n", sockfd_, err);
    if(SSL_ERROR_WANT_READ != err && SSL_ERROR_WANT_WRITE != err) {
      close();
      return false;
    }
  }
  return true;
}

int SlowSocket::recv_slow(void* buf, size_t len) {
  int ret = ssl_ ? SSL_read(ssl_, buf, len)
                 : recv(sockfd_, buf, len, 0);
  if(ssl_) {
    if(ret < 0) {
      int err = SSL_get_error(ssl_, ret);
      if(err == SSL_ERROR_WANT_WRITE) {
        requests_to_send_ = 1;
      }
    }
    if(SSL_is_init_finished(ssl_) && (state_ == eConnecting)) {
      requests_to_send_ = 1;
    }
  }
  return ret;
}

int SlowSocket::send_slow(const void* buf, size_t len, const SendType type) {
  int ret;
  if(ssl_) {
    if(!SSL_is_init_finished(ssl_)) {
      ret = SSL_do_handshake(ssl_);
      if(ret <= 0) {
        int err = SSL_get_error(ssl_, ret);
        if(SSL_ERROR_WANT_READ != err && SSL_ERROR_WANT_WRITE != err) {
          slowlog(LOG_ERROR, "socket %d: SSL connect error: %d\n", sockfd_, err);
          close();
          return -1;
        } else {
          if(SSL_ERROR_WANT_READ == err) {
            requests_to_send_ = 0;
          }
          else {
            requests_to_send_ = 1;
          }
          errno = EAGAIN;
          return -1;
        }
      } else { //connected and handhsake finished
        requests_to_send_ = 1;
      }
    } else {
      if(requests_to_send_ > 0) { //report for initial data only
        slowlog(LOG_DEBUG, "socket %d: SSL connection is using %s\n", sockfd_,
            SSL_get_cipher(ssl_));
      }
    }
  }
  // VA: this is not good. create a "prepare" method.
  // initial send
  if(buf_ == 0) {
    buf_ = buf;
    offset_ = len;
  }

  ret = ssl_ ? SSL_write(ssl_, buf_, offset_)
                 : send(sockfd_, buf_, offset_, 0);

  // entire data was sent
  if(ret > 0 && ret == offset_) {
    if(eInitialSend == type) {
      requests_to_send_ = 0;
    } else if(eFollowUpSend == type) {
      --followups_to_send_;
    }
    buf_ = 0;
    offset_ = 0;
  } else if(ret > 0 && ret < offset_) {
    buf_ = static_cast<const char*>(buf_) + ret;
    offset_ -= ret;
  }
  return ret;
}

void SlowSocket::close() {
  if (-1 == sockfd_) return;

  slowlog(LOG_DEBUG, "closing slow socket %d\n", sockfd_);
  if(ssl_) {
    SSL_free(ssl_);
    SSL_CTX_free(ssl_ctx_);
    ssl_ = NULL;
    ssl_ctx_ = NULL;
  }
  requests_to_send_ = 0;
  followups_to_send_ = 0;
  ::close(sockfd_);
  sockfd_ = -1;
}

void SlowSocket::set_state(SocketState state) {
  timeval t;
  gettimeofday(&t, 0);
  switch(state) {
    case eInit:
      break;
    case eConnecting:
      set_start(&t);
      break;
    case eConnected:
      set_connected(&t);
      break;
    case eError:
      break;
    case eClosed:
      set_stop(&t);
      break;
    default:
      break;
  }
  state_ = state;
}

void SlowSocket::set_last_read(const timeval* t) {
  if(read_interval_) {
    last_read_in_msec_ = timeval_to_milliseconds(t);
  }
}

const bool SlowSocket::is_ready_read(const timeval* t) const {
  if(!read_interval_) //don't bother doing anything
    return true;
  long now = timeval_to_milliseconds(t);
  if(last_read_in_msec_ == 0) {
    now = read_interval_ + 1;
  }
  if(now - last_read_in_msec_ > read_interval_) {
    return true;
  } else {
    return false;
  }
}

}  // namespace slowhttptest
<commit_msg>typo in log message<commit_after>/*****************************************************************************
*  Copyright 2011 Sergey Shekyan
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* *****************************************************************************/

/*****
 * Author: Sergey Shekyan shekyan@gmail.com
 *
 * Slow HTTP attack  vulnerability test tool
 *  http://code.google.com/p/slowhttptest/
 *****/

#include <errno.h>
#include <fcntl.h>
#include <netdb.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

#include <algorithm>
#include <string>

#include <openssl/ssl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>

#include "slowlog.h"
#include "slowsocket.h"
#include "slowurl.h"

namespace slowhttptest {
SlowSocket::SlowSocket()
    : sockfd_(-1),
      requests_to_send_(0),
      followups_to_send_(0),
      last_followup_timing_(0),
      offset_(0),
      ssl_(0),
      ssl_ctx_(0),
      buf_(0),
      start_in_millisecs_(0),
      connected_in_millisecs_(0),
      stop_in_millisecs_(0),
      state_(eInit),
      last_read_in_msec_(0),
      read_interval_(0),
      window_size_(-1) {
}

SlowSocket::~SlowSocket() {
  close();
}

bool SlowSocket::set_window_size(int wnd_size) {
  int actual_wnd_size = 0;
  socklen_t actual_wnd_size_len = sizeof(actual_wnd_size);
	bool ret = setsockopt(sockfd_, SOL_SOCKET, SO_RCVBUF, &wnd_size, sizeof(wnd_size));
  if(ret) {
    slowlog(LOG_ERROR, "error setting socket send buffer size to %d: %s\n", wnd_size, strerror(errno));
  } else {
    getsockopt(sockfd_, SOL_SOCKET, SO_RCVBUF, &actual_wnd_size, &actual_wnd_size_len);
    slowlog(LOG_DEBUG, "set socket %d receive buffer size to %d bytes(requested %d)\n", sockfd_, actual_wnd_size, wnd_size);
  }
  return ret;
}

int SlowSocket::set_nonblocking() {
  int flags;

  if(-1 == (flags = fcntl(sockfd_, F_GETFL, 0))) {
    flags = 0;
  }
  return fcntl(sockfd_, F_SETFL, flags | O_NONBLOCK);
}

bool SlowSocket::init(addrinfo* addr, const Url* url, int& maxfd,
                      int followups_to_send, int read_interval, int wnd_size_max) {
 	read_interval_ = read_interval * 1000;
  if(read_interval_) { // slow read test
    window_size_ = rand() % wnd_size_max;
    if(!window_size_) // null is not a good choice
      window_size_ = 1;
  }
	addrinfo* res;
  bool connect_initiated_ = false;
  for (res = addr; !connect_initiated_ && res; res = res->ai_next) {
    sockfd_ = socket(res->ai_family, res->ai_socktype,
                     res->ai_protocol);
    if(-1 == sockfd_) {
      slowlog(LOG_ERROR, "failed to create socket: %s\n", strerror(errno));
      return false;
    }

    if(-1 == set_nonblocking()) {
      slowlog(LOG_ERROR, "failed to set socket %d to non-blocking \n", sockfd_);
      return false;
    }
    if(read_interval_) {
      set_window_size(window_size_);
    }
    slowlog(LOG_DEBUG, "socket %d created \n", sockfd_);
    if(connect_initiated_ = url->isSSL() ? connect_ssl(addr) : connect_plain(addr)) {
      break; //found right addrinfo
    }
  }

  followups_to_send_ = followups_to_send;
  requests_to_send_ = 1;

  maxfd = std::max(sockfd_, maxfd);
  return true;
}

bool SlowSocket::connect_plain(addrinfo* addr) {
  errno = 0;

  if (connect(sockfd_, addr->ai_addr, addr->ai_addrlen) < 0
      && EINPROGRESS != errno) {
    slowlog(LOG_ERROR, "cannot connect socket %d: %s\n", sockfd_,
            strerror(errno));
    close();
    return false;
  }
  return true;
}

bool SlowSocket::connect_ssl(addrinfo* addr) {
  // Establish regular connection.
  if(!connect_plain(addr)) return false;

  // Init SSL related stuff.
  static bool ssl_is_initialized = false;
  if (!ssl_is_initialized) {
    SSL_library_init();
    ssl_is_initialized = true;
  }
  SSL_METHOD* method = NULL;
  method = (SSL_METHOD*)SSLv23_client_method();
  ssl_ctx_ = SSL_CTX_new(method);
  if(!ssl_ctx_) {
    slowlog(LOG_ERROR, "cannot create new SSL context\n");
    close();
    return false;
  }
  ssl_ = SSL_new(ssl_ctx_);
  if(!ssl_) {
    SSL_CTX_free(ssl_ctx_);
    slowlog(LOG_ERROR, "cannot create SSL structure for a connection\n");
    close();
    return false;
  }
  SSL_set_fd(ssl_, sockfd_);
  int ret = SSL_connect(ssl_);
  if(ret <= 0) {
    int err = SSL_get_error(ssl_, ret);
    //slowlog(LOG_ERROR, "socket %d: SSL connect error: %d\n", sockfd_, err);
    if(SSL_ERROR_WANT_READ != err && SSL_ERROR_WANT_WRITE != err) {
      close();
      return false;
    }
  }
  return true;
}

int SlowSocket::recv_slow(void* buf, size_t len) {
  int ret = ssl_ ? SSL_read(ssl_, buf, len)
                 : recv(sockfd_, buf, len, 0);
  if(ssl_) {
    if(ret < 0) {
      int err = SSL_get_error(ssl_, ret);
      if(err == SSL_ERROR_WANT_WRITE) {
        requests_to_send_ = 1;
      }
    }
    if(SSL_is_init_finished(ssl_) && (state_ == eConnecting)) {
      requests_to_send_ = 1;
    }
  }
  return ret;
}

int SlowSocket::send_slow(const void* buf, size_t len, const SendType type) {
  int ret;
  if(ssl_) {
    if(!SSL_is_init_finished(ssl_)) {
      ret = SSL_do_handshake(ssl_);
      if(ret <= 0) {
        int err = SSL_get_error(ssl_, ret);
        if(SSL_ERROR_WANT_READ != err && SSL_ERROR_WANT_WRITE != err) {
          slowlog(LOG_ERROR, "socket %d: SSL connect error: %d\n", sockfd_, err);
          close();
          return -1;
        } else {
          if(SSL_ERROR_WANT_READ == err) {
            requests_to_send_ = 0;
          }
          else {
            requests_to_send_ = 1;
          }
          errno = EAGAIN;
          return -1;
        }
      } else { //connected and handhsake finished
        requests_to_send_ = 1;
      }
    } else {
      if(requests_to_send_ > 0) { //report for initial data only
        slowlog(LOG_DEBUG, "socket %d: SSL connection is using %s\n", sockfd_,
            SSL_get_cipher(ssl_));
      }
    }
  }
  // VA: this is not good. create a "prepare" method.
  // initial send
  if(buf_ == 0) {
    buf_ = buf;
    offset_ = len;
  }

  ret = ssl_ ? SSL_write(ssl_, buf_, offset_)
                 : send(sockfd_, buf_, offset_, 0);

  // entire data was sent
  if(ret > 0 && ret == offset_) {
    if(eInitialSend == type) {
      requests_to_send_ = 0;
    } else if(eFollowUpSend == type) {
      --followups_to_send_;
    }
    buf_ = 0;
    offset_ = 0;
  } else if(ret > 0 && ret < offset_) {
    buf_ = static_cast<const char*>(buf_) + ret;
    offset_ -= ret;
  }
  return ret;
}

void SlowSocket::close() {
  if (-1 == sockfd_) return;

  slowlog(LOG_DEBUG, "closing slow socket %d\n", sockfd_);
  if(ssl_) {
    SSL_free(ssl_);
    SSL_CTX_free(ssl_ctx_);
    ssl_ = NULL;
    ssl_ctx_ = NULL;
  }
  requests_to_send_ = 0;
  followups_to_send_ = 0;
  ::close(sockfd_);
  sockfd_ = -1;
}

void SlowSocket::set_state(SocketState state) {
  timeval t;
  gettimeofday(&t, 0);
  switch(state) {
    case eInit:
      break;
    case eConnecting:
      set_start(&t);
      break;
    case eConnected:
      set_connected(&t);
      break;
    case eError:
      break;
    case eClosed:
      set_stop(&t);
      break;
    default:
      break;
  }
  state_ = state;
}

void SlowSocket::set_last_read(const timeval* t) {
  if(read_interval_) {
    last_read_in_msec_ = timeval_to_milliseconds(t);
  }
}

const bool SlowSocket::is_ready_read(const timeval* t) const {
  if(!read_interval_) //don't bother doing anything
    return true;
  long now = timeval_to_milliseconds(t);
  if(last_read_in_msec_ == 0) {
    now = read_interval_ + 1;
  }
  if(now - last_read_in_msec_ > read_interval_) {
    return true;
  } else {
    return false;
  }
}

}  // namespace slowhttptest
<|endoftext|>


<commit_before>/*****************************************************************************
*  Copyright 2011 Sergey Shekyan
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* *****************************************************************************/

/*****
 * Author: Sergey Shekyan shekyan@gmail.com
 *
 * Slow HTTP attack  vulnerability test tool
 *  http://code.google.com/p/slowhttptest/
 *****/

#include <errno.h>
#include <fcntl.h>
#include <math.h>
#include <netdb.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>

#include <algorithm>
#include <string>

#include <openssl/ssl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>

#include "slowlog.h"
#include "slowsocket.h"
#include "slowurl.h"

namespace slowhttptest {
SlowSocket::SlowSocket()
    : sockfd_(-1),
      requests_to_send_(0),
      followups_to_send_(0),
      last_followup_timing_(0),
      offset_(0),
      ssl_(0),
      ssl_ctx_(0),
      buf_(0),
      start_in_millisecs_(0),
      connected_in_millisecs_(0),
      stop_in_millisecs_(0),
      state_(eInit),
      last_read_in_msec_(0),
      read_interval_(0),
      window_size_(-1) {
}

SlowSocket::~SlowSocket() {
  close();
}

bool SlowSocket::set_window_size(int wnd_size) {
  int actual_wnd_size = 0;
  socklen_t actual_wnd_size_len = sizeof(actual_wnd_size);
	bool ret = setsockopt(sockfd_, SOL_SOCKET, SO_RCVBUF, &wnd_size, sizeof(wnd_size));
  if(ret) {
    slowlog(LOG_ERROR, "error setting socket send buffer size to %d: %s\n", wnd_size, strerror(errno));
  } else {
    getsockopt(sockfd_, SOL_SOCKET, SO_RCVBUF, &actual_wnd_size, &actual_wnd_size_len);
    slowlog(LOG_DEBUG, "set socket %d receive buffer size to %d bytes(requested %d)\n", sockfd_, actual_wnd_size, wnd_size);
  }
  return ret;
}

int SlowSocket::set_nonblocking() {
  int flags;

  if(-1 == (flags = fcntl(sockfd_, F_GETFL, 0))) {
    flags = 0;
  }
  return fcntl(sockfd_, F_SETFL, flags | O_NONBLOCK);
}

bool SlowSocket::init(addrinfo* addr, const bool isSSL, int& maxfd,
                      int followups_to_send, int read_interval,
                      int wnd_lower_limit, int wnd_upper_limit) {
 	read_interval_ = read_interval * 1000;
  if(read_interval_) { // slow read test
    if(wnd_upper_limit == wnd_lower_limit) {
      window_size_ = wnd_upper_limit;
    } else {
      window_size_ = rand() % (wnd_upper_limit - wnd_lower_limit) + wnd_lower_limit;
      // if(!window_size_) // null is not a good choice
      // window_size_ = 1;
    }
  }
	addrinfo* res;
  bool connect_initiated = false;
  bool addr_found = false;
  for (res = addr; !connect_initiated && res; res = res->ai_next) {
    addr_found = true;
    sockfd_ = socket(res->ai_family, res->ai_socktype,
                     res->ai_protocol);
    if(-1 == sockfd_) {
      slowlog(LOG_ERROR, "failed to create socket: %s\n", strerror(errno));
      return false;
    }

    if(-1 == set_nonblocking()) {
      slowlog(LOG_ERROR, "failed to set socket %d to non-blocking \n", sockfd_);
      return false;
    }
    if(read_interval_) {
      set_window_size(window_size_);
    }
    slowlog(LOG_DEBUG, "socket %d created \n", sockfd_);
    if(connect_initiated = isSSL ? connect_ssl(addr) : connect_plain(addr)) {
      break; //found right addrinfo
    }
  }
  if(!addr_found) {
    slowlog(LOG_FATAL, "addrinfo corrupted/null\n");
    return false;
  }
  followups_to_send_ = followups_to_send;
  requests_to_send_ = 1;

  maxfd = std::max(sockfd_, maxfd);
  return true;
}

bool SlowSocket::connect_plain(addrinfo* addr) {
  errno = 0;

  if (connect(sockfd_, addr->ai_addr, addr->ai_addrlen) < 0
      && EINPROGRESS != errno) {
    slowlog(LOG_ERROR, "cannot connect socket %d: %s\n", sockfd_,
            strerror(errno));
    close();
    return false;
  }
  return true;
}

bool SlowSocket::connect_ssl(addrinfo* addr) {
  // Establish regular connection.
  if(!connect_plain(addr)) return false;

  // Init SSL related stuff.
  static bool ssl_is_initialized = false;
  if (!ssl_is_initialized) {
    SSL_library_init();
    ssl_is_initialized = true;
  }
  SSL_METHOD* method = NULL;
  method = (SSL_METHOD*)SSLv23_client_method();
  ssl_ctx_ = SSL_CTX_new(method);
  if(!ssl_ctx_) {
    slowlog(LOG_ERROR, "cannot create new SSL context\n");
    close();
    return false;
  }
  ssl_ = SSL_new(ssl_ctx_);
  if(!ssl_) {
    SSL_CTX_free(ssl_ctx_);
    slowlog(LOG_ERROR, "cannot create SSL structure for a connection\n");
    close();
    return false;
  }
  SSL_set_fd(ssl_, sockfd_);
  int ret = SSL_connect(ssl_);
  if(ret <= 0) {
    int err = SSL_get_error(ssl_, ret);
    //slowlog(LOG_ERROR, "socket %d: SSL connect error: %d\n", sockfd_, err);
    if(SSL_ERROR_WANT_READ != err && SSL_ERROR_WANT_WRITE != err) {
      close();
      return false;
    }
  }
  return true;
}

int SlowSocket::recv_slow(void* buf, size_t len) {
  int ret = ssl_ ? SSL_read(ssl_, buf, len)
                 : recv(sockfd_, buf, len, 0);
  if(ssl_) {
    if(ret < 0) {
      int err = SSL_get_error(ssl_, ret);
      if(err == SSL_ERROR_WANT_WRITE) {
        requests_to_send_ = 1;
      }
    }
    if(SSL_is_init_finished(ssl_) && (state_ == eConnecting)) {
      requests_to_send_ = 1;
    }
  }
  return ret;
}

int SlowSocket::send_slow(const void* buf, size_t len, const SendType type) {
  int ret;
  if(ssl_) {
    if(!SSL_is_init_finished(ssl_)) {
      ret = SSL_do_handshake(ssl_);
      if(ret <= 0) {
        int err = SSL_get_error(ssl_, ret);
        if(SSL_ERROR_WANT_READ != err && SSL_ERROR_WANT_WRITE != err) {
          slowlog(LOG_ERROR, "socket %d: SSL connect error: %d\n", sockfd_, err);
          close();
          return -1;
        } else {
          if(SSL_ERROR_WANT_READ == err) {
            requests_to_send_ = 0;
          }
          else {
            requests_to_send_ = 1;
          }
          errno = EAGAIN;
          return -1;
        }
      } else { //connected and handhsake finished
        requests_to_send_ = 1;
      }
    } else {
      if(requests_to_send_ > 0) { //report for initial data only
        slowlog(LOG_DEBUG, "socket %d: SSL connection is using %s\n", sockfd_,
            SSL_get_cipher(ssl_));
      }
    }
  }
  // VA: this is not good. create a "prepare" method.
  // initial send
  if(buf_ == 0) {
    buf_ = buf;
    offset_ = len;
  }

  ret = ssl_ ? SSL_write(ssl_, buf_, offset_)
                 : send(sockfd_, buf_, offset_, 0);

  // entire data was sent
  if(ret > 0 && ret == offset_) {
    if(eInitialSend == type) {
      requests_to_send_ = 0;
    } else if(eFollowUpSend == type) {
      --followups_to_send_;
    }
    buf_ = 0;
    offset_ = 0;
  } else if(ret > 0 && ret < offset_) {
    buf_ = static_cast<const char*>(buf_) + ret;
    offset_ -= ret;
  }
  return ret;
}

void SlowSocket::close() {
  if (-1 == sockfd_) return;

  slowlog(LOG_DEBUG, "closing slow socket %d\n", sockfd_);
  if(ssl_) {
    SSL_free(ssl_);
    SSL_CTX_free(ssl_ctx_);
    ssl_ = NULL;
    ssl_ctx_ = NULL;
  }
  requests_to_send_ = 0;
  followups_to_send_ = 0;
  ::close(sockfd_);
  sockfd_ = -1;
}

void SlowSocket::set_state(SocketState state) {
  timeval t;
  gettimeofday(&t, 0);
  switch(state) {
    case eInit:
      break;
    case eConnecting:
      set_start(&t);
      break;
    case eConnected:
      set_connected(&t);
      break;
    case eError:
      break;
    case eClosed:
      set_stop(&t);
      break;
    default:
      break;
  }
  state_ = state;
}

void SlowSocket::set_last_read(const timeval* t) {
  if(read_interval_) {
    last_read_in_msec_ = timeval_to_milliseconds(t);
  }
}

const bool SlowSocket::is_ready_read(const timeval* t) const {
  if(!read_interval_) //don't bother doing anything
    return true;
  long now = timeval_to_milliseconds(t);
  if(last_read_in_msec_ == 0) {
    now = read_interval_ + 1;
  }
  if(now - last_read_in_msec_ > read_interval_) {
    return true;
  } else {
    return false;
  }
}

}  // namespace slowhttptest
<commit_msg>fix to supress compilation warning on osx mavericks <commit_after>/*****************************************************************************
*  Copyright 2011 Sergey Shekyan
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* *****************************************************************************/

/*****
 * Author: Sergey Shekyan shekyan@gmail.com
 *
 * Slow HTTP attack  vulnerability test tool
 *  http://code.google.com/p/slowhttptest/
 *****/

#include <errno.h>
#include <fcntl.h>
#include <math.h>
#include <netdb.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>

#include <algorithm>
#include <string>

#include <openssl/ssl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>

#include "slowlog.h"
#include "slowsocket.h"
#include "slowurl.h"

namespace slowhttptest {
SlowSocket::SlowSocket()
    : sockfd_(-1),
      requests_to_send_(0),
      followups_to_send_(0),
      last_followup_timing_(0),
      offset_(0),
      ssl_(0),
      ssl_ctx_(0),
      buf_(0),
      start_in_millisecs_(0),
      connected_in_millisecs_(0),
      stop_in_millisecs_(0),
      state_(eInit),
      last_read_in_msec_(0),
      read_interval_(0),
      window_size_(-1) {
}

SlowSocket::~SlowSocket() {
  close();
}

bool SlowSocket::set_window_size(int wnd_size) {
  int actual_wnd_size = 0;
  socklen_t actual_wnd_size_len = sizeof(actual_wnd_size);
	bool ret = setsockopt(sockfd_, SOL_SOCKET, SO_RCVBUF, &wnd_size, sizeof(wnd_size));
  if(ret) {
    slowlog(LOG_ERROR, "error setting socket send buffer size to %d: %s\n", wnd_size, strerror(errno));
  } else {
    getsockopt(sockfd_, SOL_SOCKET, SO_RCVBUF, &actual_wnd_size, &actual_wnd_size_len);
    slowlog(LOG_DEBUG, "set socket %d receive buffer size to %d bytes(requested %d)\n", sockfd_, actual_wnd_size, wnd_size);
  }
  return ret;
}

int SlowSocket::set_nonblocking() {
  int flags;

  if(-1 == (flags = fcntl(sockfd_, F_GETFL, 0))) {
    flags = 0;
  }
  return fcntl(sockfd_, F_SETFL, flags | O_NONBLOCK);
}

bool SlowSocket::init(addrinfo* addr, const bool isSSL, int& maxfd,
                      int followups_to_send, int read_interval,
                      int wnd_lower_limit, int wnd_upper_limit) {
 	read_interval_ = read_interval * 1000;
  if(read_interval_) { // slow read test
    if(wnd_upper_limit == wnd_lower_limit) {
      window_size_ = wnd_upper_limit;
    } else {
      window_size_ = rand() % (wnd_upper_limit - wnd_lower_limit) + wnd_lower_limit;
      // if(!window_size_) // null is not a good choice
      // window_size_ = 1;
    }
  }
	addrinfo* res;
  bool connect_initiated = false;
  bool addr_found = false;
  for (res = addr; !connect_initiated && res; res = res->ai_next) {
    addr_found = true;
    sockfd_ = socket(res->ai_family, res->ai_socktype,
                     res->ai_protocol);
    if(-1 == sockfd_) {
      slowlog(LOG_ERROR, "failed to create socket: %s\n", strerror(errno));
      return false;
    }

    if(-1 == set_nonblocking()) {
      slowlog(LOG_ERROR, "failed to set socket %d to non-blocking \n", sockfd_);
      return false;
    }
    if(read_interval_) {
      set_window_size(window_size_);
    }
    slowlog(LOG_DEBUG, "socket %d created \n", sockfd_);
    if((connect_initiated = isSSL ? connect_ssl(addr) : connect_plain(addr))) {
      break; // found right addrinfo
    }
  }
  if(!addr_found) {
    slowlog(LOG_FATAL, "addrinfo corrupted/null\n");
    return false;
  }
  followups_to_send_ = followups_to_send;
  requests_to_send_ = 1;

  maxfd = std::max(sockfd_, maxfd);
  return true;
}

bool SlowSocket::connect_plain(addrinfo* addr) {
  errno = 0;

  if (connect(sockfd_, addr->ai_addr, addr->ai_addrlen) < 0
      && EINPROGRESS != errno) {
    slowlog(LOG_ERROR, "cannot connect socket %d: %s\n", sockfd_,
            strerror(errno));
    close();
    return false;
  }
  return true;
}

bool SlowSocket::connect_ssl(addrinfo* addr) {
  // Establish regular connection.
  if(!connect_plain(addr)) return false;

  // Init SSL related stuff.
  static bool ssl_is_initialized = false;
  if (!ssl_is_initialized) {
    SSL_library_init();
    ssl_is_initialized = true;
  }
  SSL_METHOD* method = NULL;
  method = (SSL_METHOD*)SSLv23_client_method();
  ssl_ctx_ = SSL_CTX_new(method);
  if(!ssl_ctx_) {
    slowlog(LOG_ERROR, "cannot create new SSL context\n");
    close();
    return false;
  }
  ssl_ = SSL_new(ssl_ctx_);
  if(!ssl_) {
    SSL_CTX_free(ssl_ctx_);
    slowlog(LOG_ERROR, "cannot create SSL structure for a connection\n");
    close();
    return false;
  }
  SSL_set_fd(ssl_, sockfd_);
  int ret = SSL_connect(ssl_);
  if(ret <= 0) {
    int err = SSL_get_error(ssl_, ret);
    //slowlog(LOG_ERROR, "socket %d: SSL connect error: %d\n", sockfd_, err);
    if(SSL_ERROR_WANT_READ != err && SSL_ERROR_WANT_WRITE != err) {
      close();
      return false;
    }
  }
  return true;
}

int SlowSocket::recv_slow(void* buf, size_t len) {
  int ret = ssl_ ? SSL_read(ssl_, buf, len)
                 : recv(sockfd_, buf, len, 0);
  if(ssl_) {
    if(ret < 0) {
      int err = SSL_get_error(ssl_, ret);
      if(err == SSL_ERROR_WANT_WRITE) {
        requests_to_send_ = 1;
      }
    }
    if(SSL_is_init_finished(ssl_) && (state_ == eConnecting)) {
      requests_to_send_ = 1;
    }
  }
  return ret;
}

int SlowSocket::send_slow(const void* buf, size_t len, const SendType type) {
  int ret;
  if(ssl_) {
    if(!SSL_is_init_finished(ssl_)) {
      ret = SSL_do_handshake(ssl_);
      if(ret <= 0) {
        int err = SSL_get_error(ssl_, ret);
        if(SSL_ERROR_WANT_READ != err && SSL_ERROR_WANT_WRITE != err) {
          slowlog(LOG_ERROR, "socket %d: SSL connect error: %d\n", sockfd_, err);
          close();
          return -1;
        } else {
          if(SSL_ERROR_WANT_READ == err) {
            requests_to_send_ = 0;
          }
          else {
            requests_to_send_ = 1;
          }
          errno = EAGAIN;
          return -1;
        }
      } else { //connected and handhsake finished
        requests_to_send_ = 1;
      }
    } else {
      if(requests_to_send_ > 0) { //report for initial data only
        slowlog(LOG_DEBUG, "socket %d: SSL connection is using %s\n", sockfd_,
            SSL_get_cipher(ssl_));
      }
    }
  }
  // VA: this is not good. create a "prepare" method.
  // initial send
  if(buf_ == 0) {
    buf_ = buf;
    offset_ = len;
  }

  ret = ssl_ ? SSL_write(ssl_, buf_, offset_)
                 : send(sockfd_, buf_, offset_, 0);

  // entire data was sent
  if(ret > 0 && ret == offset_) {
    if(eInitialSend == type) {
      requests_to_send_ = 0;
    } else if(eFollowUpSend == type) {
      --followups_to_send_;
    }
    buf_ = 0;
    offset_ = 0;
  } else if(ret > 0 && ret < offset_) {
    buf_ = static_cast<const char*>(buf_) + ret;
    offset_ -= ret;
  }
  return ret;
}

void SlowSocket::close() {
  if (-1 == sockfd_) return;

  slowlog(LOG_DEBUG, "closing slow socket %d\n", sockfd_);
  if(ssl_) {
    SSL_free(ssl_);
    SSL_CTX_free(ssl_ctx_);
    ssl_ = NULL;
    ssl_ctx_ = NULL;
  }
  requests_to_send_ = 0;
  followups_to_send_ = 0;
  ::close(sockfd_);
  sockfd_ = -1;
}

void SlowSocket::set_state(SocketState state) {
  timeval t;
  gettimeofday(&t, 0);
  switch(state) {
    case eInit:
      break;
    case eConnecting:
      set_start(&t);
      break;
    case eConnected:
      set_connected(&t);
      break;
    case eError:
      break;
    case eClosed:
      set_stop(&t);
      break;
    default:
      break;
  }
  state_ = state;
}

void SlowSocket::set_last_read(const timeval* t) {
  if(read_interval_) {
    last_read_in_msec_ = timeval_to_milliseconds(t);
  }
}

const bool SlowSocket::is_ready_read(const timeval* t) const {
  if(!read_interval_) //don't bother doing anything
    return true;
  long now = timeval_to_milliseconds(t);
  if(last_read_in_msec_ == 0) {
    now = read_interval_ + 1;
  }
  if(now - last_read_in_msec_ > read_interval_) {
    return true;
  } else {
    return false;
  }
}

}  // namespace slowhttptest
<|endoftext|>


<commit_before>// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include <iostream>
//#include "ebofs/Ebofs.h"
#include "os/FileStore.h"
#include "common/common_init.h"

#undef dout_prefix
#define dout_prefix *_dout << dbeginl

struct io {
  utime_t start, ack, commit;
  bool done() {
    return ack.sec() && commit.sec();
  }
};
map<off_t,io> writes;

Mutex lock("streamtest.cc lock");

void pr(off_t off)
{
  io &i = writes[off];
  dout(2) << off << "\t"
	  << (i.ack - i.start) << "\t"
	  << (i.commit - i.start) << dendl;
  writes.erase(off);
}

void set_start(off_t off, utime_t t)
{
  Mutex::Locker l(lock);
  writes[off].start = t;
}

void set_ack(off_t off, utime_t t)
{
  Mutex::Locker l(lock);
  writes[off].ack = t;
  if (writes[off].done())
    pr(off);
}

void set_commit(off_t off, utime_t t)
{
  Mutex::Locker l(lock);
  writes[off].commit = t;
  if (writes[off].done())
    pr(off);
}

struct C_Commit : public Context {
  off_t off;
  C_Commit(off_t o) : off(o) {}
  void finish(int r) {
    set_commit(off, g_clock.now());
  }
};

int main(int argc, const char **argv)
{
  vector<const char*> args;
  argv_to_vec(argc, argv, args);
  env_to_vec(args);
  common_init(args, NULL, false, true);

  // args
  if (args.size() < 3) return -1;
  const char *filename = args[0];
  int seconds = atoi(args[1]);
  int bytes = atoi(args[2]);
  const char *journal = 0;
  if (args.size() >= 4)
    journal = args[3];

  buffer::ptr bp(bytes);
  bp.zero();
  bufferlist bl;
  bl.push_back(bp);

  float interval = 1.0 / 1000;

  cout << "#dev " << filename
       << ", " << seconds << " seconds, " << bytes << " bytes per write" << std::endl;

  //ObjectStore *fs = new Ebofs(filename, journal);
  ObjectStore *fs = new FileStore(filename);

  if (fs->mount() < 0) {
    cout << "mount failed" << std::endl;
    return -1;
  }

  ObjectStore::Transaction ft;
  ft.create_collection(coll_t());
  fs->apply_transaction(ft);

  utime_t now = g_clock.now();
  utime_t end = now;
  end += seconds;
  off_t pos = 0;
  //cout << "stop at " << end << std::endl;
  cout << "# offset\tack\tcommit" << std::endl;
  while (now < end) {
    sobject_t poid(object_t("streamtest"), 0);
    utime_t start = now;
    set_start(pos, now);
    ObjectStore::Transaction t;
    t.write(coll_t(), poid, pos, bytes, bl);
    fs->apply_transaction(t, new C_Commit(pos));
    now = g_clock.now();
    set_ack(pos, now);
    pos += bytes;

    // wait?
    utime_t next = start;
    next += interval;
    if (now < next) {
      float s = next - now;
      s *= 1000 * 1000;  // s -> us
      //cout << "sleeping for " << s << " us" << std::endl;
      usleep((int)s);
    }
  }

  fs->umount();

}

<commit_msg>streamtest: do concurrent ios; async commit AND ack<commit_after>// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include <iostream>
//#include "ebofs/Ebofs.h"
#include "os/FileStore.h"
#include "common/common_init.h"

#undef dout_prefix
#define dout_prefix *_dout << dbeginl

struct io {
  utime_t start, ack, commit;
  bool done() {
    return ack.sec() && commit.sec();
  }
};
map<off_t,io> writes;
Cond cond;
Mutex lock("streamtest.cc lock");

unsigned concurrent = 1;
void throttle()
{
  Mutex::Locker l(lock);
  while (writes.size() >= concurrent) {
    //generic_dout(0) << "waiting" << dendl;
    cond.Wait(lock);
  }
}
void pr(off_t off)
{
  io &i = writes[off];
  generic_dout(0) << off << "\t"
	  << (i.ack - i.start) << "\t"
	  << (i.commit - i.start) << dendl;
  writes.erase(off);
  cond.Signal();
}

void set_start(off_t off, utime_t t)
{
  Mutex::Locker l(lock);
  writes[off].start = t;
}

void set_ack(off_t off, utime_t t)
{
  Mutex::Locker l(lock);
  //generic_dout(0) << "ack " << off << dendl;
  writes[off].ack = t;
  if (writes[off].done())
    pr(off);
}

void set_commit(off_t off, utime_t t)
{
  Mutex::Locker l(lock);
  //generic_dout(0) << "commit " << off << dendl;
  writes[off].commit = t;
  if (writes[off].done())
    pr(off);
}

struct C_Ack : public Context {
  off_t off;
  C_Ack(off_t o) : off(o) {}
  void finish(int r) {
    set_ack(off, g_clock.now());
  }
};
struct C_Commit : public Context {
  off_t off;
  C_Commit(off_t o) : off(o) {}
  void finish(int r) {
    set_commit(off, g_clock.now());
  }
};

int main(int argc, const char **argv)
{
  vector<const char*> args;
  argv_to_vec(argc, argv, args);
  env_to_vec(args);
  common_init(args, NULL, false, true);

  // args
  if (args.size() < 3) return -1;
  const char *filename = args[0];
  int seconds = atoi(args[1]);
  int bytes = atoi(args[2]);
  const char *journal = 0;
  if (args.size() >= 4)
    journal = args[3];
  if (args.size() >= 5)
    concurrent = atoi(args[4]);

  cout << "concurrent = " << concurrent << std::endl;

  buffer::ptr bp(bytes);
  bp.zero();
  bufferlist bl;
  bl.push_back(bp);

  //float interval = 1.0 / 1000;

  cout << "#dev " << filename
       << ", " << seconds << " seconds, " << bytes << " bytes per write" << std::endl;

  //ObjectStore *fs = new Ebofs(filename, journal);
  ObjectStore *fs = new FileStore(filename, journal);

  if (fs->mount() < 0) {
    cout << "mount failed" << std::endl;
    return -1;
  }

  ObjectStore::Transaction ft;
  ft.create_collection(coll_t());
  fs->apply_transaction(ft);

  utime_t now = g_clock.now();
  utime_t end = now;
  end += seconds;
  off_t pos = 0;
  //cout << "stop at " << end << std::endl;
  cout << "# offset\tack\tcommit" << std::endl;
  while (now < end) {
    sobject_t poid(object_t("streamtest"), 0);

    set_start(pos, g_clock.now());
    ObjectStore::Transaction *t = new ObjectStore::Transaction;
    t->write(coll_t(), poid, pos, bytes, bl);
    fs->queue_transaction(t, new C_Ack(pos), new C_Commit(pos));
    pos += bytes;

    throttle();

    // wait?
    /*
    utime_t next = start;
    next += interval;
    if (now < next) {
      float s = next - now;
      s *= 1000 * 1000;  // s -> us
      //cout << "sleeping for " << s << " us" << std::endl;
      usleep((int)s);
    }
    */
  }

  fs->umount();

}

<|endoftext|>


<commit_before>#include <libkern/OSAtomic.h>

int main()
{
	volatile OSSpinlock lock;
	OSSpinlockTry(&lock);
	return 0;
}
<commit_msg>Fix spinlock-test-darwin.cpp for spinlock detection<commit_after>#include <libkern/OSAtomic.h>

int main()
{
	volatile OSSpinLock lock;
	OSSpinLockTry(&lock);
	return 0;
}
<|endoftext|>


<commit_before>#include    <cstdlib>
#include    <cmath>
#include    <fstream>
#include    <sstream>
#include    <iostream>
#include    <iomanip>
#include    <string>
#include    <vector>
#include    <algorithm>
#include    <exception>
#include    <sys/time.h>
#include    "modules/htmTree.h"
#include    "modules/kdTree.h"
#include        "misc.h"
#include        "feat.h"
#include        "structs.h"
#include        "collision.h"
#include        "global.h"
//reduce redistributes, updates  07/02/15 rnc
int main(int argc, char **argv) {
    //// Initializations ---------------------------------------------
    srand48(1234); // Make sure we have reproducability
    check_args(argc);
    Time t, time; // t for global, time for local
    init_time(t);
    Feat F;
    MTL M;

    // Read parameters file //
    F.readInputFile(argv[1]);
    printFile(argv[1]);
    // Read Secretfile
    // Secret contains the identity of each target: QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF
    Gals Secret;
    init_time_at(time,"# reading Secret file",t);

        Secret=read_Secretfile(F.Secretfile,F);
    printf("# Read %d galaxies from %s \n",Secret.size(),F.Secretfile.c_str());
    print_time(time,"# ... took :");
    std::vector<int> count(10);
    count=count_galaxies(Secret);
    printf(" Number of galaxies by type, QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF\n");
    for(int i=0;i<8;i++){if(count[i]>0)printf (" type %d number  %d  \n",i, count[i]);}
    //read the three input fits files
    init_time_at(time,"# read target, SS, SF files",t);
    MTL Targ=read_MTLfile(F.Targfile,F,0,0);
    MTL SStars=read_MTLfile(F.SStarsfile,F,1,0);
    MTL SkyF=read_MTLfile(F.SkyFfile,F,0,1);

    if(Targ.size() == 0) {
        std::cerr << "ERROR: No targets found in " << F.Targfile << std::endl;
        myexit(1);
    }

    print_time(time,"# ... took :");
    //combine the three input files
    M=Targ;
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SStars.begin(),SStars.end());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SkyF.begin(),SkyF.end());
    printf(" M size %d \n",M.size());
    F.Ngal=M.size();
    F.Ntarg=Secret.size();

    //establish priority classes
    init_time_at(time,"# establish priority clasess",t);
    assign_priority_class(M);
    std::vector <int> count_class(M.priority_list.size(),0);
    for(int i;i<M.size();++i){
        if(!M[i].SS&&!M[i].SF){
        count_class[M[i].priority_class]+=1;
        }
    }
    for(int i;i<M.priority_list.size();++i){
        printf("  class %d  priority %d  number %d\n",i,M.priority_list[i],count_class[i]);
    }
    print_time(time,"# ... took :");

    // fiber positioners
    PP pp;
    pp.read_fiber_positions(F);
    F.Nfiber = pp.fp.size()/2;
    F.Npetal = max(pp.spectrom)+1;
    F.Nfbp = (int) (F.Nfiber/F.Npetal);// fibers per petal = 500
    pp.get_neighbors(F);
    pp.compute_fibsofsp(F);
    printf("computed neighbors\n");
    std::cout.flush();
    //P tiles in order specified by surveyFile
    Plates P = read_plate_centers(F);
    F.Nplate=P.size();
    printf("# Read %d plates from %s and %d fibers from %s\n",F.Nplate,F.tileFile.c_str(),F.Nfiber,F.fibFile.c_str());

    // Computes geometries of cb and fh: pieces of positioner - used to determine possible collisions
    F.cb = create_cb(); // cb=central body
    F.fh = create_fh(); // fh=fiber holder

    //// Collect available galaxies <-> tilefibers --------------------
    // HTM Tree of galaxies
    const double MinTreeSize = 0.01;
    init_time_at(time,"# Start building HTM tree",t);

    htmTree<struct target> T(M,MinTreeSize);
    print_time(time,"# ... took :");//T.stats();
    init_time_at(time,"# collect galaxies at ",t);

    // For plates/fibers, collect available galaxies; done in parallel
    collect_galaxies_for_all(M,T,P,pp,F);
    print_time(time,"# ... took :");//T.stats();
    init_time_at(time,"# collect available tile-fibers at",t);
    // For each galaxy, computes available tilefibers  G[i].av_tfs = [(j1,k1),(j2,k2),..]
    collect_available_tilefibers(M,P,F);

    //count number of galaxies in first pass and number not in first pass
   /*
    int totalg=0;
    int inside=0;
    int all_covered=0;
    for(int g=0;g<F.Ntarg;++g)if(M[g].av_tfs.size()>0)++all_covered;
    for(int g=0;g<F.Ntarg;++g){
        Plist v=M[g].av_tfs;
        int done=0;
        for(int i=0;i<v.size()&&done==0;++i){
            if(P[v[i].f].ipass==0){
                ++inside;
                done=1;
            }
        }
        ++totalg;
    }
    printf ("total = %d  inside = %d  covered  %d\n",totalg,inside, all_covered);
    */
    //results_on_inputs("doc/figs/",G,P,F,true);

    //// Assignment |||||||||||||||||||||||||||||||||||||||||||||||||||
    printf(" Nplate %d  Ngal %d   Nfiber %d \n", F.Nplate, F.Ngal, F.Nfiber);
    Assignment A(M,F);

    print_time(t,"# Start assignment at : ");

    // Make a plan ----------------------------------------------------
    // Plans whole survey without sky fibers, standard stars
    // assumes maximum number of observations needed for QSOs, LRGs

    simple_assign(M,P,pp,F,A);

    //check to see if there are tiles with no galaxies
    //need to keep mapping of old tile list to new tile list
    //and inverse map
    A.inv_order=initList(F.Nplate,-1);
    int inv_count=0;
    for (int j=0;j<F.Nplate ;++j){

        bool not_done=true;
        for(int k=0;k<F.Nfiber && not_done;++k){
            if(A.TF[j][k]!=-1){//fiber and therefore plate is used
                A.suborder.push_back(j);//suborder[jused] is jused-th used plate
                not_done=false;
                A.inv_order[j]=inv_count;//inv_order[j] is -1 unless used
                                //and otherwise the position of plate j in list of used plates
                inv_count++;
            }
        }
    }
    F.NUsedplate=A.suborder.size();
    //diagnostic
    for(int jused=0;jused<F.NUsedplate;++jused)printf("jused = %d j = %d \n",jused A.suborder[jused]);
    printf(" Plates actually used %d \n",F.NUsedplate);

    if(F.diagnose)diagnostic(M,Secret,F,A);

    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false); // Hist of unused fibs

    // Smooth out distribution of free fibers, and increase the number of assignments

    for (int i=0; i<1; i++) redistribute_tf(M,P,pp,F,A,0);// more iterations will improve performance slightly
    for (int i=0; i<1; i++) {
        improve(M,P,pp,F,A,0);
        redistribute_tf(M,P,pp,F,A,0);
    }
    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false);
    //try assigning SF and SS before real time assignment
    for (int jused=0;jused<F.NUsedplate;++jused){
        int j=A.suborder[jused];
        assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF for each tile
        assign_unused(j,M,P,pp,F,A);
    }
    if(F.diagnose)diagnostic(M,Secret,F,A);
    init_time_at(time,"# Begin real time assignment",t);

    //Execute plan, updating targets at intervals
    for(int i=0;i<F.pass_intervals.size();i++){
        printf(" i=%d interval %d \n",i,F.pass_intervals[i]);
    }
    std::vector <int> update_intervals=F.pass_intervals;
    update_intervals.push_back(F.NUsedplate);//to end intervals at last plate
    for(int i=0;i<update_intervals.size();++i){
        printf("i %d  update_interval %d\n",i, update_intervals[i]);
    }
    for(int i=0;i<update_intervals.size()-1;++i){//go plate by used plate
        int starter=update_intervals[i];
        printf("-- interval %d\n",i);
        for (int jused=starter; jused<update_intervals[i+1] && jused<A.suborder.size()-1; jused++) {
            printf(" jused = %d\n",jused);
            std::cout.flush();
            if (0<=jused-F.Analysis) {
                printf(" updating  jused = %d \n",jused);
                update_plan_from_one_obs(jused,Secret,M,P,pp,F,A);
            }
            else printf("\n no update\n");
            std::cout.flush();
            printf("did update\n");
            if (F.PlotPyplotTile && jused%F.PyplotInterval-1==0) pyplotTile(jused,"doc/figs",Secret,M,P,pp,F,A);

            // Update corrects all future occurrences of wrong QSOs etc and tries to observe something else

        }
        printf("-- redistribute %d\n",i);
        redistribute_tf(M,P,pp,F,A,starter);
        printf("-- improve %d\n",i);
        improve(M,P,pp,F,A,starter);
        printf("-- improve again %d\n",i);
        redistribute_tf(M,P,pp,F,A,starter);
        printf("-- diagnose %d\n",i);
        if(F.diagnose)diagnostic(M,Secret,F,A);

    }
    // check on SS and SF

    printf("-- Checking SS/SF\n");
    List SS_hist=initList(11,0);
    List SF_hist=initList(41,0);
    for(int jused=0;jused<F.NUsedplate;++jused){
        int j=A.suborder[jused];
        for (int p=0;p<F.Npetal;++p){
            int count_SS=0;
            int count_SF=0;
            for (int k=0;k<F.Nfbp;++k){
                int kk=pp.fibers_of_sp[p][k];
                int g=A.TF[j][kk];
                if(g!=-1 && M[g].SS)count_SS++;
                if(g!=-1 && M[g].SF)count_SF++;
            }
            SS_hist[count_SS]++;
            SF_hist[count_SF]++;
        }
    }
    printf(" SS distribution \n");
    for(int i=0;i<10;i++)printf("%8d",SS_hist[i]);
    printf("\n %8d \n",SS_hist[10]);

    printf(" SF distribution \n");
    for(int i=0;i<10;i++)printf("%8d",SF_hist[i]);
    printf("\n");
    for(int i=10;i<20;i++)printf("%8d",SF_hist[i]);
    printf("\n");
    for(int i=20;i<30;i++)printf("%8d",SF_hist[i]);
    printf("\n");
    for(int i=30;i<40;i++)printf("%8d",SF_hist[i]);
    printf("\n %8d \n",SF_hist[40]);

    // Results -------------------------------------------------------
    if (F.PrintAscii){
        for (int jused=0; jused<F.NUsedplate; jused++){
            int j=A.suborder[jused];
            write_FAtile_ascii(A.suborder[jused],F.outDir,M,P,pp,F,A);
        }
    }

    if (F.PrintFits) {
        for (int jused=0; jused<F.NUsedplate; jused++){
            int j=A.suborder[jused];
            fa_write(A.suborder[jused],F.outDir,M,P,pp,F,A); // Write outpu
        }
    }

	display_results("doc/figs/",Secret,M,P,pp,F,A,F.Nplate,true);
	if (F.Verif) A.verif(P,M,pp,F); // Verification that the assignment is sane

    print_time(t,"# Finished !... in");

    return(0);

}
<commit_msg>diagnostic on suborder<commit_after>#include    <cstdlib>
#include    <cmath>
#include    <fstream>
#include    <sstream>
#include    <iostream>
#include    <iomanip>
#include    <string>
#include    <vector>
#include    <algorithm>
#include    <exception>
#include    <sys/time.h>
#include    "modules/htmTree.h"
#include    "modules/kdTree.h"
#include        "misc.h"
#include        "feat.h"
#include        "structs.h"
#include        "collision.h"
#include        "global.h"
//reduce redistributes, updates  07/02/15 rnc
int main(int argc, char **argv) {
    //// Initializations ---------------------------------------------
    srand48(1234); // Make sure we have reproducability
    check_args(argc);
    Time t, time; // t for global, time for local
    init_time(t);
    Feat F;
    MTL M;

    // Read parameters file //
    F.readInputFile(argv[1]);
    printFile(argv[1]);
    // Read Secretfile
    // Secret contains the identity of each target: QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF
    Gals Secret;
    init_time_at(time,"# reading Secret file",t);

        Secret=read_Secretfile(F.Secretfile,F);
    printf("# Read %d galaxies from %s \n",Secret.size(),F.Secretfile.c_str());
    print_time(time,"# ... took :");
    std::vector<int> count(10);
    count=count_galaxies(Secret);
    printf(" Number of galaxies by type, QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF\n");
    for(int i=0;i<8;i++){if(count[i]>0)printf (" type %d number  %d  \n",i, count[i]);}
    //read the three input fits files
    init_time_at(time,"# read target, SS, SF files",t);
    MTL Targ=read_MTLfile(F.Targfile,F,0,0);
    MTL SStars=read_MTLfile(F.SStarsfile,F,1,0);
    MTL SkyF=read_MTLfile(F.SkyFfile,F,0,1);

    if(Targ.size() == 0) {
        std::cerr << "ERROR: No targets found in " << F.Targfile << std::endl;
        myexit(1);
    }

    print_time(time,"# ... took :");
    //combine the three input files
    M=Targ;
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SStars.begin(),SStars.end());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SkyF.begin(),SkyF.end());
    printf(" M size %d \n",M.size());
    F.Ngal=M.size();
    F.Ntarg=Secret.size();

    //establish priority classes
    init_time_at(time,"# establish priority clasess",t);
    assign_priority_class(M);
    std::vector <int> count_class(M.priority_list.size(),0);
    for(int i;i<M.size();++i){
        if(!M[i].SS&&!M[i].SF){
        count_class[M[i].priority_class]+=1;
        }
    }
    for(int i;i<M.priority_list.size();++i){
        printf("  class %d  priority %d  number %d\n",i,M.priority_list[i],count_class[i]);
    }
    print_time(time,"# ... took :");

    // fiber positioners
    PP pp;
    pp.read_fiber_positions(F);
    F.Nfiber = pp.fp.size()/2;
    F.Npetal = max(pp.spectrom)+1;
    F.Nfbp = (int) (F.Nfiber/F.Npetal);// fibers per petal = 500
    pp.get_neighbors(F);
    pp.compute_fibsofsp(F);
    printf("computed neighbors\n");
    std::cout.flush();
    //P tiles in order specified by surveyFile
    Plates P = read_plate_centers(F);
    F.Nplate=P.size();
    printf("# Read %d plates from %s and %d fibers from %s\n",F.Nplate,F.tileFile.c_str(),F.Nfiber,F.fibFile.c_str());

    // Computes geometries of cb and fh: pieces of positioner - used to determine possible collisions
    F.cb = create_cb(); // cb=central body
    F.fh = create_fh(); // fh=fiber holder

    //// Collect available galaxies <-> tilefibers --------------------
    // HTM Tree of galaxies
    const double MinTreeSize = 0.01;
    init_time_at(time,"# Start building HTM tree",t);

    htmTree<struct target> T(M,MinTreeSize);
    print_time(time,"# ... took :");//T.stats();
    init_time_at(time,"# collect galaxies at ",t);

    // For plates/fibers, collect available galaxies; done in parallel
    collect_galaxies_for_all(M,T,P,pp,F);
    print_time(time,"# ... took :");//T.stats();
    init_time_at(time,"# collect available tile-fibers at",t);
    // For each galaxy, computes available tilefibers  G[i].av_tfs = [(j1,k1),(j2,k2),..]
    collect_available_tilefibers(M,P,F);

    //count number of galaxies in first pass and number not in first pass
   /*
    int totalg=0;
    int inside=0;
    int all_covered=0;
    for(int g=0;g<F.Ntarg;++g)if(M[g].av_tfs.size()>0)++all_covered;
    for(int g=0;g<F.Ntarg;++g){
        Plist v=M[g].av_tfs;
        int done=0;
        for(int i=0;i<v.size()&&done==0;++i){
            if(P[v[i].f].ipass==0){
                ++inside;
                done=1;
            }
        }
        ++totalg;
    }
    printf ("total = %d  inside = %d  covered  %d\n",totalg,inside, all_covered);
    */
    //results_on_inputs("doc/figs/",G,P,F,true);

    //// Assignment |||||||||||||||||||||||||||||||||||||||||||||||||||
    printf(" Nplate %d  Ngal %d   Nfiber %d \n", F.Nplate, F.Ngal, F.Nfiber);
    Assignment A(M,F);

    print_time(t,"# Start assignment at : ");

    // Make a plan ----------------------------------------------------
    // Plans whole survey without sky fibers, standard stars
    // assumes maximum number of observations needed for QSOs, LRGs

    simple_assign(M,P,pp,F,A);

    //check to see if there are tiles with no galaxies
    //need to keep mapping of old tile list to new tile list
    //and inverse map
    A.inv_order=initList(F.Nplate,-1);
    int inv_count=0;
    for (int j=0;j<F.Nplate ;++j){

        bool not_done=true;
        for(int k=0;k<F.Nfiber && not_done;++k){
            if(A.TF[j][k]!=-1){//fiber and therefore plate is used
                A.suborder.push_back(j);//suborder[jused] is jused-th used plate
                not_done=false;
                A.inv_order[j]=inv_count;//inv_order[j] is -1 unless used
                                //and otherwise the position of plate j in list of used plates
                inv_count++;
            }
        }
    }
    F.NUsedplate=A.suborder.size();
    //diagnostic
    for(int jused=0;jused<F.NUsedplate;++jused)printf("jused = %d j = %d \n",jused ,A.suborder[jused]);
    printf(" Plates actually used %d \n",F.NUsedplate);

    if(F.diagnose)diagnostic(M,Secret,F,A);

    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false); // Hist of unused fibs

    // Smooth out distribution of free fibers, and increase the number of assignments

    for (int i=0; i<1; i++) redistribute_tf(M,P,pp,F,A,0);// more iterations will improve performance slightly
    for (int i=0; i<1; i++) {
        improve(M,P,pp,F,A,0);
        redistribute_tf(M,P,pp,F,A,0);
    }
    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false);
    //try assigning SF and SS before real time assignment
    for (int jused=0;jused<F.NUsedplate;++jused){
        int j=A.suborder[jused];
        assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF for each tile
        assign_unused(j,M,P,pp,F,A);
    }
    if(F.diagnose)diagnostic(M,Secret,F,A);
    init_time_at(time,"# Begin real time assignment",t);

    //Execute plan, updating targets at intervals
    for(int i=0;i<F.pass_intervals.size();i++){
        printf(" i=%d interval %d \n",i,F.pass_intervals[i]);
    }
    std::vector <int> update_intervals=F.pass_intervals;
    update_intervals.push_back(F.NUsedplate);//to end intervals at last plate
    for(int i=0;i<update_intervals.size();++i){
        printf("i %d  update_interval %d\n",i, update_intervals[i]);
    }
    for(int i=0;i<update_intervals.size()-1;++i){//go plate by used plate
        int starter=update_intervals[i];
        printf("-- interval %d\n",i);
        for (int jused=starter; jused<update_intervals[i+1] && jused<A.suborder.size()-1; jused++) {
            printf(" jused = %d\n",jused);
            std::cout.flush();
            if (0<=jused-F.Analysis) {
                printf(" updating  jused = %d \n",jused);
                update_plan_from_one_obs(jused,Secret,M,P,pp,F,A);
            }
            else printf("\n no update\n");
            std::cout.flush();
            printf("did update\n");
            if (F.PlotPyplotTile && jused%F.PyplotInterval-1==0) pyplotTile(jused,"doc/figs",Secret,M,P,pp,F,A);

            // Update corrects all future occurrences of wrong QSOs etc and tries to observe something else

        }
        printf("-- redistribute %d\n",i);
        redistribute_tf(M,P,pp,F,A,starter);
        printf("-- improve %d\n",i);
        improve(M,P,pp,F,A,starter);
        printf("-- improve again %d\n",i);
        redistribute_tf(M,P,pp,F,A,starter);
        printf("-- diagnose %d\n",i);
        if(F.diagnose)diagnostic(M,Secret,F,A);

    }
    // check on SS and SF

    printf("-- Checking SS/SF\n");
    List SS_hist=initList(11,0);
    List SF_hist=initList(41,0);
    for(int jused=0;jused<F.NUsedplate;++jused){
        int j=A.suborder[jused];
        for (int p=0;p<F.Npetal;++p){
            int count_SS=0;
            int count_SF=0;
            for (int k=0;k<F.Nfbp;++k){
                int kk=pp.fibers_of_sp[p][k];
                int g=A.TF[j][kk];
                if(g!=-1 && M[g].SS)count_SS++;
                if(g!=-1 && M[g].SF)count_SF++;
            }
            SS_hist[count_SS]++;
            SF_hist[count_SF]++;
        }
    }
    printf(" SS distribution \n");
    for(int i=0;i<10;i++)printf("%8d",SS_hist[i]);
    printf("\n %8d \n",SS_hist[10]);

    printf(" SF distribution \n");
    for(int i=0;i<10;i++)printf("%8d",SF_hist[i]);
    printf("\n");
    for(int i=10;i<20;i++)printf("%8d",SF_hist[i]);
    printf("\n");
    for(int i=20;i<30;i++)printf("%8d",SF_hist[i]);
    printf("\n");
    for(int i=30;i<40;i++)printf("%8d",SF_hist[i]);
    printf("\n %8d \n",SF_hist[40]);

    // Results -------------------------------------------------------
    if (F.PrintAscii){
        for (int jused=0; jused<F.NUsedplate; jused++){
            int j=A.suborder[jused];
            write_FAtile_ascii(A.suborder[jused],F.outDir,M,P,pp,F,A);
        }
    }

    if (F.PrintFits) {
        for (int jused=0; jused<F.NUsedplate; jused++){
            int j=A.suborder[jused];
            fa_write(A.suborder[jused],F.outDir,M,P,pp,F,A); // Write outpu
        }
    }

	display_results("doc/figs/",Secret,M,P,pp,F,A,F.Nplate,true);
	if (F.Verif) A.verif(P,M,pp,F); // Verification that the assignment is sane

    print_time(t,"# Finished !... in");

    return(0);

}
<|endoftext|>