<commit_before>// Copyright (c) 2012 Plenluno All rights reserved.

#include <assert.h>

#include "libnode/http/server.h"

#include "./parser.h"
#include "./server_request_impl.h"
#include "./server_response_impl.h"
#include "../net/server_impl.h"
#include "../net/socket_impl.h"

namespace libj {
namespace node {
namespace http {

class ServerImpl : public FlagMixin, public Server {
 public:
    static Ptr create() {
        ServerImpl* httpSrv = new ServerImpl();
        httpSrv->server_->setFlag(net::ServerImpl::ALLOW_HALF_OPEN);
        httpSrv->addListener(
            EVENT_CONNECTION,
            ServerOnConnection::create(httpSrv));
        httpSrv->addListener(
            EVENT_CLIENT_ERROR,
            ServerOnClientError::create());
        return Ptr(httpSrv);
    }

 private:
    static void freeParser(Parser* parser) {
        assert(parser);
        parser->free();
        delete parser;
    }

    static void httpSocketSetup(net::SocketImpl::Ptr socket) {
        SocketOnDrain::Ptr onDrain = SocketOnDrain::create(socket);
        socket->removeListener(net::Socket::EVENT_DRAIN, onDrain);
        socket->on(net::Socket::EVENT_DRAIN, onDrain);
    }

    static void abortIncoming(JsArray::Ptr incomings) {
        static const String::CPtr strAborted = String::intern("aborted");

        while (incomings->length()) {
            IncomingMessage::Ptr req =
                toPtr<IncomingMessage>(incomings->shift());
            req->emit(strAborted);
            req->emit(IncomingMessage::EVENT_CLOSE);
        }
    }

 private:
    class SocketOnDrain : LIBJ_JS_FUNCTION(SocketOnDrain)
        static Ptr create(net::SocketImpl::Ptr socket) {
            return Ptr(new SocketOnDrain(&(*socket)));
        }

        Value operator()(JsArray::Ptr args) {
            OutgoingMessage* httpMessage = socket_->httpMessage();
            if (httpMessage)
                httpMessage->emit(OutgoingMessage::EVENT_DRAIN);
            return libj::Status::OK;
        }

     private:
        net::SocketImpl* socket_;

        SocketOnDrain(net::SocketImpl* sock) : socket_(sock) {}
    };

    class SocketOnClose : LIBJ_JS_FUNCTION(SocketOnClose)
     public:
        static Ptr create(Parser* parser, JsArray::Ptr incomings) {
            return Ptr(new SocketOnClose(parser, incomings));
        }

        Value operator()(JsArray::Ptr args) {
            freeParser(parser_);
            parser_ = NULL;
            abortIncoming(incomings_);
            return libj::Status::OK;
        }

     private:
        Parser* parser_;
        JsArray::Ptr incomings_;

        SocketOnClose(Parser* parser, JsArray::Ptr incomings)
            : parser_(parser)
            , incomings_(incomings) {}
    };

    class SocketOnTimeout : LIBJ_JS_FUNCTION(SocketOnTimeout)
     public:
        static Ptr create(net::SocketImpl::Ptr socket) {
            return Ptr(new SocketOnTimeout(&(*socket)));
        }

        Value operator()(JsArray::Ptr args) {
            socket_->destroy();
            return libj::Status::OK;
        }

     private:
        net::SocketImpl* socket_;

        SocketOnTimeout(net::SocketImpl* sock) : socket_(sock) {}
    };

    class SocketOnError : LIBJ_JS_FUNCTION(SocketOnError)
     public:
        static Ptr create(
            ServerImpl* server,
            net::SocketImpl::Ptr socket) {
            return Ptr(new SocketOnError(server, &(*socket)));
        }

        Value operator()(JsArray::Ptr args) {
            Error::CPtr err = args->getCPtr<Error>(0);
            self_->emit(EVENT_CLIENT_ERROR, err, socket_);
            return libj::Status::OK;
        }

     private:
        ServerImpl* self_;
        net::SocketImpl* socket_;

        SocketOnError(
            ServerImpl* srv,
            net::SocketImpl* sock)
            : self_(srv)
            , socket_(sock) {}
    };

    class SocketOnData : LIBJ_JS_FUNCTION(SocketOnData)
     public:
        static Ptr create(
            ServerImpl* server,
            Parser* parser,
            net::SocketImpl::Ptr socket,
            JsFunction::Ptr onClose) {
            return Ptr(new SocketOnData(server, parser, &(*socket), onClose));
        }

        Value operator()(JsArray::Ptr args) {
            static const String::CPtr methodConnect = String::create("CONNECT");

            Buffer::CPtr buf = args->getCPtr<Buffer>(0);
            Int bytesParsed = parser_->execute(buf);
            IncomingMessage::Ptr req = parser_->incoming();
            if (bytesParsed < 0) {
                libj::Error::CPtr err =
                    libj::Error::create(libj::Error::ILLEGAL_DATA_FORMAT);
                socket_->destroy(err);
            } else if (req && req->hasFlag(IncomingMessage::UPGRADE)) {
                socket_->setOnData(JsFunction::null());
                socket_->setOnEnd(JsFunction::null());
                socket_->removeListener(net::Socket::EVENT_CLOSE, onClose_);

                parser_->finish();
                freeParser(parser_);
                parser_ = NULL;

                Boolean isConnect = req->method()->equals(methodConnect);
                String::CPtr event = isConnect ? EVENT_CONNECT : EVENT_UPGRADE;
                if (self_->listeners(event)->length()) {
                    self_->emit(
                        event,
                        ServerRequestImpl::create(req),
                        req->socket(),
                        buf->slice(bytesParsed));
                } else {
                    socket_->destroy();
                }
            }
            return libj::Status::OK;
        }

     private:
        ServerImpl* self_;
        Parser* parser_;
        net::SocketImpl* socket_;
        JsFunction::Ptr onClose_;

        SocketOnData(
            ServerImpl* srv,
            Parser* parser,
            net::SocketImpl* sock,
            JsFunction::Ptr onClose)
            : self_(srv)
            , parser_(parser)
            , socket_(sock)
            , onClose_(onClose) {}
    };

    class SocketOnEnd : LIBJ_JS_FUNCTION(SocketOnEnd)
     public:
        static Ptr create(
            ServerImpl* server,
            Parser* parser,
            net::SocketImpl::Ptr socket,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings) {
            return Ptr(new SocketOnEnd(
                server, parser, &(*socket), incomings, outgoings));
        }

        Value operator()(JsArray::Ptr args) {
            if (!parser_->finish()) {
                libj::Error::CPtr err =
                    libj::Error::create(libj::Error::ILLEGAL_DATA_FORMAT);
                socket_->destroy(err);
            }

            if (!self_->hasFlag(HTTP_ALLOW_HALF_OPEN)) {
                abortIncoming(incomings_);
                if (socket_->writable()) socket_->end();
            } else if (outgoings_->length()) {
                Size lastIndex = outgoings_->length() - 1;
                OutgoingMessage::Ptr lastMsg =
                    outgoings_->getPtr<OutgoingMessage>(lastIndex);
                lastMsg->setFlag(OutgoingMessage::LAST);
            } else if (socket_->httpMessage()) {
                socket_->httpMessage()->setFlag(OutgoingMessage::LAST);
            } else if (socket_->writable()) {
                socket_->end();
            }
            return libj::Status::OK;
        }

     private:
        ServerImpl* self_;
        Parser* parser_;
        net::SocketImpl* socket_;
        JsArray::Ptr incomings_;
        JsArray::Ptr outgoings_;

        SocketOnEnd(
            ServerImpl* srv,
            Parser* parser,
            net::SocketImpl* sock,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings)
            : self_(srv)
            , parser_(parser)
            , socket_(sock)
            , incomings_(incomings)
            , outgoings_(outgoings) {}
    };

    class OutgoingMessageOnFinish : LIBJ_JS_FUNCTION(OutgoingMessageOnFinish)
     public:
        static Ptr create(
            OutgoingMessage::Ptr res,
            net::SocketImpl::Ptr socket,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings) {
            return Ptr(new OutgoingMessageOnFinish(
                &(*res), socket, incomings, outgoings));
        }

        Value operator()(JsArray::Ptr args) {
            incomings_->shift();
            res_->detachSocket(socket_);
            if (res_->hasFlag(OutgoingMessage::LAST)) {
                socket_->destroySoon();
            } else {
                OutgoingMessage::Ptr msg =
                    toPtr<OutgoingMessage>(outgoings_->shift());
                if (msg) {
                    msg->assignSocket(socket_);
                }
            }
            return libj::Status::OK;
        }

     private:
        OutgoingMessage* res_;
        net::SocketImpl::Ptr socket_;
        JsArray::Ptr incomings_;
        JsArray::Ptr outgoings_;

        OutgoingMessageOnFinish(
            OutgoingMessage* res,
            net::SocketImpl::Ptr socket,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings)
            : res_(res)
            , socket_(socket)
            , incomings_(incomings)
            , outgoings_(outgoings) {}
    };

    class ParserOnIncoming : LIBJ_JS_FUNCTION(ParserOnIncoming)
     public:
        static Ptr create(
            ServerImpl* server,
            net::SocketImpl::Ptr socket,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings) {
            return Ptr(new ParserOnIncoming(
                server, socket, incomings, outgoings));
        }

        Value operator()(JsArray::Ptr args) {
            static const String::CPtr strExpect = String::intern("expect");
            static const String::CPtr strFinish = String::intern("finish");
            static const String::CPtr strV11    = String::intern("1.1");
            static const String::CPtr contExp = String::create("100-continue");

            IncomingMessage::Ptr in = args->getPtr<IncomingMessage>(0);
            Boolean shouldKeepAlive = false;
            to<Boolean>(args->get(1), &shouldKeepAlive);

            OutgoingMessage::Ptr out = OutgoingMessage::create();
            out->setFlag(OutgoingMessage::SERVER_RESPONSE);
            incomings_->push(in);
            if (shouldKeepAlive)
                out->setFlag(OutgoingMessage::SHOULD_KEEP_ALIVE);

            OutgoingMessage* httpMessage = socket_->httpMessage();
            if (httpMessage) {
                outgoings_->push(out);
            } else {
                out->assignSocket(socket_);
            }

            out->on(
                strFinish,
                OutgoingMessageOnFinish::create(
                    out, socket_, incomings_, outgoings_));

            ServerRequest::Ptr req = ServerRequestImpl::create(in);
            ServerResponse::Ptr res = ServerResponseImpl::create(out);
            String::CPtr expectHeader = in->getHeader(strExpect);
            if (in->httpVersion()->equals(strV11) &&
                expectHeader &&
                expectHeader->toLowerCase()->equals(contExp)) {
                out->setFlag(OutgoingMessage::EXPECT_CONTINUE);
                if (self_->listeners(EVENT_CHECK_CONTINUE)->length()) {
                    self_->emit(EVENT_CHECK_CONTINUE, req, res);
                } else {
                    out->writeContinue();
                    self_->emit(EVENT_REQUEST, req, res);
                }
            } else {
                self_->emit(EVENT_REQUEST, req, res);
            }
            return false;
        }

     private:
        ServerImpl* self_;
        net::SocketImpl::Ptr socket_;
        JsArray::Ptr incomings_;
        JsArray::Ptr outgoings_;

        ParserOnIncoming(
            ServerImpl* srv,
            net::SocketImpl::Ptr sock,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings)
            : self_(srv)
            , socket_(sock)
            , incomings_(incomings)
            , outgoings_(outgoings) {}
    };

    class ServerOnConnection : LIBJ_JS_FUNCTION(ServerOnConnection)
     public:
        static Ptr create(ServerImpl* srv) {
            return Ptr(new ServerOnConnection(srv));
        }

        Value operator()(JsArray::Ptr args) {
            net::SocketImpl::Ptr socket = toPtr<net::SocketImpl>(args->get(0));
            assert(socket);

            JsArray::Ptr incomings = JsArray::create();
            JsArray::Ptr outgoings = JsArray::create();

            httpSocketSetup(socket);

#if 0
            socket->setTimeout(2 * 60 * 1000);
            socket->once(
                net::Socket::EVENT_TIMEOUT,
                SocketOnTimeout::create(socket));
#endif

            Size maxHeaders;
            if (self_->maxHeadersCount_) {
                maxHeaders = self_->maxHeadersCount_;
            } else {
                maxHeaders = 2000;
            }
            Parser* parser = new Parser(HTTP_REQUEST, socket, maxHeaders);
            socket->setParser(parser);

            JsFunction::Ptr socketOnClose =
                SocketOnClose::create(parser, incomings);
            socket->addListener(
                EVENT_ERROR,
                SocketOnError::create(self_, socket));
            socket->addListener(
                EVENT_CLOSE,
                socketOnClose);

            socket->setOnData(
                SocketOnData::create(
                    self_, parser, socket, socketOnClose));
            socket->setOnEnd(
                SocketOnEnd::create(
                    self_, parser, socket, incomings, outgoings));

            parser->setOnIncoming(
                ParserOnIncoming::create(
                    self_, socket, incomings, outgoings));
            return libj::Status::OK;
        }

     private:
        ServerImpl* self_;

        ServerOnConnection(ServerImpl* srv) : self_(srv) {}
    };

    class ServerOnClientError : LIBJ_JS_FUNCTION(ServerOnClientError)
     public:
        static Ptr create() {
            return Ptr(new ServerOnClientError());
        }

        Value operator()(JsArray::Ptr args) {
            libj::Error::CPtr err = args->getCPtr<libj::Error>(0);
            net::SocketImpl* socket = NULL;
            to<net::SocketImpl*>(args->get(1), &socket);
            assert(socket);
            socket->destroy(err);
            return libj::Status::OK;
        }
    };

 public:
    enum Flag {
        HTTP_ALLOW_HALF_OPEN = 1 << 0,
    };

 private:
    net::ServerImpl::Ptr server_;
    Size maxHeadersCount_;

    ServerImpl()
        : server_(net::ServerImpl::create())
        , maxHeadersCount_(0) {}

    LIBNODE_NET_SERVER_IMPL(server_);
};

LIBJ_SYMBOL_DEF(Server::EVENT_REQUEST,        "request");
LIBJ_SYMBOL_DEF(Server::EVENT_CHECK_CONTINUE, "checkContinue");
LIBJ_SYMBOL_DEF(Server::EVENT_CONNECT,        "connect");
LIBJ_SYMBOL_DEF(Server::EVENT_UPGRADE,        "upgrade");
LIBJ_SYMBOL_DEF(Server::EVENT_CLIENT_ERROR,   "clientError");

Server::Ptr Server::create() {
    return ServerImpl::create();
}

}  // namespace http
}  // namespace node
}  // namespace libj
<commit_msg>refactor http::Server using Symbol<commit_after>// Copyright (c) 2012 Plenluno All rights reserved.

#include <assert.h>

#include "libnode/http/server.h"

#include "./parser.h"
#include "./server_request_impl.h"
#include "./server_response_impl.h"
#include "../net/server_impl.h"
#include "../net/socket_impl.h"

namespace libj {
namespace node {
namespace http {

class ServerImpl : public FlagMixin, public Server {
 public:
    static Ptr create() {
        ServerImpl* httpSrv = new ServerImpl();
        httpSrv->server_->setFlag(net::ServerImpl::ALLOW_HALF_OPEN);
        httpSrv->addListener(
            EVENT_CONNECTION,
            ServerOnConnection::create(httpSrv));
        httpSrv->addListener(
            EVENT_CLIENT_ERROR,
            ServerOnClientError::create());
        return Ptr(httpSrv);
    }

 private:
    static void freeParser(Parser* parser) {
        assert(parser);
        parser->free();
        delete parser;
    }

    static void httpSocketSetup(net::SocketImpl::Ptr socket) {
        SocketOnDrain::Ptr onDrain = SocketOnDrain::create(socket);
        socket->removeListener(net::Socket::EVENT_DRAIN, onDrain);
        socket->on(net::Socket::EVENT_DRAIN, onDrain);
    }

    static void abortIncoming(JsArray::Ptr incomings) {
        LIBJ_STATIC_SYMBOL_DEF(EVENT_ABORTED, "aborted");

        while (incomings->length()) {
            IncomingMessage::Ptr req =
                toPtr<IncomingMessage>(incomings->shift());
            req->emit(EVENT_ABORTED);
            req->emit(IncomingMessage::EVENT_CLOSE);
        }
    }

 private:
    class SocketOnDrain : LIBJ_JS_FUNCTION(SocketOnDrain)
        static Ptr create(net::SocketImpl::Ptr socket) {
            return Ptr(new SocketOnDrain(&(*socket)));
        }

        Value operator()(JsArray::Ptr args) {
            OutgoingMessage* httpMessage = socket_->httpMessage();
            if (httpMessage)
                httpMessage->emit(OutgoingMessage::EVENT_DRAIN);
            return libj::Status::OK;
        }

     private:
        net::SocketImpl* socket_;

        SocketOnDrain(net::SocketImpl* sock) : socket_(sock) {}
    };

    class SocketOnClose : LIBJ_JS_FUNCTION(SocketOnClose)
     public:
        static Ptr create(Parser* parser, JsArray::Ptr incomings) {
            return Ptr(new SocketOnClose(parser, incomings));
        }

        Value operator()(JsArray::Ptr args) {
            freeParser(parser_);
            parser_ = NULL;
            abortIncoming(incomings_);
            return libj::Status::OK;
        }

     private:
        Parser* parser_;
        JsArray::Ptr incomings_;

        SocketOnClose(Parser* parser, JsArray::Ptr incomings)
            : parser_(parser)
            , incomings_(incomings) {}
    };

    class SocketOnTimeout : LIBJ_JS_FUNCTION(SocketOnTimeout)
     public:
        static Ptr create(net::SocketImpl::Ptr socket) {
            return Ptr(new SocketOnTimeout(&(*socket)));
        }

        Value operator()(JsArray::Ptr args) {
            socket_->destroy();
            return libj::Status::OK;
        }

     private:
        net::SocketImpl* socket_;

        SocketOnTimeout(net::SocketImpl* sock) : socket_(sock) {}
    };

    class SocketOnError : LIBJ_JS_FUNCTION(SocketOnError)
     public:
        static Ptr create(
            ServerImpl* server,
            net::SocketImpl::Ptr socket) {
            return Ptr(new SocketOnError(server, &(*socket)));
        }

        Value operator()(JsArray::Ptr args) {
            Error::CPtr err = args->getCPtr<Error>(0);
            self_->emit(EVENT_CLIENT_ERROR, err, socket_);
            return libj::Status::OK;
        }

     private:
        ServerImpl* self_;
        net::SocketImpl* socket_;

        SocketOnError(
            ServerImpl* srv,
            net::SocketImpl* sock)
            : self_(srv)
            , socket_(sock) {}
    };

    class SocketOnData : LIBJ_JS_FUNCTION(SocketOnData)
     public:
        static Ptr create(
            ServerImpl* server,
            Parser* parser,
            net::SocketImpl::Ptr socket,
            JsFunction::Ptr onClose) {
            return Ptr(new SocketOnData(server, parser, &(*socket), onClose));
        }

        Value operator()(JsArray::Ptr args) {
            LIBJ_STATIC_SYMBOL_DEF(symConnect, "CONNECT");

            Buffer::CPtr buf = args->getCPtr<Buffer>(0);
            Int bytesParsed = parser_->execute(buf);
            IncomingMessage::Ptr req = parser_->incoming();
            if (bytesParsed < 0) {
                libj::Error::CPtr err =
                    libj::Error::create(libj::Error::ILLEGAL_DATA_FORMAT);
                socket_->destroy(err);
            } else if (req && req->hasFlag(IncomingMessage::UPGRADE)) {
                socket_->setOnData(JsFunction::null());
                socket_->setOnEnd(JsFunction::null());
                socket_->removeListener(net::Socket::EVENT_CLOSE, onClose_);

                parser_->finish();
                freeParser(parser_);
                parser_ = NULL;

                Boolean isConnect = req->method()->equals(symConnect);
                String::CPtr event = isConnect ? EVENT_CONNECT : EVENT_UPGRADE;
                if (self_->listeners(event)->length()) {
                    self_->emit(
                        event,
                        ServerRequestImpl::create(req),
                        req->socket(),
                        buf->slice(bytesParsed));
                } else {
                    socket_->destroy();
                }
            }
            return libj::Status::OK;
        }

     private:
        ServerImpl* self_;
        Parser* parser_;
        net::SocketImpl* socket_;
        JsFunction::Ptr onClose_;

        SocketOnData(
            ServerImpl* srv,
            Parser* parser,
            net::SocketImpl* sock,
            JsFunction::Ptr onClose)
            : self_(srv)
            , parser_(parser)
            , socket_(sock)
            , onClose_(onClose) {}
    };

    class SocketOnEnd : LIBJ_JS_FUNCTION(SocketOnEnd)
     public:
        static Ptr create(
            ServerImpl* server,
            Parser* parser,
            net::SocketImpl::Ptr socket,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings) {
            return Ptr(new SocketOnEnd(
                server, parser, &(*socket), incomings, outgoings));
        }

        Value operator()(JsArray::Ptr args) {
            if (!parser_->finish()) {
                libj::Error::CPtr err =
                    libj::Error::create(libj::Error::ILLEGAL_DATA_FORMAT);
                socket_->destroy(err);
            }

            if (!self_->hasFlag(HTTP_ALLOW_HALF_OPEN)) {
                abortIncoming(incomings_);
                if (socket_->writable()) socket_->end();
            } else if (outgoings_->length()) {
                Size lastIndex = outgoings_->length() - 1;
                OutgoingMessage::Ptr lastMsg =
                    outgoings_->getPtr<OutgoingMessage>(lastIndex);
                lastMsg->setFlag(OutgoingMessage::LAST);
            } else if (socket_->httpMessage()) {
                socket_->httpMessage()->setFlag(OutgoingMessage::LAST);
            } else if (socket_->writable()) {
                socket_->end();
            }
            return libj::Status::OK;
        }

     private:
        ServerImpl* self_;
        Parser* parser_;
        net::SocketImpl* socket_;
        JsArray::Ptr incomings_;
        JsArray::Ptr outgoings_;

        SocketOnEnd(
            ServerImpl* srv,
            Parser* parser,
            net::SocketImpl* sock,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings)
            : self_(srv)
            , parser_(parser)
            , socket_(sock)
            , incomings_(incomings)
            , outgoings_(outgoings) {}
    };

    class OutgoingMessageOnFinish : LIBJ_JS_FUNCTION(OutgoingMessageOnFinish)
     public:
        static Ptr create(
            OutgoingMessage::Ptr res,
            net::SocketImpl::Ptr socket,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings) {
            return Ptr(new OutgoingMessageOnFinish(
                &(*res), socket, incomings, outgoings));
        }

        Value operator()(JsArray::Ptr args) {
            incomings_->shift();
            res_->detachSocket(socket_);
            if (res_->hasFlag(OutgoingMessage::LAST)) {
                socket_->destroySoon();
            } else {
                OutgoingMessage::Ptr msg =
                    toPtr<OutgoingMessage>(outgoings_->shift());
                if (msg) {
                    msg->assignSocket(socket_);
                }
            }
            return libj::Status::OK;
        }

     private:
        OutgoingMessage* res_;
        net::SocketImpl::Ptr socket_;
        JsArray::Ptr incomings_;
        JsArray::Ptr outgoings_;

        OutgoingMessageOnFinish(
            OutgoingMessage* res,
            net::SocketImpl::Ptr socket,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings)
            : res_(res)
            , socket_(socket)
            , incomings_(incomings)
            , outgoings_(outgoings) {}
    };

    class ParserOnIncoming : LIBJ_JS_FUNCTION(ParserOnIncoming)
     public:
        static Ptr create(
            ServerImpl* server,
            net::SocketImpl::Ptr socket,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings) {
            return Ptr(new ParserOnIncoming(
                server, socket, incomings, outgoings));
        }

        Value operator()(JsArray::Ptr args) {
            LIBJ_STATIC_SYMBOL_DEF(EVENT_FINISH, "finish");
            LIBJ_STATIC_SYMBOL_DEF(symV11,       "1.1");
            LIBJ_STATIC_SYMBOL_DEF(symExpect,    "expect");
            LIBJ_STATIC_SYMBOL_DEF(symContExp,   "100-continue");

            IncomingMessage::Ptr in = args->getPtr<IncomingMessage>(0);
            Boolean shouldKeepAlive = false;
            to<Boolean>(args->get(1), &shouldKeepAlive);

            OutgoingMessage::Ptr out = OutgoingMessage::create();
            out->setFlag(OutgoingMessage::SERVER_RESPONSE);
            incomings_->push(in);
            if (shouldKeepAlive)
                out->setFlag(OutgoingMessage::SHOULD_KEEP_ALIVE);

            OutgoingMessage* httpMessage = socket_->httpMessage();
            if (httpMessage) {
                outgoings_->push(out);
            } else {
                out->assignSocket(socket_);
            }

            out->on(
                EVENT_FINISH,
                OutgoingMessageOnFinish::create(
                    out, socket_, incomings_, outgoings_));

            ServerRequest::Ptr req = ServerRequestImpl::create(in);
            ServerResponse::Ptr res = ServerResponseImpl::create(out);
            String::CPtr expectHeader = in->getHeader(symExpect);
            if (in->httpVersion()->equals(symV11) &&
                expectHeader &&
                expectHeader->toLowerCase()->equals(symContExp)) {
                out->setFlag(OutgoingMessage::EXPECT_CONTINUE);
                if (self_->listeners(EVENT_CHECK_CONTINUE)->length()) {
                    self_->emit(EVENT_CHECK_CONTINUE, req, res);
                } else {
                    out->writeContinue();
                    self_->emit(EVENT_REQUEST, req, res);
                }
            } else {
                self_->emit(EVENT_REQUEST, req, res);
            }
            return false;
        }

     private:
        ServerImpl* self_;
        net::SocketImpl::Ptr socket_;
        JsArray::Ptr incomings_;
        JsArray::Ptr outgoings_;

        ParserOnIncoming(
            ServerImpl* srv,
            net::SocketImpl::Ptr sock,
            JsArray::Ptr incomings,
            JsArray::Ptr outgoings)
            : self_(srv)
            , socket_(sock)
            , incomings_(incomings)
            , outgoings_(outgoings) {}
    };

    class ServerOnConnection : LIBJ_JS_FUNCTION(ServerOnConnection)
     public:
        static Ptr create(ServerImpl* srv) {
            return Ptr(new ServerOnConnection(srv));
        }

        Value operator()(JsArray::Ptr args) {
            net::SocketImpl::Ptr socket = toPtr<net::SocketImpl>(args->get(0));
            assert(socket);

            JsArray::Ptr incomings = JsArray::create();
            JsArray::Ptr outgoings = JsArray::create();

            httpSocketSetup(socket);

#if 0
            socket->setTimeout(2 * 60 * 1000);
            socket->once(
                net::Socket::EVENT_TIMEOUT,
                SocketOnTimeout::create(socket));
#endif

            Size maxHeaders;
            if (self_->maxHeadersCount_) {
                maxHeaders = self_->maxHeadersCount_;
            } else {
                maxHeaders = 2000;
            }
            Parser* parser = new Parser(HTTP_REQUEST, socket, maxHeaders);
            socket->setParser(parser);

            JsFunction::Ptr socketOnClose =
                SocketOnClose::create(parser, incomings);
            socket->addListener(
                EVENT_ERROR,
                SocketOnError::create(self_, socket));
            socket->addListener(
                EVENT_CLOSE,
                socketOnClose);

            socket->setOnData(
                SocketOnData::create(
                    self_, parser, socket, socketOnClose));
            socket->setOnEnd(
                SocketOnEnd::create(
                    self_, parser, socket, incomings, outgoings));

            parser->setOnIncoming(
                ParserOnIncoming::create(
                    self_, socket, incomings, outgoings));
            return libj::Status::OK;
        }

     private:
        ServerImpl* self_;

        ServerOnConnection(ServerImpl* srv) : self_(srv) {}
    };

    class ServerOnClientError : LIBJ_JS_FUNCTION(ServerOnClientError)
     public:
        static Ptr create() {
            return Ptr(new ServerOnClientError());
        }

        Value operator()(JsArray::Ptr args) {
            libj::Error::CPtr err = args->getCPtr<libj::Error>(0);
            net::SocketImpl* socket = NULL;
            to<net::SocketImpl*>(args->get(1), &socket);
            assert(socket);
            socket->destroy(err);
            return libj::Status::OK;
        }
    };

 public:
    enum Flag {
        HTTP_ALLOW_HALF_OPEN = 1 << 0,
    };

 private:
    net::ServerImpl::Ptr server_;
    Size maxHeadersCount_;

    ServerImpl()
        : server_(net::ServerImpl::create())
        , maxHeadersCount_(0) {}

    LIBNODE_NET_SERVER_IMPL(server_);
};

LIBJ_SYMBOL_DEF(Server::EVENT_REQUEST,        "request");
LIBJ_SYMBOL_DEF(Server::EVENT_CHECK_CONTINUE, "checkContinue");
LIBJ_SYMBOL_DEF(Server::EVENT_CONNECT,        "connect");
LIBJ_SYMBOL_DEF(Server::EVENT_UPGRADE,        "upgrade");
LIBJ_SYMBOL_DEF(Server::EVENT_CLIENT_ERROR,   "clientError");

Server::Ptr Server::create() {
    return ServerImpl::create();
}

}  // namespace http
}  // namespace node
}  // namespace libj
<|endoftext|>


<commit_before>#pragma once
#include <map>
#include <set>
#include <tuple>
#include <regex>
#include <string>
#include <thread>
#include "net/acceptor.hpp"
#include "http/headers.hpp"

namespace http {

using namespace std::literals;

class controller
{
public:

    using callback = std::function<std::string(std::string_view,std::string_view)>;

    void text(const std::string& content)
    {
        m_content_type = "text/plain"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void html(const std::string& content)
    {
        m_content_type = "text/html"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void css(const std::string& content)
    {
        m_content_type = "text/css"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void script(const std::string& content)
    {
        m_content_type = "application/javascript"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void json(const std::string& content)
    {
        m_content_type = "application/json"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void xml(const std::string& content)
    {
        m_content_type = "application/xml"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void response(std::string_view content_type, callback cb)
    {
        m_content_type = content_type;
        m_callback = cb;
    }

    std::tuple<std::string,std::string> render(std::string_view request = "", std::string_view body = "") const
    {
        return {m_callback(request,body),m_content_type};
    }

    void content_type(std::string_view type)
    {
        m_content_type = type;
    }

    const std::string& content_type() const
    {
        return m_content_type;
    }

private:

    std::string m_content_type = "*/*"s;

    callback m_callback = [](std::string_view, std::string_view){return "Not Found"s;};
};

class server
{
public:

    controller& get(const std::string& path)
    {
        m_methods.insert("GET"s);
        return m_router[path]["GET"s];
    }

    controller& head(const std::string& path)
    {
        m_methods.insert("HEAD"s);
        return m_router[path]["HEAD"s];
    }

    controller& post(const std::string& path)
    {
        m_methods.insert("POST"s);
        return m_router[path]["POST"s];
    }

    controller& put(const std::string& path)
    {
        m_methods.insert("PUT"s);
        return m_router[path]["PUT"s];
    }

    controller& patch(const std::string& path)
    {
        m_methods.insert("PATCH"s);
        return m_router[path]["PATCH"s];
    }

    controller& destroy(const std::string& path)
    {
        m_methods.insert("DELETE"s);
        return m_router[path]["DELETE"s];
    }

    void listen(std::string_view serice_or_port = "http")
    {
        auto endpoint = net::acceptor{"0.0.0.0", serice_or_port};
        endpoint.timeout(0s); // no timeout
        slog << notice << "Started up at " << endpoint.host() << ":" << endpoint.service_or_port() << flush;
        while(true)
        {
            auto client = endpoint.accept();
            slog << notice << "Accepted connection from " << std::get<1>(client) << ":" << std::get<2>(client) << flush;
            std::thread{[&]{handle(std::move(std::get<0>(client)));}}.detach();
        }
    }

    void public_paths(std::string_view regex)
    {
        m_public_paths = regex;
    }

    void credentials(std::initializer_list<std::string> tokens)
    {
        m_credentials.insert(tokens);
    }

private:

    auto date() const
    {
        return ext::to_rfc1123(std::chrono::system_clock::now());
    }

    const auto& host() const
    {
        static auto s_host = syslog::gethostname();
        return s_host;
    }

    auto methods() const
    {
        auto tmp = ""s;
        for(const auto&  m : m_methods)
        {
            if(not tmp.empty()) tmp += ", "s;
            tmp += m;
        }
        return tmp;
    }

    const auto& content_type() const
    {
        return m_content_type;
    }

    void handle(endpointstream client)
    try
    {
        while(client)
        {
            auto method = ""s, uri = ""s, version = ""s;
            client >> method >> uri >> version;

            if(not client.good()) break;

            slog << info << "HTTP request \"" << method << ' ' << uri << ' ' << version << "\"" << flush;

            client >> std::ws;

            auto headers = http::headers{};
            client >> headers;
            // headers object will do the logging into syslog

            client.ignore(2); // crlf

            auto content_length = 0ull;
            if(headers.contains("content-length"))
                content_length = std::stoll(headers["content-length"]);

            auto authorization = ""s;
            if(headers.contains("authorization"))
                authorization = headers["authorization"];

            auto body = std::string(content_length,' ');

            for(auto& c : body)
                c = client.get();

            slog << debug << "HTTP request body \"" << body << "\"" << flush;

            if(version != "HTTP/1.1")
            {
                client << "HTTP/1.1 505 HTTP Version Not Supported" << crlf
                       << "Date: "         << date()                << crlf
                       << "Server: "       << host()                << crlf
                       << "Content-Type: " << content_type()        << crlf
                       << "Content-Length: 0"                       << crlf
                       << crlf << flush;
            }
            else if(not m_methods.contains(method))
            {
                client << "HTTP/1.1 400 Bad Request"                    << crlf
                       << "Date: "         << date()                    << crlf
                       << "Server: "       << host()                    << crlf
                       << "Content-Type: " << content_type()            << crlf
                       << "Content-Length: 0"                           << crlf
                       << "Access-Control-Allow-Methods: " << methods() << crlf
                       << crlf << flush;
            }
            else if(uri == "/favicon.ico")
            {
                client << "HTTP/1.1 404 Not Found"           << crlf
                       << "Date: "         << date()         << crlf
                       << "Server: "       << host()         << crlf
                       << "Content-Type: " << content_type() << crlf
                       << "Content-Length: 0"                << crlf
                       << crlf << flush;
            }
            else if(not std::regex_match(uri,std::regex(m_public_paths)) and not m_credentials.contains(authorization))
            {
                client << "HTTP/1.1 401 Unauthorized status"                     << crlf
                       << "Date: "         << date()                             << crlf
                       << "Server: "       << host()                             << crlf
                       << "Content-Type: " << content_type()                     << crlf
                       << "Content-Length: 0"                                    << crlf
                       << "WWW-Authenticate: Basic realm=\"User Visible Realm\"" << crlf
                       << crlf << flush;
            }
            else
            {
                auto content = ""s,  type = ""s;

                for(auto& [path,controller] : m_router)
                    if(std::regex_match(uri,std::regex(path)))
                        std::tie(content,type) = controller[method].render(uri,body);

                if(not type.empty())
                    client << "HTTP/1.1 200 OK"                                           << crlf
                           << "Date: " << date()                                          << crlf
                           << "Server: " << host()                                        << crlf
                           << "Content-Type: " << type                                    << crlf
                           << "Content-Length: " << content.length()                      << crlf
                           << "Access-Control-Allow-Methods: " << methods()               << crlf
                           << "Cache-Control: private"                                    << crlf
                           << "Access-Control-Allow-Origin: http://localhost:8080"        << crlf // FIXME
                           << "Access-Control-Allow-Headers: Content-Type, Authorization" << crlf
                           << "Access-Control-Allow-Credentials: true"                    << crlf
                           << crlf
                           << (method != "HEAD"s ? content : ""s) << flush;
                else
                    client << "HTTP/1.0 404 Not Found"                                    << crlf
                           << "Date: " << date()                                          << crlf
                           << "Server: " << host()                                        << crlf
                           << "Content-Type: " << content_type()                          << crlf
                           << "Content-Length: 0"                                         << crlf
                           << crlf << flush;
            }
        }
        slog << info << "connection closed" << flush;
    }
    catch(const std::exception& e)
    {
        client << "HTTP/1.0 500 Internal Server Error" << crlf
               << "Date: " << date()                   << crlf
               << "Server: " << host()                 << crlf
               << "Content-Type: " << content_type()   << crlf
               << "Content-Length: 0"                  << crlf
               << crlf << flush;

        net::slog << net::error << e.what() << net::flush;
    }

    using router = std::map<std::string,std::map<std::string,controller>, std::less<>>;

    router m_router = {};

    std::string m_content_type = "*/*"s;

    std::set<std::string> m_methods = {"HEAD"s, "OPTIONS"s};

    std::string m_public_paths = ".*";

    std::set<std::string> m_credentials = {};
};

} // namespace http
<commit_msg>Added back deleted this_thread::sleep_for<commit_after>#pragma once
#include <map>
#include <set>
#include <tuple>
#include <regex>
#include <string>
#include <thread>
#include "net/acceptor.hpp"
#include "http/headers.hpp"

namespace http {

using namespace std::literals;

class controller
{
public:

    using callback = std::function<std::string(std::string_view,std::string_view)>;

    void text(const std::string& content)
    {
        m_content_type = "text/plain"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void html(const std::string& content)
    {
        m_content_type = "text/html"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void css(const std::string& content)
    {
        m_content_type = "text/css"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void script(const std::string& content)
    {
        m_content_type = "application/javascript"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void json(const std::string& content)
    {
        m_content_type = "application/json"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void xml(const std::string& content)
    {
        m_content_type = "application/xml"s;
        m_callback = [&](std::string_view,std::string_view){return content;};
    }

    void response(std::string_view content_type, callback cb)
    {
        m_content_type = content_type;
        m_callback = cb;
    }

    std::tuple<std::string,std::string> render(std::string_view request = "", std::string_view body = "") const
    {
        return {m_callback(request,body),m_content_type};
    }

    void content_type(std::string_view type)
    {
        m_content_type = type;
    }

    const std::string& content_type() const
    {
        return m_content_type;
    }

private:

    std::string m_content_type = "*/*"s;

    callback m_callback = [](std::string_view, std::string_view){return "Not Found"s;};
};

class server
{
public:

    controller& get(const std::string& path)
    {
        m_methods.insert("GET"s);
        return m_router[path]["GET"s];
    }

    controller& head(const std::string& path)
    {
        m_methods.insert("HEAD"s);
        return m_router[path]["HEAD"s];
    }

    controller& post(const std::string& path)
    {
        m_methods.insert("POST"s);
        return m_router[path]["POST"s];
    }

    controller& put(const std::string& path)
    {
        m_methods.insert("PUT"s);
        return m_router[path]["PUT"s];
    }

    controller& patch(const std::string& path)
    {
        m_methods.insert("PATCH"s);
        return m_router[path]["PATCH"s];
    }

    controller& destroy(const std::string& path)
    {
        m_methods.insert("DELETE"s);
        return m_router[path]["DELETE"s];
    }

    void listen(std::string_view serice_or_port = "http")
    {
        auto endpoint = net::acceptor{"0.0.0.0", serice_or_port};
        endpoint.timeout(0s); // no timeout
        slog << notice << "Started up at " << endpoint.host() << ":" << endpoint.service_or_port() << flush;
        while(true)
        {
            auto client = endpoint.accept();
            slog << notice << "Accepted connection from " << std::get<1>(client) << ":" << std::get<2>(client) << flush;
            std::thread{[&]{handle(std::move(std::get<0>(client)));}}.detach();
            this_thread::sleep_for(1ms);
        }
    }

    void public_paths(std::string_view regex)
    {
        m_public_paths = regex;
    }

    void credentials(std::initializer_list<std::string> tokens)
    {
        m_credentials.insert(tokens);
    }

private:

    auto date() const
    {
        return ext::to_rfc1123(std::chrono::system_clock::now());
    }

    const auto& host() const
    {
        static auto s_host = syslog::gethostname();
        return s_host;
    }

    auto methods() const
    {
        auto tmp = ""s;
        for(const auto&  m : m_methods)
        {
            if(not tmp.empty()) tmp += ", "s;
            tmp += m;
        }
        return tmp;
    }

    const auto& content_type() const
    {
        return m_content_type;
    }

    void handle(endpointstream client)
    try
    {
        while(client)
        {
            auto method = ""s, uri = ""s, version = ""s;
            client >> method >> uri >> version;

            if(not client.good()) break;

            slog << info << "HTTP request \"" << method << ' ' << uri << ' ' << version << "\"" << flush;

            client >> std::ws;

            auto headers = http::headers{};
            client >> headers;
            // headers object will do the logging into syslog

            client.ignore(2); // crlf

            auto content_length = 0ull;
            if(headers.contains("content-length"))
                content_length = std::stoll(headers["content-length"]);

            auto authorization = ""s;
            if(headers.contains("authorization"))
                authorization = headers["authorization"];

            auto body = std::string(content_length,' ');

            for(auto& c : body)
                c = client.get();

            slog << debug << "HTTP request body \"" << body << "\"" << flush;

            if(version != "HTTP/1.1")
            {
                client << "HTTP/1.1 505 HTTP Version Not Supported" << crlf
                       << "Date: "         << date()                << crlf
                       << "Server: "       << host()                << crlf
                       << "Content-Type: " << content_type()        << crlf
                       << "Content-Length: 0"                       << crlf
                       << crlf << flush;
            }
            else if(not m_methods.contains(method))
            {
                client << "HTTP/1.1 400 Bad Request"                    << crlf
                       << "Date: "         << date()                    << crlf
                       << "Server: "       << host()                    << crlf
                       << "Content-Type: " << content_type()            << crlf
                       << "Content-Length: 0"                           << crlf
                       << "Access-Control-Allow-Methods: " << methods() << crlf
                       << crlf << flush;
            }
            else if(uri == "/favicon.ico")
            {
                client << "HTTP/1.1 404 Not Found"           << crlf
                       << "Date: "         << date()         << crlf
                       << "Server: "       << host()         << crlf
                       << "Content-Type: " << content_type() << crlf
                       << "Content-Length: 0"                << crlf
                       << crlf << flush;
            }
            else if(not std::regex_match(uri,std::regex(m_public_paths)) and not m_credentials.contains(authorization))
            {
                client << "HTTP/1.1 401 Unauthorized status"                     << crlf
                       << "Date: "         << date()                             << crlf
                       << "Server: "       << host()                             << crlf
                       << "Content-Type: " << content_type()                     << crlf
                       << "Content-Length: 0"                                    << crlf
                       << "WWW-Authenticate: Basic realm=\"User Visible Realm\"" << crlf
                       << crlf << flush;
            }
            else
            {
                auto content = ""s,  type = ""s;

                for(auto& [path,controller] : m_router)
                    if(std::regex_match(uri,std::regex(path)))
                        std::tie(content,type) = controller[method].render(uri,body);

                if(not type.empty())
                    client << "HTTP/1.1 200 OK"                                           << crlf
                           << "Date: " << date()                                          << crlf
                           << "Server: " << host()                                        << crlf
                           << "Content-Type: " << type                                    << crlf
                           << "Content-Length: " << content.length()                      << crlf
                           << "Access-Control-Allow-Methods: " << methods()               << crlf
                           << "Cache-Control: private"                                    << crlf
                           << "Access-Control-Allow-Origin: http://localhost:8080"        << crlf // FIXME
                           << "Access-Control-Allow-Headers: Content-Type, Authorization" << crlf
                           << "Access-Control-Allow-Credentials: true"                    << crlf
                           << crlf
                           << (method != "HEAD"s ? content : ""s) << flush;
                else
                    client << "HTTP/1.0 404 Not Found"                                    << crlf
                           << "Date: " << date()                                          << crlf
                           << "Server: " << host()                                        << crlf
                           << "Content-Type: " << content_type()                          << crlf
                           << "Content-Length: 0"                                         << crlf
                           << crlf << flush;
            }
        }
        slog << info << "connection closed" << flush;
    }
    catch(const std::exception& e)
    {
        client << "HTTP/1.0 500 Internal Server Error" << crlf
               << "Date: " << date()                   << crlf
               << "Server: " << host()                 << crlf
               << "Content-Type: " << content_type()   << crlf
               << "Content-Length: 0"                  << crlf
               << crlf << flush;

        net::slog << net::error << e.what() << net::flush;
    }

    using router = std::map<std::string,std::map<std::string,controller>, std::less<>>;

    router m_router = {};

    std::string m_content_type = "*/*"s;

    std::set<std::string> m_methods = {"HEAD"s, "OPTIONS"s};

    std::string m_public_paths = ".*";

    std::set<std::string> m_credentials = {};
};

} // namespace http
<|endoftext|>


<commit_before>#include <type_traits>
#include <loom/infer_grid.hpp>
#include <loom/hyper_kernel.hpp>

namespace loom
{

using distributions::sample_from_scores_overwrite;
using distributions::fast_log;
using distributions::fast_lgamma;

template<class GridPrior>
inline void HyperKernel::infer_outer_clustering_hypers (
        const GridPrior & grid_prior,
        rng_t & rng)
{
    if (grid_prior.size()) {
        std::vector<int> counts;
        counts.reserve(cross_cat_.kinds.size());
        for (const auto & kind : cross_cat_.kinds) {
            counts.push_back(kind.featureids.size());
        }
        cross_cat_.feature_clustering =
            sample_clustering_posterior(grid_prior, counts, rng);
    }
}

inline void HyperKernel::infer_inner_clustering_hypers (
        ProductModel & model,
        ProductMixture & mixture,
        const HyperPrior & hyper_prior,
        rng_t & rng)
{
    const auto & grid_prior = hyper_prior.clustering();
    if (grid_prior.size()) {
        const auto & counts = mixture.clustering.counts();
        model.clustering = sample_clustering_posterior(grid_prior, counts, rng);
        mixture.clustering.init(model.clustering);
    }
}

struct HyperKernel::infer_feature_hypers_fun
{
    const HyperPrior & hyper_prior;
    ProductMixture::Features & mixtures;
    rng_t & rng;

    template<class T>
    void operator() (T * t, size_t i, typename T::Shared & shared)
    {
        auto & mixture = mixtures[t][i];
        typedef typename std::remove_reference<decltype(mixture)>::type Mixture;
        InferShared<Mixture> infer_shared(shared, mixture, rng);
        const auto & grid_prior = protobuf::Fields<T>::get(hyper_prior);
        distributions::for_each_gridpoint(grid_prior, infer_shared);
        mixture.init(shared, rng);
    }

    void operator() (DPD * t, size_t i, DPD::Shared & shared);
};

void HyperKernel::infer_feature_hypers_fun::operator() (
        DPD * t,
        size_t i,
        DPD::Shared & shared)
{
    auto & mixture = mixtures[t][i];
    typedef typename std::remove_reference<decltype(mixture)>::type Mixture;
    InferShared<Mixture> infer_shared(shared, mixture, rng);
    const auto & grid_prior = protobuf::Fields<DPD>::get(hyper_prior);
    VectorFloat scores;

    // sample aux_counts
    typedef uint32_t count_t;
    std::unordered_map<DPD::Value, count_t> aux_counts;
    for (const auto & group : mixture.groups()) {
        for (const auto & i : group.counts) {
            auto value = i.first;
            auto count = i.second;
            float beta = shared.betas.get(value);
            LOOM_ASSERT_LT(0, beta);
            float log_prior = log(shared.alpha * beta);
            distributions::get_log_stirling1_row(count, scores);
            LOOM_ASSERT_EQ(scores.size(), count + 1);
            for (size_t k = 0; k <= count; ++k) {
                scores[k] += k * log_prior;
            }
            size_t aux_count = sample_from_scores_overwrite(rng, scores);
            LOOM_ASSERT_LT(0, aux_count);
            aux_counts[value] += aux_count;
        }
    }

    // only infer hypers if all values have been observed
    if (LOOM_LIKELY(aux_counts.size() == shared.betas.size())) {

        // grid gibbs gamma | aux_counts
        {
            size_t aux_total = 0;
            for (const auto & i : aux_counts) {
                aux_total += i.second;
            }
            scores.reserve(grid_prior.gamma_size());
            for (float gamma : grid_prior.gamma()) {
                float score = aux_counts.size() * fast_log(gamma)
                            + fast_lgamma(gamma)
                            - fast_lgamma(gamma + aux_total);
                scores.push_back(score);
            }
            size_t index = sample_from_scores_overwrite(rng, scores);
            shared.gamma = grid_prior.gamma(index);
        }

        // sample beta0, betas | aux_counts, gamma
        {
            std::vector<DPD::Value> values;
            std::vector<float> betas;
            values.reserve(aux_counts.size() + 1);
            betas.reserve(aux_counts.size() + 1);
            for (const auto & i : aux_counts) {
                values.push_back(i.first);
                betas.push_back(i.second);
            }
            values.push_back(shared.OTHER());
            betas.push_back(shared.gamma);

            distributions::sample_dirichlet(
                rng,
                betas.size(),
                betas.data(),
                betas.data());

            for (size_t i = 0, size = aux_counts.size(); i < size; ++i) {
                shared.betas.get(values[i]) = betas[i];
            }
            shared.beta0 = betas.back();
        }

        mixture.init(shared, rng);

        // grid gibbs alpha | beta0, betas, gamma
        {
            for (auto alpha : grid_prior.alpha()) {
                infer_shared.add().alpha = alpha;
            }
            infer_shared.done();
        }
    }

    mixture.init(shared, rng);
}

inline void HyperKernel::infer_feature_hypers (
        ProductModel & model,
        ProductMixture & mixture,
        const HyperPrior & hyper_prior,
        size_t featureid,
        rng_t & rng)
{
    infer_feature_hypers_fun fun = {hyper_prior, mixture.features, rng};
    for_one_feature(fun, model.features, featureid);
}

void HyperKernel::run (rng_t & rng)
{
    Timer::Scope timer(timer_);
    LOOM_ASSERT(run_, "hyper kernel should not be run");

    const size_t kind_count = cross_cat_.kinds.size();
    const size_t feature_count = cross_cat_.featureid_to_kindid.size();
    const auto & outer_prior = cross_cat_.hyper_prior.outer_prior();
    const auto & inner_prior = cross_cat_.hyper_prior.inner_prior();
    const size_t task_count = 1 + kind_count + feature_count;
    const auto seed = rng();

    #pragma omp parallel for if(parallel_) schedule(dynamic, 1)
    for (size_t taskid = 0; taskid < task_count; ++taskid) {
        rng_t rng(seed + taskid);
        if (taskid == 0) {

            infer_outer_clustering_hypers(outer_prior, rng);

        } else if (taskid < 1 + kind_count) {

            size_t kindid = taskid - 1;
            auto & kind = cross_cat_.kinds[kindid];
            infer_inner_clustering_hypers(
                kind.model,
                kind.mixture,
                inner_prior,
                rng);

        } else {

            size_t featureid = taskid - 1 - kind_count;
            size_t kindid = cross_cat_.featureid_to_kindid[featureid];
            auto & kind = cross_cat_.kinds[kindid];
            infer_feature_hypers(
                kind.model,
                kind.mixture,
                inner_prior,
                featureid,
                rng);
        }
    }
}

} // namespace loom

<commit_msg>Get DPD hyper parameter inference working<commit_after>#include <type_traits>
#include <loom/infer_grid.hpp>
#include <loom/hyper_kernel.hpp>

namespace loom
{

using distributions::sample_from_scores_overwrite;
using distributions::fast_log;
using distributions::fast_lgamma;

template<class GridPrior>
inline void HyperKernel::infer_outer_clustering_hypers (
        const GridPrior & grid_prior,
        rng_t & rng)
{
    if (grid_prior.size()) {
        std::vector<int> counts;
        counts.reserve(cross_cat_.kinds.size());
        for (const auto & kind : cross_cat_.kinds) {
            counts.push_back(kind.featureids.size());
        }
        cross_cat_.feature_clustering =
            sample_clustering_posterior(grid_prior, counts, rng);
    }
}

inline void HyperKernel::infer_inner_clustering_hypers (
        ProductModel & model,
        ProductMixture & mixture,
        const HyperPrior & hyper_prior,
        rng_t & rng)
{
    const auto & grid_prior = hyper_prior.clustering();
    if (grid_prior.size()) {
        const auto & counts = mixture.clustering.counts();
        model.clustering = sample_clustering_posterior(grid_prior, counts, rng);
        mixture.clustering.init(model.clustering);
    }
}

struct HyperKernel::infer_feature_hypers_fun
{
    const HyperPrior & hyper_prior;
    ProductMixture::Features & mixtures;
    rng_t & rng;

    template<class T>
    void operator() (T * t, size_t i, typename T::Shared & shared)
    {
        auto & mixture = mixtures[t][i];
        typedef typename std::remove_reference<decltype(mixture)>::type Mixture;
        InferShared<Mixture> infer_shared(shared, mixture, rng);
        const auto & grid_prior = protobuf::Fields<T>::get(hyper_prior);
        distributions::for_each_gridpoint(grid_prior, infer_shared);
        mixture.init(shared, rng);
    }

    void operator() (DPD * t, size_t i, DPD::Shared & shared);
};

void HyperKernel::infer_feature_hypers_fun::operator() (
        DPD * t,
        size_t i,
        DPD::Shared & shared)
{
    auto & mixture = mixtures[t][i];
    typedef typename std::remove_reference<decltype(mixture)>::type Mixture;
    InferShared<Mixture> infer_shared(shared, mixture, rng);
    const auto & grid_prior = protobuf::Fields<DPD>::get(hyper_prior);
    VectorFloat scores;

    // sample aux_counts
    typedef uint32_t count_t;
    std::unordered_map<DPD::Value, count_t> aux_counts;
    for (const auto & group : mixture.groups()) {
        for (const auto & i : group.counts) {
            auto value = i.first;
            auto count = i.second;
            float beta = shared.betas.get(value);
            LOOM_ASSERT_LT(0, beta);
            float log_prior = log(shared.alpha * beta);
            distributions::get_log_stirling1_row(count, scores);
            LOOM_ASSERT_EQ(scores.size(), count + 1);
            for (size_t k = 0; k <= count; ++k) {
                scores[k] += k * log_prior;
            }
            size_t aux_count = sample_from_scores_overwrite(rng, scores);
            LOOM_ASSERT_LT(0, aux_count);
            aux_counts[value] += aux_count;
        }
    }

    // only infer hypers if all values have been observed
    if (LOOM_LIKELY(aux_counts.size() == shared.betas.size())) {

        // grid gibbs gamma | aux_counts
        if (grid_prior.gamma_size()) {
            size_t aux_total = 0;
            for (const auto & i : aux_counts) {
                aux_total += i.second;
            }
            scores.clear();
            scores.reserve(grid_prior.gamma_size());
            for (float gamma : grid_prior.gamma()) {
                float score = aux_counts.size() * fast_log(gamma)
                            + fast_lgamma(gamma)
                            - fast_lgamma(gamma + aux_total);
                scores.push_back(score);
            }
            size_t index = sample_from_scores_overwrite(rng, scores);
            shared.gamma = grid_prior.gamma(index);
        }

        // sample beta0, betas | aux_counts, gamma
        if (grid_prior.alpha_size()) {
            std::vector<DPD::Value> values;
            std::vector<float> betas;
            values.reserve(aux_counts.size() + 1);
            betas.reserve(aux_counts.size() + 1);
            for (const auto & i : aux_counts) {
                values.push_back(i.first);
                betas.push_back(i.second);
            }
            values.push_back(shared.OTHER());
            betas.push_back(shared.gamma);

            distributions::sample_dirichlet(
                rng,
                betas.size(),
                betas.data(),
                betas.data());

            for (size_t i = 0, size = aux_counts.size(); i < size; ++i) {
                shared.betas.get(values[i]) = betas[i];
            }
            shared.beta0 = betas.back();
        }

        // grid gibbs alpha | beta0, betas, gamma
        if (grid_prior.alpha_size()) {
            mixture.init(shared, rng);
            for (auto alpha : grid_prior.alpha()) {
                infer_shared.add().alpha = alpha;
            }
            infer_shared.done();
        }
    }

    mixture.init(shared, rng);
}

inline void HyperKernel::infer_feature_hypers (
        ProductModel & model,
        ProductMixture & mixture,
        const HyperPrior & hyper_prior,
        size_t featureid,
        rng_t & rng)
{
    infer_feature_hypers_fun fun = {hyper_prior, mixture.features, rng};
    for_one_feature(fun, model.features, featureid);
}

void HyperKernel::run (rng_t & rng)
{
    Timer::Scope timer(timer_);
    LOOM_ASSERT(run_, "hyper kernel should not be run");

    const size_t kind_count = cross_cat_.kinds.size();
    const size_t feature_count = cross_cat_.featureid_to_kindid.size();
    const auto & outer_prior = cross_cat_.hyper_prior.outer_prior();
    const auto & inner_prior = cross_cat_.hyper_prior.inner_prior();
    const size_t task_count = 1 + kind_count + feature_count;
    const auto seed = rng();

    #pragma omp parallel for if(parallel_) schedule(dynamic, 1)
    for (size_t taskid = 0; taskid < task_count; ++taskid) {
        rng_t rng(seed + taskid);
        if (taskid == 0) {

            infer_outer_clustering_hypers(outer_prior, rng);

        } else if (taskid < 1 + kind_count) {

            size_t kindid = taskid - 1;
            auto & kind = cross_cat_.kinds[kindid];
            infer_inner_clustering_hypers(
                kind.model,
                kind.mixture,
                inner_prior,
                rng);

        } else {

            size_t featureid = taskid - 1 - kind_count;
            size_t kindid = cross_cat_.featureid_to_kindid[featureid];
            auto & kind = cross_cat_.kinds[kindid];
            infer_feature_hypers(
                kind.model,
                kind.mixture,
                inner_prior,
                featureid,
                rng);
        }
    }
}

} // namespace loom

<|endoftext|>


<commit_before>//
//  inpal_prime.cpp
//  Inverse Palindrome Library
//
//  Created by Bryan Triana on 7/21/16.
//  Copyright © 2016 Inverse Palindrome. All rights reserved.
//

#include "inpal_prime.hpp"
#include "inpal_algorithm.hpp"

std::vector<std::size_t> inpal::prime::prime_list(std::size_t range)
{
    const auto primes = prime_sieve(range);
    std::vector<std::size_t> p_list;

    for(std::size_t i=2; i<=range; i++) if(primes[i]) p_list.push_back(i);

    return p_list;
}

std::vector<bool> inpal::prime::prime_sieve(std::size_t range)
{
    std::vector<bool> p_test(range+1, false);

    //defines square root of range for future usage
    const std::size_t root = ceil(sqrt(range));

    //sieve axioms
    for(std::size_t x=1; x<=root; x++)
        for(std::size_t y=1; y<=root; y++)
        {
            std::size_t i = (4*x*x)+(y*y);
            if (i<=range && (i%12==1 || i%12==5))
            {
                p_test[i].flip();
            }

            i = (3*x*x)+(y*y);
            if(i<=range && i%12==7)
            {
                p_test[i].flip();
            }

            i = (3*x*x)-(y*y);
            if(x>y && i<=range && i%12==11)
            {
                p_test[i].flip();
            }
        }

    //marks 2,3,5 and 7 as prime numbers to deal with input smaller than 7
    p_test[2]=p_test[3]=p_test[5]=p_test[7]=true;

    //marks all multiples of primes as non primes
    for(std::size_t r=5; r<=root; r++)
    {
        if((p_test[r]))
        {
            for(std::size_t j=r*r; j<=range; j+=r*r)
            {
                p_test[j]=false;
            }
        }
    }

    return p_test;
}

std::vector<std::size_t> inpal::prime::factor_list(std::size_t num)
{
    std::vector<std::size_t> factors;
    std::vector<std::size_t> primes;

    std::size_t factor = algorithm::pollard_rho(num);
    factors.push_back(num/factor);
    factors.push_back(factor);

    do
    {
        std::size_t m = factors.back();
        factors.pop_back();

        if(m==1) continue;
        if(prime_test(m))
        {
            primes.push_back(m);

            //decomposes the factors into primes
            for(std::size_t i=0; i<factors.size(); i++)
            {
                std::size_t k = factors[i];
                if(k%m==0)
                {
                    do k/=m;
                    while(k%m==0);

                    factors[i]=k;
                }
            }
        }
        else
        {
            factor=algorithm::pollard_rho(m);
            factors.push_back(m/factor);
            factors.push_back(factor);
        }
    }
    while(!factors.empty());

    std::sort(primes.begin(), primes.end());

    return primes;
}

std::size_t inpal::prime::prime_locate(std::size_t pos)
{
    //index starts at 1 instead of 0, eg 1st prime is 2
    pos = pos-1;

    //return values for input less or equal to 10
    const auto small_primes = prime_list(29);
    if(pos<10) return small_primes[pos];

    //denotes the limit of the sieve
    std::size_t limit = pos*log(pos)+pos*log(log(pos));
    const auto primes = prime_list(limit);

    return primes[pos];
}

std::size_t inpal::prime::max_prime(std::size_t num)
{
    const auto primes = prime_sieve(num);
    auto it = std::find(primes.rbegin(), primes.rend(), true);

    return primes.size()-std::distance(primes.rbegin(), it)-1;
}

std::size_t inpal::prime::prime_count(std::size_t range)
{
    const auto primes = prime_sieve(range);

    return std::count(primes.begin(), primes.end(), true);
}

double inpal::prime::prime_density(double range)
{
    return prime_count(range)/range;
}

bool inpal::prime::prime_test(std::size_t num)
{
    if(num!=2 && num%2==0) return false;

    const std::size_t iteration = 20;
    std::size_t s = num-1;
    while(s%2==0) s/=2;

    for(std::size_t i=0; i<iteration; i++)
    {
        std::size_t a = rand()%(num-1)+1;
        std::size_t b = s;
        std::size_t mod = algorithm::modulo(a,b,num);

        while(b!=num-1 && mod!=1 && mod!=num-1)
        {
            mod=algorithm::mulmod(mod,mod,num);
            b*=2;
        }
        if(mod!=num-1 && b%2==0) return false;
    }

    return true;
}

bool inpal::prime::twin_test(std::size_t num)
{
    return prime_test(num) && (prime_test(num-2) || prime_test(num+2));
}

bool inpal::prime::cousin_test(std::size_t num)
{
    return  prime_test(num) && (prime_test(num-4) || prime_test(num+4));
}

bool inpal::prime::sexy_test(std::size_t num)
{
    return num!=3 && prime_test(num) && (prime_test(num-6) || prime_test(num+6));
}

std::size_t inpal::prime::max_palprime(std::size_t num)
{
    const auto primes = prime_sieve(num);

    for(std::size_t i=num; i>=2; --i) if(primes[i] && algorithm::pal_test(i)) return i;

    return 2;
}

std::size_t inpal::prime::max_factor(std::size_t num)
{
    return factor_list(num).back();
}

std::size_t inpal::prime::factor_count(std::size_t num)
{
    return factor_list(num).size();
}

<commit_msg>max_prime now works with prime_test<commit_after>//
//  inpal_prime.cpp
//  Inverse Palindrome Library
//
//  Created by Bryan Triana on 7/21/16.
//  Copyright © 2016 Inverse Palindrome. All rights reserved.
//

#include "inpal_prime.hpp"
#include "inpal_algorithm.hpp"

std::vector<std::size_t> inpal::prime::prime_list(std::size_t range)
{
    const auto primes = prime_sieve(range);
    std::vector<std::size_t> p_list;

    for(std::size_t i=2; i<=range; i++) if(primes[i]) p_list.push_back(i);

    return p_list;
}

std::vector<bool> inpal::prime::prime_sieve(std::size_t range)
{
    std::vector<bool> p_test(range+1, false);

    //defines square root of range for future usage
    const std::size_t root = ceil(sqrt(range));

    //sieve axioms
    for(std::size_t x=1; x<=root; x++)
        for(std::size_t y=1; y<=root; y++)
        {
            std::size_t i = (4*x*x)+(y*y);
            if (i<=range && (i%12==1 || i%12==5))
            {
                p_test[i].flip();
            }

            i = (3*x*x)+(y*y);
            if(i<=range && i%12==7)
            {
                p_test[i].flip();
            }

            i = (3*x*x)-(y*y);
            if(x>y && i<=range && i%12==11)
            {
                p_test[i].flip();
            }
        }

    //marks 2,3,5 and 7 as prime numbers to deal with input smaller than 7
    p_test[2]=p_test[3]=p_test[5]=p_test[7]=true;

    //marks all multiples of primes as non primes
    for(std::size_t r=5; r<=root; r++)
    {
        if((p_test[r]))
        {
            for(std::size_t j=r*r; j<=range; j+=r*r)
            {
                p_test[j]=false;
            }
        }
    }

    return p_test;
}

std::vector<std::size_t> inpal::prime::factor_list(std::size_t num)
{
    std::vector<std::size_t> factors;
    std::vector<std::size_t> primes;

    std::size_t factor = algorithm::pollard_rho(num);
    factors.push_back(num/factor);
    factors.push_back(factor);

    do
    {
        std::size_t m = factors.back();
        factors.pop_back();

        if(m==1) continue;
        if(prime_test(m))
        {
            primes.push_back(m);

            //decomposes the factors into primes
            for(std::size_t i=0; i<factors.size(); i++)
            {
                std::size_t k = factors[i];
                if(k%m==0)
                {
                    do k/=m;
                    while(k%m==0);

                    factors[i]=k;
                }
            }
        }
        else
        {
            factor=algorithm::pollard_rho(m);
            factors.push_back(m/factor);
            factors.push_back(factor);
        }
    }
    while(factors.size());

    std::sort(primes.begin(), primes.end());

    return primes;
}

std::size_t inpal::prime::prime_locate(std::size_t pos)
{
    //index starts at 1 instead of 0, eg 1st prime is 2
    pos = pos-1;

    //return values for input less or equal to 10
    const auto small_primes = prime_list(29);
    if(pos<10) return small_primes[pos];

    //denotes the limit of the sieve
    std::size_t limit = pos*log(pos)+pos*log(log(pos));
    const auto primes = prime_list(limit);

    return primes[pos];
}

std::size_t inpal::prime::max_prime(std::size_t range)
{
    while(range)
    {
        if(prime_test(range)) return range;
        range = range%2==0 ? range-1 : range-2;
    }

    return 2;
}

std::size_t inpal::prime::prime_count(std::size_t range)
{
    const auto primes = prime_sieve(range);

    return std::count(primes.begin(), primes.end(), true);
}

double inpal::prime::prime_density(double range)
{
    return prime_count(range)/range;
}

bool inpal::prime::prime_test(std::size_t num)
{
    if(num!=2 && num%2==0) return false;

    const std::size_t iteration = 20;
    std::size_t s = num-1;
    while(s%2==0) s/=2;

    for(std::size_t i=0; i<iteration; i++)
    {
        std::size_t a = rand()%(num-1)+1;
        std::size_t b = s;
        std::size_t mod = algorithm::modulo(a,b,num);

        while(b!=num-1 && mod!=1 && mod!=num-1)
        {
            mod=algorithm::mulmod(mod,mod,num);
            b*=2;
        }
        if(mod!=num-1 && b%2==0) return false;
    }

    return true;
}

bool inpal::prime::twin_test(std::size_t num)
{
    return prime_test(num) && (prime_test(num-2) || prime_test(num+2));
}

bool inpal::prime::cousin_test(std::size_t num)
{
    return  prime_test(num) && (prime_test(num-4) || prime_test(num+4));
}

bool inpal::prime::sexy_test(std::size_t num)
{
    return num!=3 && prime_test(num) && (prime_test(num-6) || prime_test(num+6));
}

std::size_t inpal::prime::max_palprime(std::size_t num)
{
    const auto primes = prime_sieve(num);

    for(std::size_t i=num; i>=2; --i) if(primes[i] && algorithm::pal_test(i)) return i;

    return 2;
}

std::size_t inpal::prime::max_factor(std::size_t num)
{
    return factor_list(num).back();
}

std::size_t inpal::prime::factor_count(std::size_t num)
{
    return factor_list(num).size();
}

<|endoftext|>


<commit_before>#ifndef __JS_JSON_STREAM_HPP_20140928__
#define __JS_JSON_STREAM_HPP_20140928__

#include <string>
#include "jsonlite.hpp"
#include <sstream>

namespace jslite {

class JsonTokenzier;

class JsonStream {
public:
	JsonStream();
	~JsonStream();

	//in operation
	int Print(const Json& json);

	void set_obj_sep(const std::string& sep);
	void set_array_sep(const std::string& sep);
	void set_indent_sep(const std::string& sep);
	void set_comma_sep(const std::string& sep);
	void set_colon_sep(const std::string& sep);

	//out operating
	int Parse(Json& json);

	//others
	std::string str() const;
	void str(const std::string& s);
	JsonStream& operator << (const std::string& s);
	JsonStream& operator << (const char *s);

protected:
	//in operation
	int32_t PrintValue(const Json& json);
	int32_t PrintObject(const Json& json);
	int32_t PrintArray(const Json& json);
	int32_t PrintString(const Json& json);

	void FormattingBegin(const std::string& sep);
	void FormattingIndent();
	void FormattingComma();
	void FormattingEnd(const std::string& sep);

	//out operating
	int32_t ParseValue(Json &json, size_t depth = 0);
	int32_t ParseArray(Json &json, size_t depth);
	int32_t ParseObject(Json &json, size_t depth);
	void TrimSpace();

private:
	JsonTokenzier *tokenizer_;
	std::stringstream ss_;

	std::string obj_sep_;
	std::string array_sep_;
	std::string indent_sep_;
	std::string comma_sep_;
	std::string colon_sep_;
	uint32_t    indent_;
};

struct JOpt {
	JOpt(void (*func)(JsonStream&, const std::string&), const std::string& val) : func_(func), val_(val) {}
	void (*func_)(JsonStream&, const std::string&);
	std::string val_;
};

JOpt obj_sep(const std::string& val);
JOpt array_sep(const std::string& val);
JOpt indent_sep(const std::string& val);
JOpt comma_sep(const std::string& val);
JsonStream& default_sep(JsonStream& jstm);

//////////////////////////////////////////////////////////////
// addtional operators
JsonStream& operator << (JsonStream& jstm, const Json& json);
JsonStream& operator << (JsonStream& jstm, JsonStream& (__cdecl *func)(JsonStream&));
JsonStream& operator << (JsonStream& jstm, const JOpt& opt);
JsonStream& operator >> (JsonStream& jstm, Json& json);

std::ostream& operator << (std::ostream& os, const Json& json);

} //namespace jslite

#endif //__JS_JSON_STREAM_HPP_20140928__<commit_msg>* JsonStream class are including JsonParser<commit_after>#ifndef __JS_JSON_STREAM_HPP_20140928__
#define __JS_JSON_STREAM_HPP_20140928__

#include <string>
#include <sstream>

#include "jsonlite.hpp"

namespace jslite {

class JsonTokenzier;

class JsonStream {
public:
	JsonStream();
	~JsonStream();

	//in operation
	int Print(const Json& json);

	void set_obj_sep(const std::string& sep);
	void set_array_sep(const std::string& sep);
	void set_indent_sep(const std::string& sep);
	void set_comma_sep(const std::string& sep);
	void set_colon_sep(const std::string& sep);

	//out operating
	int Parse(Json& json);

	//others
	std::string str() const;
	void str(const std::string& s);
	JsonStream& operator << (const std::string& s);
	JsonStream& operator << (const char *s);

protected:
	//in operation
	int32_t PrintValue(const Json& json);
	int32_t PrintObject(const Json& json);
	int32_t PrintArray(const Json& json);
	int32_t PrintString(const Json& json);

	void FormattingBegin(const std::string& sep);
	void FormattingIndent();
	void FormattingComma();
	void FormattingEnd(const std::string& sep);

	//out operating
	int32_t ParseValue(Json &json, size_t depth = 0);
	int32_t ParseArray(Json &json, size_t depth);
	int32_t ParseObject(Json &json, size_t depth);
	void TrimSpace();

private:
	JsonTokenzier *tokenizer_;
	std::stringstream ss_;

	std::string obj_sep_;
	std::string array_sep_;
	std::string indent_sep_;
	std::string comma_sep_;
	std::string colon_sep_;
	uint32_t    indent_;
};

struct JOpt {
	JOpt(void (*func)(JsonStream&, const std::string&), const std::string& val) : func_(func), val_(val) {}
	void (*func_)(JsonStream&, const std::string&);
	std::string val_;
};

JOpt obj_sep(const std::string& val);
JOpt array_sep(const std::string& val);
JOpt indent_sep(const std::string& val);
JOpt comma_sep(const std::string& val);
JsonStream& default_sep(JsonStream& jstm);

//////////////////////////////////////////////////////////////
// addtional operators
JsonStream& operator << (JsonStream& jstm, const Json& json);
JsonStream& operator << (JsonStream& jstm, JsonStream& (__cdecl *func)(JsonStream&));
JsonStream& operator << (JsonStream& jstm, const JOpt& opt);
JsonStream& operator >> (JsonStream& jstm, Json& json);

std::ostream& operator << (std::ostream& os, const Json& json);

} //namespace jslite

#endif //__JS_JSON_STREAM_HPP_20140928__<|endoftext|>


<commit_before>/*
**  Copyright (C) 2013 Aldebaran Robotics
**  See COPYING for the license
*/

#include <qitype/jsoncodec.hpp>
#include <qitype/anyvalue.hpp>

#include "jsoncodec_p.hpp"

namespace qi {

  const std::string  JsonDecoderPrivate::_strictJsonTokenSet = "[]{},:\"";
  const std::string  JsonDecoderPrivate::_permissiveJsonTokenSet = "[]{},:";

  JsonDecoderPrivate::JsonDecoderPrivate(const std::string &in)
    :_begin(in.begin()),
      _end(in.end()),
      _it(_begin),
      _rule(JSONRule_Strict)
  {}

  JsonDecoderPrivate::JsonDecoderPrivate(const std::string::const_iterator &begin,
                    const std::string::const_iterator &end)
    :_begin(begin),
      _end(end),
      _it(_begin),
      _rule(JSONRule_Strict)
  {}

  std::string::const_iterator JsonDecoderPrivate::decode(AnyValue &out, JSONRule rule)
  {
    _rule = rule;
    _it = _begin;
    if (!decodeValue(out))
      throw std::runtime_error("parse error");
    return _it;
  }

  void JsonDecoderPrivate::skipWhiteSpaces()
  {
    while (_it != _end && *_it == ' ')
      ++_it;
  }

  bool JsonDecoderPrivate::getDigits(std::string &result)
  {
    std::string::const_iterator begin = _it;

    while (_it != _end && *_it >= '0' && *_it <= '9')
      ++_it;
    if (_it == begin)
        return false;
    result = std::string(begin, _it);
    return true;
  }

  bool JsonDecoderPrivate::getInteger(std::string &result)
  {
    std::string::const_iterator save = _it;
    std::string integerStr;

    if (_it == _end)
      return false;
    if (*_it == '-')
    {
      ++_it;
      integerStr = "-";
    }
    std::string digitsStr;

    if (!getDigits(digitsStr))
    {
      _it = save;
      return false;
    }
    integerStr += digitsStr;

    result = integerStr;
    return true;
  }

  bool JsonDecoderPrivate::getInteger(qi::int64_t &result)
  {
    std::string integerStr;

    if (!getInteger(integerStr))
      return false;
    result = ::atol(integerStr.c_str());
    return true;
  }

  bool JsonDecoderPrivate::getExponent(std::string &result)
  {
    std::string::const_iterator save = _it;

    if (_it == _end || (*_it != 'e' && *_it != 'E'))
      return false;
    ++_it;
    std::string exponentStr;

    exponentStr += 'e';
    if (*_it == '+' || *_it == '-')
    {
      exponentStr += *_it;
      ++_it;
    }
    else
      exponentStr += '+';
    std::string integerStr;
    if (!getDigits(integerStr))
    {
      _it = save;
      return false;
    }

    result = exponentStr + integerStr;
    return true;
  }

  bool JsonDecoderPrivate::getFloat(double &result)
  {
    std::string floatStr;
    std::string beforePoint;
    std::string afterPoint;
    std::string exponent;
    std::string::const_iterator save = _it;

    if (!getInteger(beforePoint))
      return false;
    if (!getExponent(exponent))
    {
      if (*_it != '.')
      {
        _it = save;
        return false;
      }
      ++_it;
      if (!getDigits(afterPoint))
      {
        _it = save;
        return false;
      }
      getExponent(exponent);
      floatStr = beforePoint + "." + afterPoint + exponent;
    }
    else
      floatStr = beforePoint + exponent;

    result = ::atof(floatStr.c_str());
    return true;
  }

  bool JsonDecoderPrivate::decodeArray(AnyValue &value)
  {
    std::string::const_iterator save = _it;

    if (_it == _end || *_it != '[')
      return false;
    ++_it;
    std::vector<AnyValue>   tmpArray;

    while (true)
    {
      AnyValue subElement;

      if (!decodeValue(subElement))
        break;
      tmpArray.push_back(subElement);
      if (*_it != ',')
        break;
      ++_it;
    }
    if (*_it != ']')
    {
      _it = save;
      return false;
    }
    ++_it;
    value = AnyValue(tmpArray);
    return true;
  }

  bool JsonDecoderPrivate::decodeFloat(AnyValue &value)
  {
    double tmpFloat;

    if (!getFloat(tmpFloat))
      return false;
    value = AnyValue(tmpFloat);
    return true;
  }

  bool JsonDecoderPrivate::decodeInteger(AnyValue &value)
  {
    qi::int64_t tmpInteger;

    if (!getInteger(tmpInteger))
      return false;
    value = AnyValue(tmpInteger);
    return true;
  }

  bool JsonDecoderPrivate::getCleanString(std::string &result)
  {
    std::string::const_iterator save = _it;

    if (_it == _end)
      return false;
    if (!isBeginningOfJsonString(_it))
      return false;
    std::string tmpString;

    if (_rule == JSONRule_Strict)
      ++_it;
    while (_it != _end && !isEndOfJsonString(_it))
    {
      if (*_it == '\\')
      {
        if (_it + 1 == _end)
        {
          _it = save;
          return false;
        }
        switch (*(_it + 1))
        {
        case '"' :tmpString += '"'; break;
        case '\\':tmpString += '\\';break;
        case '/' :tmpString += '/'; break;
        case 'b' :tmpString += '\b';break;
        case 'f' :tmpString += '\f';break;
        case 'n' :tmpString += '\n';break;
        case 'r' :tmpString += '\r';break;
        case 't' :tmpString += '\t';break;
        default:
          _it = save;
          return false;
        }
        _it += 2;
      }
      else
      {
        tmpString += *_it;
        ++_it;
      }
    }
    if (_rule == JSONRule_Strict && _it == _end)
    {
      _it = save;
      return false;
    }
    if (_rule == JSONRule_Strict)
      ++_it;
    result = tmpString;
    return true;
  }

  bool JsonDecoderPrivate::decodeString(AnyValue &value)
  {
    std::string tmpString;

    if (!getCleanString(tmpString))
      return false;
    value = AnyValue(tmpString);
    return true;
  }

  bool JsonDecoderPrivate::decodeObject(AnyValue &value)
  {
    std::string::const_iterator save = _it;

    if (_it == _end || *_it != '{')
      return false;
    ++_it;

    std::map<std::string, AnyValue> tmpMap;
    while (true)
    {
      skipWhiteSpaces();
      std::string key;

      if (!getCleanString(key))
        break;
      skipWhiteSpaces();
      if (_it == _end || *_it != ':')
      {
        _it = save;
        return false;
      }
      ++_it;
      AnyValue tmpValue;
      if (!decodeValue(tmpValue))
      {
        _it = save;
        return false;
      }
      if (_it == _end)
        break;
      tmpMap[key] = tmpValue;
      if (*_it != ',')
        break;
      ++_it;
    }
    if (_it == _end || *_it != '}')
    {
      _it = save;
      return false;
    }
    ++_it;
    value = AnyValue(tmpMap);
    return true;
  }

  bool JsonDecoderPrivate::match(std::string const& expected)
  {
    std::string::const_iterator save = _it;
    std::string::const_iterator begin = expected.begin();

    while (_it != _end && begin != expected.end())
    {
      if (*_it != *begin)
      {
        _it = save;
        return false;
      }
      ++_it;
      ++begin;
    }
    if (begin != expected.end())
      return false;
    return true;
  }

  bool JsonDecoderPrivate::decodeSpecial(AnyValue &value)
  {
    if (_it == _end)
      return false;
    if (match("true"))
      value = AnyValue(true);
    else if (match("false"))
      value = AnyValue::from(false);
    else if (match("null"))
      value = AnyValue();
    else
      return false;
    return true;
  }

  bool JsonDecoderPrivate::decodeValue(AnyValue &value)
  {
    skipWhiteSpaces();
    if (decodeSpecial(value)
        || decodeFloat(value)
        || decodeInteger(value)
        || decodeArray(value)
        || decodeObject(value)
        || decodeString(value)
        )
    {
      skipWhiteSpaces();
      return true;
    }
    return false;
  }

  bool JsonDecoderPrivate::isJsonToken(const std::string::const_iterator &it) const
  {
    if (_rule == JSONRule_Strict && _strictJsonTokenSet.find(*it) != std::string::npos)
        return true;
    if (_rule == JSONRule_Permissive && _permissiveJsonTokenSet.find(*it) != std::string::npos)
        return true;
    return false;
  }

  bool JsonDecoderPrivate::isEndOfJsonString(const std::string::const_iterator &it) const
  {
    if (_rule == JSONRule_Strict)
      return *it == '"';
    else
     return *it == ' ' || isJsonToken(it);
  }

  bool JsonDecoderPrivate::isBeginningOfJsonString(const std::string::const_iterator &it) const
  {
    if (_rule == JSONRule_Strict)
      return *it == '"';
    else
      return *it != ' ' && !isJsonToken(it);
  }

  std::string::const_iterator decodeJSON(const std::string::const_iterator &begin,
                                         const std::string::const_iterator &end,
                                         AnyValue &target, JSONRule rule)
  {
    JsonDecoderPrivate parser(begin, end);
    return parser.decode(target);
  }

  AnyValue decodeJSON(const std::string &in, JSONRule rule)
  {
    AnyValue value;
    JsonDecoderPrivate parser(in);

    parser.decode(value, rule);
    return value;
  }

}
<commit_msg>decodeJSON: fix bug, add test<commit_after>/*
**  Copyright (C) 2013 Aldebaran Robotics
**  See COPYING for the license
*/

#include <qitype/jsoncodec.hpp>
#include <qitype/anyvalue.hpp>

#include "jsoncodec_p.hpp"

namespace qi {

  const std::string  JsonDecoderPrivate::_strictJsonTokenSet = "[]{},:\"";
  const std::string  JsonDecoderPrivate::_permissiveJsonTokenSet = "[]{},:";

  JsonDecoderPrivate::JsonDecoderPrivate(const std::string &in)
    :_begin(in.begin()),
      _end(in.end()),
      _it(_begin),
      _rule(JSONRule_Strict)
  {}

  JsonDecoderPrivate::JsonDecoderPrivate(const std::string::const_iterator &begin,
                    const std::string::const_iterator &end)
    :_begin(begin),
      _end(end),
      _it(_begin),
      _rule(JSONRule_Strict)
  {}

  std::string::const_iterator JsonDecoderPrivate::decode(AnyValue &out, JSONRule rule)
  {
    _rule = rule;
    _it = _begin;
    if (!decodeValue(out))
      throw std::runtime_error("parse error");
    return _it;
  }

  void JsonDecoderPrivate::skipWhiteSpaces()
  {
    while (_it != _end && *_it == ' ')
      ++_it;
  }

  bool JsonDecoderPrivate::getDigits(std::string &result)
  {
    std::string::const_iterator begin = _it;

    while (_it != _end && *_it >= '0' && *_it <= '9')
      ++_it;
    if (_it == begin)
        return false;
    result = std::string(begin, _it);
    return true;
  }

  bool JsonDecoderPrivate::getInteger(std::string &result)
  {
    std::string::const_iterator save = _it;
    std::string integerStr;

    if (_it == _end)
      return false;
    if (*_it == '-')
    {
      ++_it;
      integerStr = "-";
    }
    std::string digitsStr;

    if (!getDigits(digitsStr))
    {
      _it = save;
      return false;
    }
    integerStr += digitsStr;

    result = integerStr;
    return true;
  }

  bool JsonDecoderPrivate::getInteger(qi::int64_t &result)
  {
    std::string integerStr;

    if (!getInteger(integerStr))
      return false;
    result = ::atol(integerStr.c_str());
    return true;
  }

  bool JsonDecoderPrivate::getExponent(std::string &result)
  {
    std::string::const_iterator save = _it;

    if (_it == _end || (*_it != 'e' && *_it != 'E'))
      return false;
    ++_it;
    std::string exponentStr;

    exponentStr += 'e';
    if (*_it == '+' || *_it == '-')
    {
      exponentStr += *_it;
      ++_it;
    }
    else
      exponentStr += '+';
    std::string integerStr;
    if (!getDigits(integerStr))
    {
      _it = save;
      return false;
    }

    result = exponentStr + integerStr;
    return true;
  }

  bool JsonDecoderPrivate::getFloat(double &result)
  {
    std::string floatStr;
    std::string beforePoint;
    std::string afterPoint;
    std::string exponent;
    std::string::const_iterator save = _it;

    if (!getInteger(beforePoint))
      return false;
    if (!getExponent(exponent))
    {
      if (*_it != '.')
      {
        _it = save;
        return false;
      }
      ++_it;
      if (!getDigits(afterPoint))
      {
        _it = save;
        return false;
      }
      getExponent(exponent);
      floatStr = beforePoint + "." + afterPoint + exponent;
    }
    else
      floatStr = beforePoint + exponent;

    result = ::atof(floatStr.c_str());
    return true;
  }

  bool JsonDecoderPrivate::decodeArray(AnyValue &value)
  {
    std::string::const_iterator save = _it;

    if (_it == _end || *_it != '[')
      return false;
    ++_it;
    std::vector<AnyValue>   tmpArray;

    while (true)
    {
      AnyValue subElement;

      if (!decodeValue(subElement))
        break;
      tmpArray.push_back(subElement);
      if (*_it != ',')
        break;
      ++_it;
    }
    if (*_it != ']')
    {
      _it = save;
      return false;
    }
    ++_it;
    value = AnyValue(tmpArray);
    return true;
  }

  bool JsonDecoderPrivate::decodeFloat(AnyValue &value)
  {
    double tmpFloat;

    if (!getFloat(tmpFloat))
      return false;
    value = AnyValue(tmpFloat);
    return true;
  }

  bool JsonDecoderPrivate::decodeInteger(AnyValue &value)
  {
    qi::int64_t tmpInteger;

    if (!getInteger(tmpInteger))
      return false;
    value = AnyValue(tmpInteger);
    return true;
  }

  bool JsonDecoderPrivate::getCleanString(std::string &result)
  {
    std::string::const_iterator save = _it;

    if (_it == _end)
      return false;
    if (!isBeginningOfJsonString(_it))
      return false;
    std::string tmpString;

    if (_rule == JSONRule_Strict)
      ++_it;
    while (_it != _end && !isEndOfJsonString(_it))
    {
      if (*_it == '\\')
      {
        if (_it + 1 == _end)
        {
          _it = save;
          return false;
        }
        switch (*(_it + 1))
        {
        case '"' :tmpString += '"'; break;
        case '\\':tmpString += '\\';break;
        case '/' :tmpString += '/'; break;
        case 'b' :tmpString += '\b';break;
        case 'f' :tmpString += '\f';break;
        case 'n' :tmpString += '\n';break;
        case 'r' :tmpString += '\r';break;
        case 't' :tmpString += '\t';break;
        default:
          _it = save;
          return false;
        }
        _it += 2;
      }
      else
      {
        tmpString += *_it;
        ++_it;
      }
    }
    if (_rule == JSONRule_Strict && _it == _end)
    {
      _it = save;
      return false;
    }
    if (_rule == JSONRule_Strict)
      ++_it;
    result = tmpString;
    return true;
  }

  bool JsonDecoderPrivate::decodeString(AnyValue &value)
  {
    std::string tmpString;

    if (!getCleanString(tmpString))
      return false;
    value = AnyValue(tmpString);
    return true;
  }

  bool JsonDecoderPrivate::decodeObject(AnyValue &value)
  {
    std::string::const_iterator save = _it;

    if (_it == _end || *_it != '{')
      return false;
    ++_it;

    std::map<std::string, AnyValue> tmpMap;
    while (true)
    {
      skipWhiteSpaces();
      std::string key;

      if (!getCleanString(key))
        break;
      skipWhiteSpaces();
      if (_it == _end || *_it != ':')
      {
        _it = save;
        return false;
      }
      ++_it;
      AnyValue tmpValue;
      if (!decodeValue(tmpValue))
      {
        _it = save;
        return false;
      }
      if (_it == _end)
        break;
      tmpMap[key] = tmpValue;
      if (*_it != ',')
        break;
      ++_it;
    }
    if (_it == _end || *_it != '}')
    {
      _it = save;
      return false;
    }
    ++_it;
    value = AnyValue(tmpMap);
    return true;
  }

  bool JsonDecoderPrivate::match(std::string const& expected)
  {
    std::string::const_iterator save = _it;
    std::string::const_iterator begin = expected.begin();

    while (_it != _end && begin != expected.end())
    {
      if (*_it != *begin)
      {
        _it = save;
        return false;
      }
      ++_it;
      ++begin;
    }
    if (begin != expected.end())
    {
      _it = save;
      return false;
    }
    return true;
  }

  bool JsonDecoderPrivate::decodeSpecial(AnyValue &value)
  {
    if (_it == _end)
      return false;
    if (match("true"))
      value = AnyValue(true);
    else if (match("false"))
      value = AnyValue::from(false);
    else if (match("null"))
      value = AnyValue();
    else
      return false;
    return true;
  }

  bool JsonDecoderPrivate::decodeValue(AnyValue &value)
  {
    skipWhiteSpaces();
    if (decodeSpecial(value)
        || decodeFloat(value)
        || decodeInteger(value)
        || decodeArray(value)
        || decodeObject(value)
        || decodeString(value)
        )
    {
      skipWhiteSpaces();
      return true;
    }
    return false;
  }

  bool JsonDecoderPrivate::isJsonToken(const std::string::const_iterator &it) const
  {
    if (_rule == JSONRule_Strict && _strictJsonTokenSet.find(*it) != std::string::npos)
        return true;
    if (_rule == JSONRule_Permissive && _permissiveJsonTokenSet.find(*it) != std::string::npos)
        return true;
    return false;
  }

  bool JsonDecoderPrivate::isEndOfJsonString(const std::string::const_iterator &it) const
  {
    if (_rule == JSONRule_Strict)
      return *it == '"';
    else
     return *it == ' ' || isJsonToken(it);
  }

  bool JsonDecoderPrivate::isBeginningOfJsonString(const std::string::const_iterator &it) const
  {
    if (_rule == JSONRule_Strict)
      return *it == '"';
    else
      return *it != ' ' && !isJsonToken(it);
  }

  std::string::const_iterator decodeJSON(const std::string::const_iterator &begin,
                                         const std::string::const_iterator &end,
                                         AnyValue &target, JSONRule rule)
  {
    JsonDecoderPrivate parser(begin, end);
    return parser.decode(target);
  }

  AnyValue decodeJSON(const std::string &in, JSONRule rule)
  {
    AnyValue value;
    JsonDecoderPrivate parser(in);

    parser.decode(value, rule);
    return value;
  }

}
<|endoftext|>