<commit_before>// Author: Jakob Blomer CERN  07/2018

/*************************************************************************
 * Copyright (C) 1995-2017, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

#ifndef ROOT_RSQLITEDS
#define ROOT_RSQLITEDS

#include "ROOT/RDataFrame.hxx"
#include "ROOT/RDataSource.hxx"

#include <map>
#include <memory>
#include <mutex>

#include <sqlite3.h>

namespace ROOT {

namespace RDF {

class RSqliteDS final : public ROOT::RDF::RDataSource {
private:
   enum class Types {
     kInteger,
     kReal,
     kText,
     kBlob,
     kNull
   };

   struct Value_t {
     Value_t() : fType(Types::kNull), fNull(nullptr), fPtr(nullptr), fIsActive(false) {}

     Types fType;
     Long64_t fInteger;
     double fReal;
     std::string fText;
     std::vector<unsigned char> fBlob;
     void *fNull;
     void *fPtr;
     bool fIsActive;
   };

   void SqliteError(int errcode);

   struct sqlite3 *fDb;
   struct sqlite3_stmt *fQuery;
   unsigned int fNSlots;
   ULong64_t fNRow;
   std::vector<std::string> fColumnNames;
   std::vector<Types> fColumnTypes;
   std::map<Types, std::string> fTypeNames;
   std::vector<Value_t> fValues;
   std::mutex fLock;

public:
   RSqliteDS(std::string_view fileName, std::string_view query);
   ~RSqliteDS();
   void SetNSlots(unsigned int nSlots) final;
   const std::vector<std::string> &GetColumnNames() const final;
   bool HasColumn(std::string_view colName) const final;
   std::string GetTypeName(std::string_view colName) const final;
   std::vector<std::pair<ULong64_t, ULong64_t>> GetEntryRanges() final;
   bool SetEntry(unsigned int slot, ULong64_t entry) final;
   void Initialise() final;

protected:
   Record_t GetColumnReadersImpl(std::string_view name, const std::type_info &) final;
};

////////////////////////////////////////////////////////////////////////////////////////////////
/// \brief Factory method to create a SQlite RDataFrame.
/// \param[in] fileName Path of the sqlite file.
/// \param[in] SQL query that defines the data set.
RDataFrame MakeSqliteDataFrame(std::string_view fileName, std::string_view query);

} // ns RDF

} // ns ROOT

#endif
<commit_msg>[DF] Cosmetics in sqlite ds<commit_after>// Author: Jakob Blomer CERN  07/2018

/*************************************************************************
 * Copyright (C) 1995-2017, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

#ifndef ROOT_RSQLITEDS
#define ROOT_RSQLITEDS

#include "ROOT/RDataFrame.hxx"
#include "ROOT/RDataSource.hxx"

#include <map>
#include <memory>
#include <mutex>

#include <sqlite3.h>

namespace ROOT {

namespace RDF {

class RSqliteDS final : public ROOT::RDF::RDataSource {
private:
   enum class Types {
     kInteger,
     kReal,
     kText,
     kBlob,
     kNull
   };

   struct Value_t {
     Value_t() : fType(Types::kNull), fNull(nullptr), fPtr(nullptr), fIsActive(false) {}

     Types fType;
     Long64_t fInteger;
     double fReal;
     std::string fText;
     std::vector<unsigned char> fBlob;
     void *fNull;
     void *fPtr;
     bool fIsActive;
   };

   void SqliteError(int errcode);

   sqlite3 *fDb;
   sqlite3_stmt *fQuery;
   unsigned int fNSlots;
   ULong64_t fNRow;
   std::vector<std::string> fColumnNames;
   std::vector<Types> fColumnTypes;
   std::map<Types, std::string> fTypeNames;
   std::vector<Value_t> fValues;
   std::mutex fLock;

public:
   RSqliteDS(std::string_view fileName, std::string_view query);
   ~RSqliteDS();
   void SetNSlots(unsigned int nSlots) final;
   const std::vector<std::string> &GetColumnNames() const final;
   bool HasColumn(std::string_view colName) const final;
   std::string GetTypeName(std::string_view colName) const final;
   std::vector<std::pair<ULong64_t, ULong64_t>> GetEntryRanges() final;
   bool SetEntry(unsigned int slot, ULong64_t entry) final;
   void Initialise() final;

protected:
   Record_t GetColumnReadersImpl(std::string_view name, const std::type_info &) final;
};

////////////////////////////////////////////////////////////////////////////////////////////////
/// \brief Factory method to create a SQlite RDataFrame.
/// \param[in] fileName Path of the sqlite file.
/// \param[in] SQL query that defines the data set.
RDataFrame MakeSqliteDataFrame(std::string_view fileName, std::string_view query);

} // ns RDF

} // ns ROOT

#endif
<|endoftext|>


<commit_before>
#include "lepus/vm_context.h"
#include "parser/input_stream.h"
#include "lepus/scanner.h"
#include "lepus/parser.h"
#include "lepus/exception.h"
#include "lepus/semantic_analysis.h"
#include "lepus/code_generator.h"
#include "lepus/value.h"
#include "lepus/builtin.h"
#include "lepus/table.h"

namespace lepus {

#define GET_CONST_VALUE(i) \
        (function->GetConstValue(Instruction::GetParamBx(i)))
#define GET_Global_VALUE(i) \
        (global()->Get(Instruction::GetParamBx(i)))
#define GET_REGISTER_A(i)  \
        (frame->register_ + Instruction::GetParamA(i))
#define GET_REGISTER_B(i)  \
        (frame->register_ + Instruction::GetParamB(i))
#define GET_REGISTER_C(i)  \
        (frame->register_ + Instruction::GetParamC(i))

#define GET_UPVALUE_B(i)  (closure->GetUpvalue(Instruction::GetParamB(i)))

#define GET_REGISTER_ABC(i)                                 \
    a = GET_REGISTER_A(i);                                  \
    b = GET_REGISTER_B(i);                                  \
    c = GET_REGISTER_C(i);

    void VMContext::Initialize() {
        RegisterBuiltin(this);
    }

    void VMContext::Execute(const std::string& source) {
        try {
            parser::InputStream input;
            input.Write(source);
            Scanner scanner(&input, &string_pool_);
            Parser parser(&scanner);
            SemanticAnalysis semantic_analysis;
            CodeGenerator code_generator(this);
            base::ScopedPtr<ASTree> root;
            root.Reset(parser.Parse());
            root->Accept(&semantic_analysis, nullptr);
            root->Accept(&code_generator, &top_level_variables_);
        }catch(const lepus::Exception& exception) {
            std::cout<<exception.message()<<std::endl;
        }
        CallFunction(heap().top_ - 1, 0, nullptr);
        Run();
    }

    Value VMContext::Call(const std::string& name, const std::vector<Value>& args) {
        Value ret;
        auto reg_info = top_level_variables_.find(string_pool()->NewString(name));
        if(reg_info == top_level_variables_.end())
            return Value();
        int reg = reg_info->second;
        Value* function = heap_.top_;
        *(heap_.top_++) = *(heap_.base() + reg + 1);
        for(std::size_t i = 0; i < args.size(); ++i) {
            *(heap_.top_++) = args[i];
        }
        CallFunction(function, args.size(), &ret);
        Run();
        return ret;
    }

    int VMContext::GetParamsSize() {
        return heap().top_ - frames_.back().register_;
    }

    Value* VMContext::GetParam(int index) {
        return frames_.back().register_ + index;
    }

    bool VMContext::CallFunction(Value* function, int argc, Value* ret) {
        if(function->type_ == Value_Closure) {
            heap_.top_ = function + 1;
            Frame frame;
            frame.return_ = ret;
            frame.function_ = function;
            frame.instruction_ = static_cast<Function*>(function->closure_->function())->GetOpCodes();
            frame.end_ = frame.instruction_ + static_cast<Function*>(function->closure_->function())->OpCodeSize();
            frame.register_ = heap_.top_;
            frames_.push_back(frame);
            return true;
        }else if(function->type_ == Value_CFunction) {
            heap_.top_ = function + argc + 1;
            Frame frame;
            frame.return_ = ret;
            frame.function_ = function;
            frame.register_ = function + 1;
            frames_.push_back(frame);
            void* cfunction = function->native_function_;
            Value v = reinterpret_cast<CFunction>(cfunction)(this);
            if(ret){
                *ret = v;
            }
            frames_.pop_back();
            return false;
        }
        return false;
    }

    void VMContext::Run() {
        while(!frames_.empty()) {
            RunFrame();
        }
        heap_.top_ = heap_.base() + top_level_variables_.size() + 1;
    }

    void VMContext::RunFrame() {
        Frame *frame = &frames_.back();
        Closure* closure = frame->function_->closure_;
        Function *function = closure->function();
        Value *a = nullptr;
        Value *b = nullptr;
        Value *c = nullptr;
        while(frame->instruction_ < frame->end_) {
            Instruction i = *frame->instruction_++;
            switch (Instruction::GetOpCode(i)) {
                case TypeOp_LoadNil:
                    a = GET_REGISTER_A(i);
                    a->SetNil();
                    break;
                case TypeOp_LoadConst:
                    a = GET_REGISTER_A(i);
                    b = GET_CONST_VALUE(i);
                    *a = *b;
                    break;
                case TypeOp_Move:
                    a = GET_REGISTER_A(i);
                    b = GET_REGISTER_B(i);
                    *a = *b;
                    break;
                case TypeOp_GetUpvalue:
                    a = GET_REGISTER_A(i);
                    b = GET_UPVALUE_B(i);
                    *a = *b;
                    break;
                case TypeOp_SetUpvalue:
                    a = GET_REGISTER_A(i);
                    b = GET_UPVALUE_B(i);
                    *b = *a;
                    break;
                case TypeOp_GetGlobal:
                    a = GET_REGISTER_A(i);
                    b = GET_Global_VALUE(i);
                    *a = *b;
                    break;
                case TypeOp_SetGlobal:
                    break;
                case TypeOp_Closure:
                {
                    a = GET_REGISTER_A(i);
                    int index = Instruction::GetParamBx(i);
                    GenerateClosure(a, index);
                }
                    break;
                case TypeOp_Call:
                {
                    a = GET_REGISTER_A(i);
                    int argc = Instruction::GetParamB(i);
                    c = GET_REGISTER_C(i);
                    if(CallFunction(a, argc, c))
                        return;
                }
                    break;
                case TypeOp_Ret:
                    a = GET_REGISTER_A(i);
                    if(frame->return_ != nullptr) {
                        *frame->return_ = *a;
                    }
                    frames_.pop_back();
                    return;
                case TypeOp_JmpFalse:
                    a = GET_REGISTER_A(i);
                    if (a->IsFalse())
                        frame->instruction_ += -1 + Instruction::GetParamsBx(i);
                    break;
                case TypeOp_Jmp:
                    frame->instruction_ += -1 + Instruction::GetParamsBx(i);
                    break;
                case TypeOp_Neg:
                    a = GET_REGISTER_A(i);
                    a->number_ = -a->number_;
                    break;
                case TypeOp_Not:
                    a = GET_REGISTER_A(i);
                    a->boolean_ = !a->boolean_;
                    break;
                case TypeOp_Len:
                    break;
                case TypeOp_Add:
                    GET_REGISTER_ABC(i);
                    a->number_ = b->number_ + c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_Sub:
                    GET_REGISTER_ABC(i);
                    a->number_ = b->number_ - c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_Mul:
                    GET_REGISTER_ABC(i);
                    a->number_ = b->number_ * c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_Div:
                    GET_REGISTER_ABC(i);
                    a->number_ = b->number_ / c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_Pow:
                    break;
                case TypeOp_Mod:
                    GET_REGISTER_ABC(i);
                    a->number_ = int(b->number_/c->number_);
                    a->number_ = b->number_ - a->number_ * c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_And:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Boolean)
                        a->boolean_ = (b->boolean_ && c->boolean_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_Or:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Boolean)
                        a->boolean_ = (b->boolean_ || c->boolean_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_Less:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Number)
                        a->boolean_ = (b->number_ < c->number_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_Greater:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Number)
                        a->boolean_ = (b->number_ > c->number_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_Equal:
                    GET_REGISTER_ABC(i);
                    a->boolean_ = (*b == *c);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_UnEqual:
                    GET_REGISTER_ABC(i);
                    a->boolean_ = (*b != *c);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_LessEqual:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Number)
                        a->boolean_ = (b->number_ <= c->number_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_GreaterEqual:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Number)
                        a->boolean_ = (b->number_ >= c->number_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_NewTable:
                    break;
                case TypeOp_SetTable:
                    break;
                case TypeOp_GetTable:
                    GET_REGISTER_ABC(i);
                    if(b->type_ == Value_Table && c->type_ == Value_String) {
                        *a = static_cast<Dictonary*>(b->table_)->GetValue(c->str_);
                    }else if(b->type_ == Value_String && c->type_ == Value_String){
                        Value* v = global()->Find(string_pool()->NewString("String"));
                        *a = static_cast<Dictonary*>(v->table_)->GetValue(c->str_);
                    }

                    break;
                case TypeOp_Switch:
                {
                    a = GET_REGISTER_A(i);
                    int index = Instruction::GetParamBx(i);
                    int jmp = function->GetSwitch(index)->Switch(a);
                    frame->instruction_ += -1 + jmp;
                }
                    break;
                case TypeOp_Inc:
                    a = GET_REGISTER_A(i);
                    if (a->type_ == Value_Number)
                        a->number_ += 1;
                    break;
                case TypeOp_Dec:
                    a = GET_REGISTER_A(i);
                    if (a->type_ == Value_Number)
                        a->number_ -= 1;
                    break;
                case TypeOp_Noop:
                    break;
                default:
                    break;
            }
        }
        frames_.pop_back();
    }

    void VMContext::GenerateClosure(Value* value, int index) {
        Frame* frame = &frames_.back();
        Closure* current_closure = frame->function_->closure_;
        Function *function = current_closure->function()->GetChildFunction(index);

        Closure* closure = new Closure(function);

        std::size_t upvalues_count = function->UpvaluesSize();
        for(std::size_t i = 0; i < upvalues_count; ++i) {
            UpvalueInfo* info = function->GetUpvalue(i);
            if(info->in_parent_vars_) {
                Value* v = frame->register_ + info->register_;
                closure->AddUpvalue(v);
            }else{
                closure->AddUpvalue(current_closure->GetUpvalue(info->register_));
            }
        }

        value->type_ = Value_Closure;
        value->closure_ = closure;
    }

}
<commit_msg>修复bug<commit_after>
#include "lepus/vm_context.h"
#include "parser/input_stream.h"
#include "lepus/scanner.h"
#include "lepus/parser.h"
#include "lepus/exception.h"
#include "lepus/semantic_analysis.h"
#include "lepus/code_generator.h"
#include "lepus/value.h"
#include "lepus/builtin.h"
#include "lepus/table.h"

namespace lepus {

#define GET_CONST_VALUE(i) \
        (function->GetConstValue(Instruction::GetParamBx(i)))
#define GET_Global_VALUE(i) \
        (global()->Get(Instruction::GetParamBx(i)))
#define GET_REGISTER_A(i)  \
        (frame->register_ + Instruction::GetParamA(i))
#define GET_REGISTER_B(i)  \
        (frame->register_ + Instruction::GetParamB(i))
#define GET_REGISTER_C(i)  \
        (frame->register_ + Instruction::GetParamC(i))

#define GET_UPVALUE_B(i)  (closure->GetUpvalue(Instruction::GetParamB(i)))

#define GET_REGISTER_ABC(i)                                 \
    a = GET_REGISTER_A(i);                                  \
    b = GET_REGISTER_B(i);                                  \
    c = GET_REGISTER_C(i);

    void VMContext::Initialize() {
        RegisterBuiltin(this);
    }

    void VMContext::Execute(const std::string& source) {
        try {
            parser::InputStream input;
            input.Write(source);
            Scanner scanner(&input, &string_pool_);
            Parser parser(&scanner);
            SemanticAnalysis semantic_analysis;
            CodeGenerator code_generator(this);
            base::ScopedPtr<ASTree> root;
            root.Reset(parser.Parse());
            root->Accept(&semantic_analysis, nullptr);
            root->Accept(&code_generator, &top_level_variables_);
        }catch(const lepus::Exception& exception) {
            std::cout<<exception.message()<<std::endl;
        }
        CallFunction(heap().top_ - 1, 0, nullptr);
        Run();
    }

    Value VMContext::Call(const std::string& name, const std::vector<Value>& args) {
        Value ret;
        auto reg_info = top_level_variables_.find(string_pool()->NewString(name));
        if(reg_info == top_level_variables_.end())
            return Value();
        int reg = reg_info->second;
        Value* function = heap_.top_;
        *(heap_.top_++) = *(heap_.base() + reg + 1);
        for(std::size_t i = 0; i < args.size(); ++i) {
            *(heap_.top_++) = args[i];
        }
        CallFunction(function, args.size(), &ret);
        Run();
        return ret;
    }

    int VMContext::GetParamsSize() {
        return heap().top_ - frames_.back().register_;
    }

    Value* VMContext::GetParam(int index) {
        return frames_.back().register_ + index;
    }

    bool VMContext::CallFunction(Value* function, int argc, Value* ret) {
        if(function->type_ == Value_Closure) {
            heap_.top_ = function + 1;
            Frame frame;
            frame.return_ = ret;
            frame.function_ = function;
            frame.instruction_ = static_cast<Function*>(function->closure_->function())->GetOpCodes();
            frame.end_ = frame.instruction_ + static_cast<Function*>(function->closure_->function())->OpCodeSize();
            frame.register_ = heap_.top_;
            frames_.push_back(frame);
            return true;
        }else if(function->type_ == Value_CFunction) {
            heap_.top_ = function + argc + 1;
            Frame frame;
            frame.return_ = ret;
            frame.function_ = function;
            frame.register_ = function + 1;
            frames_.push_back(frame);
            void* cfunction = function->native_function_;
            Value v = reinterpret_cast<CFunction>(cfunction)(this);
            if(ret){
                *ret = v;
            }
            frames_.pop_back();
            return false;
        }
        return false;
    }

    void VMContext::Run() {
        while(!frames_.empty()) {
            RunFrame();
        }
        heap_.top_ = heap_.base() + top_level_variables_.size() + 1;
    }

    void VMContext::RunFrame() {
        Frame *frame = &frames_.back();
        Closure* closure = frame->function_->closure_;
        Function *function = closure->function();
        Value *a = nullptr;
        Value *b = nullptr;
        Value *c = nullptr;
        while(frame->instruction_ < frame->end_) {
            Instruction i = *frame->instruction_++;
            switch (Instruction::GetOpCode(i)) {
                case TypeOp_LoadNil:
                    a = GET_REGISTER_A(i);
                    a->SetNil();
                    break;
                case TypeOp_LoadConst:
                    a = GET_REGISTER_A(i);
                    b = GET_CONST_VALUE(i);
                    *a = *b;
                    break;
                case TypeOp_Move:
                    a = GET_REGISTER_A(i);
                    b = GET_REGISTER_B(i);
                    *a = *b;
                    break;
                case TypeOp_GetUpvalue:
                    a = GET_REGISTER_A(i);
                    b = GET_UPVALUE_B(i);
                    *a = *b;
                    break;
                case TypeOp_SetUpvalue:
                    a = GET_REGISTER_A(i);
                    b = GET_UPVALUE_B(i);
                    *b = *a;
                    break;
                case TypeOp_GetGlobal:
                    a = GET_REGISTER_A(i);
                    b = GET_Global_VALUE(i);
                    *a = *b;
                    break;
                case TypeOp_SetGlobal:
                    break;
                case TypeOp_Closure:
                {
                    a = GET_REGISTER_A(i);
                    int index = Instruction::GetParamBx(i);
                    GenerateClosure(a, index);
                }
                    break;
                case TypeOp_Call:
                {
                    a = GET_REGISTER_A(i);
                    int argc = Instruction::GetParamB(i);
                    c = GET_REGISTER_C(i);
                    if(CallFunction(a, argc, c))
                        return;
                }
                    break;
                case TypeOp_Ret:
                    a = GET_REGISTER_A(i);
                    if(frame->return_ != nullptr) {
                        *frame->return_ = *a;
                    }
                    frames_.pop_back();
                    return;
                case TypeOp_JmpFalse:
                    a = GET_REGISTER_A(i);
                    if (a->IsFalse())
                        frame->instruction_ += -1 + Instruction::GetParamsBx(i);
                    break;
                case TypeOp_Jmp:
                    frame->instruction_ += -1 + Instruction::GetParamsBx(i);
                    break;
                case TypeOp_Neg:
                    a = GET_REGISTER_A(i);
                    a->number_ = -a->number_;
                    break;
                case TypeOp_Not:
                    a = GET_REGISTER_A(i);
                    a->boolean_ = !a->boolean_;
                    break;
                case TypeOp_Len:
                    break;
                case TypeOp_Add:
                    GET_REGISTER_ABC(i);
                    a->number_ = b->number_ + c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_Sub:
                    GET_REGISTER_ABC(i);
                    a->number_ = b->number_ - c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_Mul:
                    GET_REGISTER_ABC(i);
                    a->number_ = b->number_ * c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_Div:
                    GET_REGISTER_ABC(i);
                    a->number_ = b->number_ / c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_Pow:
                    break;
                case TypeOp_Mod:
                    GET_REGISTER_ABC(i);
                    a->number_ = int(b->number_/c->number_);
                    a->number_ = b->number_ - a->number_ * c->number_;
                    a->type_ = Value_Number;
                    break;
                case TypeOp_And:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Boolean)
                        a->boolean_ = (b->boolean_ && c->boolean_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_Or:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Boolean)
                        a->boolean_ = (b->boolean_ || c->boolean_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_Less:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Number)
                        a->boolean_ = (b->number_ < c->number_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_Greater:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Number)
                        a->boolean_ = (b->number_ > c->number_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_Equal:
                    GET_REGISTER_ABC(i);
                    a->boolean_ = (*b == *c);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_UnEqual:
                    GET_REGISTER_ABC(i);
                    a->boolean_ = (*b != *c);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_LessEqual:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Number)
                        a->boolean_ = (b->number_ <= c->number_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_GreaterEqual:
                    GET_REGISTER_ABC(i);
                    if (b->type_ == Value_Number)
                        a->boolean_ = (b->number_ >= c->number_);
                    a->type_ = Value_Boolean;
                    break;
                case TypeOp_NewTable:
                    break;
                case TypeOp_SetTable:
                    break;
                case TypeOp_GetTable:
                    GET_REGISTER_ABC(i);
                    if(b->type_ == Value_Table && c->type_ == Value_String) {
                        *a = static_cast<Dictonary*>(b->table_)->GetValue(c->str_);
                    }else if(b->type_ == Value_String && c->type_ == Value_String){
                        Value* v = global()->Find(string_pool()->NewString("String"));
                        *a = static_cast<Dictonary*>(v->table_)->GetValue(c->str_);
                    }

                    break;
                case TypeOp_Switch:
                {
                    a = GET_REGISTER_A(i);
                    int index = Instruction::GetParamBx(i);
                    int jmp = function->GetSwitch(index)->Switch(a);
                    frame->instruction_ += -1 + jmp;
                }
                    break;
                case TypeOp_Inc:
                    a = GET_REGISTER_A(i);
                    if (a->type_ == Value_Number)
                        a->number_ += 1;
                    break;
                case TypeOp_Dec:
                    a = GET_REGISTER_A(i);
                    if (a->type_ == Value_Number)
                        a->number_ -= 1;
                    break;
                case TypeOp_Noop:
                    break;
                default:
                    break;
            }
        }
        if(frame->return_ != nullptr) {
            frame->return_->SetNil();
        }
        frames_.pop_back();
    }

    void VMContext::GenerateClosure(Value* value, int index) {
        Frame* frame = &frames_.back();
        Closure* current_closure = frame->function_->closure_;
        Function *function = current_closure->function()->GetChildFunction(index);

        Closure* closure = new Closure(function);

        std::size_t upvalues_count = function->UpvaluesSize();
        for(std::size_t i = 0; i < upvalues_count; ++i) {
            UpvalueInfo* info = function->GetUpvalue(i);
            if(info->in_parent_vars_) {
                Value* v = frame->register_ + info->register_;
                closure->AddUpvalue(v);
            }else{
                closure->AddUpvalue(current_closure->GetUpvalue(info->register_));
            }
        }

        value->type_ = Value_Closure;
        value->closure_ = closure;
    }

}
<|endoftext|>


<commit_before>// This file is part of gltfpack; see gltfpack.h for version/license details
#include "gltfpack.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#endif

struct BasisSettings
{
	int etc1s_l;
	int etc1s_q;
	int uastc_l;
	float uastc_q;
};

static const char* kMimeTypes[][2] = {
    {"image/jpeg", ".jpg"},
    {"image/jpeg", ".jpeg"},
    {"image/png", ".png"},
};

static const BasisSettings kBasisSettings[10] = {
    {1, 1, 0, 1.5f},
    {1, 6, 0, 1.f},
    {1, 20, 1, 1.0f},
    {1, 50, 1, 0.75f},
    {1, 90, 1, 0.5f},
    {1, 128, 1, 0.4f},
    {1, 160, 1, 0.34f},
    {1, 192, 1, 0.29f}, // default
    {1, 224, 2, 0.26f},
    {1, 255, 2, 0.f},
};

void analyzeImages(cgltf_data* data, std::vector<ImageInfo>& images)
{
	for (size_t i = 0; i < data->materials_count; ++i)
	{
		const cgltf_material& material = data->materials[i];

		if (material.has_pbr_metallic_roughness)
		{
			const cgltf_pbr_metallic_roughness& pbr = material.pbr_metallic_roughness;

			if (pbr.base_color_texture.texture && pbr.base_color_texture.texture->image)
				images[pbr.base_color_texture.texture->image - data->images].srgb = true;
		}

		if (material.has_pbr_specular_glossiness)
		{
			const cgltf_pbr_specular_glossiness& pbr = material.pbr_specular_glossiness;

			if (pbr.diffuse_texture.texture && pbr.diffuse_texture.texture->image)
				images[pbr.diffuse_texture.texture->image - data->images].srgb = true;
		}

		if (material.emissive_texture.texture && material.emissive_texture.texture->image)
			images[material.emissive_texture.texture->image - data->images].srgb = true;

		if (material.normal_texture.texture && material.normal_texture.texture->image)
			images[material.normal_texture.texture->image - data->images].normal_map = true;
	}
}

const char* inferMimeType(const char* path)
{
	const char* ext = strrchr(path, '.');
	if (!ext)
		return "";

	std::string extl = ext;
	for (size_t i = 0; i < extl.length(); ++i)
		extl[i] = char(tolower(extl[i]));

	for (size_t i = 0; i < sizeof(kMimeTypes) / sizeof(kMimeTypes[0]); ++i)
		if (extl == kMimeTypes[i][1])
			return kMimeTypes[i][0];

	return "";
}

static const char* mimeExtension(const char* mime_type)
{
	for (size_t i = 0; i < sizeof(kMimeTypes) / sizeof(kMimeTypes[0]); ++i)
		if (strcmp(kMimeTypes[i][0], mime_type) == 0)
			return kMimeTypes[i][1];

	return ".raw";
}

#ifdef __EMSCRIPTEN__
EM_JS(int, execute, (const char* cmd, bool ignore_stdout, bool ignore_stderr), {
	var cp = require('child_process');
	var stdio = [ 'ignore', ignore_stdout ? 'ignore' : 'inherit', ignore_stderr ? 'ignore' : 'inherit' ];
	var ret = cp.spawnSync(UTF8ToString(cmd), [], {shell:true, stdio:stdio });
	return ret.status == null ? 256 : ret.status;
});

EM_JS(const char*, readenv, (const char* name), {
	var val = process.env[UTF8ToString(name)];
	if (!val)
		return 0;
	var ret = _malloc(lengthBytesUTF8(val) + 1);
	stringToUTF8(val, ret, lengthBytesUTF8(val) + 1);
	return ret;
});
#else
static int execute(const char* cmd_, bool ignore_stdout, bool ignore_stderr)
{
#ifdef _WIN32
	std::string ignore = "nul";
#else
	std::string ignore = "/dev/null";
#endif

	std::string cmd = cmd_;

	if (ignore_stdout)
		(cmd += " >") += ignore;
	if (ignore_stderr)
		(cmd += " 2>") += ignore;

	return system(cmd.c_str());
}

static const char* readenv(const char* name)
{
	return getenv(name);
}
#endif

bool checkBasis(bool verbose)
{
	const char* basisu_path = readenv("BASISU_PATH");
	std::string cmd = basisu_path ? basisu_path : "basisu";

	cmd += " -version";

	int rc = execute(cmd.c_str(), /* ignore_stdout= */ true, /* ignore_stderr= */ true);
	if (verbose)
		printf("%s => %d\n", cmd.c_str(), rc);

	return rc == 0;
}

bool encodeBasis(const std::string& data, const char* mime_type, std::string& result, bool normal_map, bool srgb, int quality, float scale, bool pow2, bool uastc, bool verbose)
{
	(void)scale;
	(void)pow2;

	TempFile temp_input(mimeExtension(mime_type));
	TempFile temp_output(".basis");

	if (!writeFile(temp_input.path.c_str(), data))
		return false;

	const char* basisu_path = readenv("BASISU_PATH");
	std::string cmd = basisu_path ? basisu_path : "basisu";

	const BasisSettings& bs = kBasisSettings[quality <= 0 ? 0 : quality > 9 ? 9 : quality - 1];

	cmd += " -mipmap";

	if (normal_map)
	{
		cmd += " -normal_map";
		// for optimal quality we should specify seperate_rg_to_color_alpha but this requires renderer awareness
	}
	else if (!srgb)
	{
		cmd += " -linear";
	}

	if (uastc)
	{
		char settings[128];
		sprintf(settings, " -uastc_level %d -uastc_rdo_q %.2f", bs.uastc_l, bs.uastc_q);

		cmd += " -uastc";
		cmd += settings;
		cmd += " -uastc_rdo_d 1024";
	}
	else
	{
		char settings[128];
		sprintf(settings, " -comp_level %d -q %d", bs.etc1s_l, bs.etc1s_q);

		cmd += settings;
	}

	cmd += " -file ";
	cmd += temp_input.path;
	cmd += " -output_file ";
	cmd += temp_output.path;

	int rc = execute(cmd.c_str(), /* ignore_stdout= */ true, /* ignore_stderr= */ false);
	if (verbose)
		printf("%s => %d\n", cmd.c_str(), rc);

	return rc == 0 && readFile(temp_output.path.c_str(), result);
}

bool checkKtx(bool verbose)
{
	const char* toktx_path = readenv("TOKTX_PATH");
	std::string cmd = toktx_path ? toktx_path : "toktx";

	cmd += " --version";

	int rc = execute(cmd.c_str(), /* ignore_stdout= */ true, /* ignore_stderr= */ true);
	if (verbose)
		printf("%s => %d\n", cmd.c_str(), rc);

	return rc == 0;
}

static int readInt16(const std::string& data, size_t offset)
{
	return (unsigned char)data[offset] * 256 + (unsigned char)data[offset + 1];
}

static bool getDimensionsPng(const std::string& data, int& width, int& height)
{
	if (data.size() < 8 + 8 + 13 + 4)
		return false;

	const char* signature = "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a";
	if (data.compare(0, 8, signature) != 0)
		return false;

	if (data.compare(12, 4, "IHDR") != 0)
		return false;

	width = readInt16(data, 18);
	height = readInt16(data, 22);

	return true;
}

static bool getDimensionsJpeg(const std::string& data, int& width, int& height)
{
	size_t offset = 0;

	// note, this can stop parsing before reaching the end but we stop at SOF anyway
	while (offset + 4 <= data.size())
	{
		if (data[offset] != '\xff')
			return false;

		char marker = data[offset];

		if (marker == '\xff')
		{
			offset++;
			continue; // padding
		}

		// d0..d9 correspond to SOI, RSTn, EOI
		if (marker == 0 || unsigned(marker - '\xd0') <= 9)
		{
			offset += 2;
			continue; // no payload
		}

		// c0..c1 correspond to SOF0, SOF1
		if (marker == '\xc0' || marker == '\xc2')
		{
			if (offset + 10 > data.size())
				return false;

			width = readInt16(data, offset + 7);
			height = readInt16(data, offset + 5);

			return true;
		}

		offset += 2 + readInt16(data, offset + 2);
	}

	return false;
}

static bool getDimensions(const std::string& data, const char* mime_type, int& width, int& height)
{
	if (strcmp(mime_type, "image/png") == 0)
		return getDimensionsPng(data, width, height);
	if (strcmp(mime_type, "image/jpeg") == 0)
		return getDimensionsJpeg(data, width, height);

	return false;
}

static int roundPow2(int value)
{
	int result = 1;

	while (result < value)
		result <<= 1;

	// to prevent odd texture sizes from increasing the size too much, we round to nearest power of 2 above a certain size
	if (value > 128 && result * 3 / 4 > value)
		result >>= 1;

	return result;
}

static int roundBlock(int value)
{
	return (value + 3) & ~3;
}

bool encodeKtx(const std::string& data, const char* mime_type, std::string& result, bool normal_map, bool srgb, int quality, float scale, bool pow2, bool uastc, bool verbose)
{
	int width = 0, height = 0;
	if (!getDimensions(data, mime_type, width, height))
		return false;

	TempFile temp_input(mimeExtension(mime_type));
	TempFile temp_output(".ktx2");

	if (!writeFile(temp_input.path.c_str(), data))
		return false;

	const char* toktx_path = readenv("TOKTX_PATH");
	std::string cmd = toktx_path ? toktx_path : "toktx";

	const BasisSettings& bs = kBasisSettings[quality <= 0 ? 0 : quality > 9 ? 9 : quality - 1];

	cmd += " --t2";
	cmd += " --2d";
	cmd += " --genmipmap";
	cmd += " --nowarn";

	int newWidth = roundBlock(int(width * scale)), newHeight = roundBlock(int(height * scale));

	if (pow2)
	{
		newWidth = roundPow2(newWidth);
		newHeight = roundPow2(newHeight);
	}

	if (newWidth != width || newHeight != height)
	{
		char wh[128];
		sprintf(wh, " --resize %dx%d", newWidth, newHeight);
		cmd += wh;
	}

	if (uastc)
	{
		char settings[128];
		sprintf(settings, " %d --uastc_rdo_q %.2f", bs.uastc_l, bs.uastc_q);

		cmd += " --uastc";
		cmd += settings;
		cmd += " --uastc_rdo_d 1024";
		cmd += " --zcmp 9";
	}
	else
	{
		char settings[128];
		sprintf(settings, " --clevel %d --qlevel %d", bs.etc1s_l, bs.etc1s_q);

		cmd += " --bcmp";
		cmd += settings;

		// for optimal quality we should specify separate_rg_to_color_alpha but this requires renderer awareness
		if (normal_map)
			cmd += " --normal_map";
	}

	if (srgb)
		cmd += " --srgb";
	else
		cmd += " --linear";

	cmd += " -- ";
	cmd += temp_output.path;
	cmd += " ";
	cmd += temp_input.path;

	int rc = execute(cmd.c_str(), /* ignore_stdout= */ false, /* ignore_stderr= */ false);
	if (verbose)
		printf("%s => %d\n", cmd.c_str(), rc);

	return rc == 0 && readFile(temp_output.path.c_str(), result);
}
<commit_msg>gltfpack: Fix JPEG frame parsing<commit_after>// This file is part of gltfpack; see gltfpack.h for version/license details
#include "gltfpack.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#endif

struct BasisSettings
{
	int etc1s_l;
	int etc1s_q;
	int uastc_l;
	float uastc_q;
};

static const char* kMimeTypes[][2] = {
    {"image/jpeg", ".jpg"},
    {"image/jpeg", ".jpeg"},
    {"image/png", ".png"},
};

static const BasisSettings kBasisSettings[10] = {
    {1, 1, 0, 1.5f},
    {1, 6, 0, 1.f},
    {1, 20, 1, 1.0f},
    {1, 50, 1, 0.75f},
    {1, 90, 1, 0.5f},
    {1, 128, 1, 0.4f},
    {1, 160, 1, 0.34f},
    {1, 192, 1, 0.29f}, // default
    {1, 224, 2, 0.26f},
    {1, 255, 2, 0.f},
};

void analyzeImages(cgltf_data* data, std::vector<ImageInfo>& images)
{
	for (size_t i = 0; i < data->materials_count; ++i)
	{
		const cgltf_material& material = data->materials[i];

		if (material.has_pbr_metallic_roughness)
		{
			const cgltf_pbr_metallic_roughness& pbr = material.pbr_metallic_roughness;

			if (pbr.base_color_texture.texture && pbr.base_color_texture.texture->image)
				images[pbr.base_color_texture.texture->image - data->images].srgb = true;
		}

		if (material.has_pbr_specular_glossiness)
		{
			const cgltf_pbr_specular_glossiness& pbr = material.pbr_specular_glossiness;

			if (pbr.diffuse_texture.texture && pbr.diffuse_texture.texture->image)
				images[pbr.diffuse_texture.texture->image - data->images].srgb = true;
		}

		if (material.emissive_texture.texture && material.emissive_texture.texture->image)
			images[material.emissive_texture.texture->image - data->images].srgb = true;

		if (material.normal_texture.texture && material.normal_texture.texture->image)
			images[material.normal_texture.texture->image - data->images].normal_map = true;
	}
}

const char* inferMimeType(const char* path)
{
	const char* ext = strrchr(path, '.');
	if (!ext)
		return "";

	std::string extl = ext;
	for (size_t i = 0; i < extl.length(); ++i)
		extl[i] = char(tolower(extl[i]));

	for (size_t i = 0; i < sizeof(kMimeTypes) / sizeof(kMimeTypes[0]); ++i)
		if (extl == kMimeTypes[i][1])
			return kMimeTypes[i][0];

	return "";
}

static const char* mimeExtension(const char* mime_type)
{
	for (size_t i = 0; i < sizeof(kMimeTypes) / sizeof(kMimeTypes[0]); ++i)
		if (strcmp(kMimeTypes[i][0], mime_type) == 0)
			return kMimeTypes[i][1];

	return ".raw";
}

#ifdef __EMSCRIPTEN__
EM_JS(int, execute, (const char* cmd, bool ignore_stdout, bool ignore_stderr), {
	var cp = require('child_process');
	var stdio = [ 'ignore', ignore_stdout ? 'ignore' : 'inherit', ignore_stderr ? 'ignore' : 'inherit' ];
	var ret = cp.spawnSync(UTF8ToString(cmd), [], {shell:true, stdio:stdio });
	return ret.status == null ? 256 : ret.status;
});

EM_JS(const char*, readenv, (const char* name), {
	var val = process.env[UTF8ToString(name)];
	if (!val)
		return 0;
	var ret = _malloc(lengthBytesUTF8(val) + 1);
	stringToUTF8(val, ret, lengthBytesUTF8(val) + 1);
	return ret;
});
#else
static int execute(const char* cmd_, bool ignore_stdout, bool ignore_stderr)
{
#ifdef _WIN32
	std::string ignore = "nul";
#else
	std::string ignore = "/dev/null";
#endif

	std::string cmd = cmd_;

	if (ignore_stdout)
		(cmd += " >") += ignore;
	if (ignore_stderr)
		(cmd += " 2>") += ignore;

	return system(cmd.c_str());
}

static const char* readenv(const char* name)
{
	return getenv(name);
}
#endif

bool checkBasis(bool verbose)
{
	const char* basisu_path = readenv("BASISU_PATH");
	std::string cmd = basisu_path ? basisu_path : "basisu";

	cmd += " -version";

	int rc = execute(cmd.c_str(), /* ignore_stdout= */ true, /* ignore_stderr= */ true);
	if (verbose)
		printf("%s => %d\n", cmd.c_str(), rc);

	return rc == 0;
}

bool encodeBasis(const std::string& data, const char* mime_type, std::string& result, bool normal_map, bool srgb, int quality, float scale, bool pow2, bool uastc, bool verbose)
{
	(void)scale;
	(void)pow2;

	TempFile temp_input(mimeExtension(mime_type));
	TempFile temp_output(".basis");

	if (!writeFile(temp_input.path.c_str(), data))
		return false;

	const char* basisu_path = readenv("BASISU_PATH");
	std::string cmd = basisu_path ? basisu_path : "basisu";

	const BasisSettings& bs = kBasisSettings[quality <= 0 ? 0 : quality > 9 ? 9 : quality - 1];

	cmd += " -mipmap";

	if (normal_map)
	{
		cmd += " -normal_map";
		// for optimal quality we should specify seperate_rg_to_color_alpha but this requires renderer awareness
	}
	else if (!srgb)
	{
		cmd += " -linear";
	}

	if (uastc)
	{
		char settings[128];
		sprintf(settings, " -uastc_level %d -uastc_rdo_q %.2f", bs.uastc_l, bs.uastc_q);

		cmd += " -uastc";
		cmd += settings;
		cmd += " -uastc_rdo_d 1024";
	}
	else
	{
		char settings[128];
		sprintf(settings, " -comp_level %d -q %d", bs.etc1s_l, bs.etc1s_q);

		cmd += settings;
	}

	cmd += " -file ";
	cmd += temp_input.path;
	cmd += " -output_file ";
	cmd += temp_output.path;

	int rc = execute(cmd.c_str(), /* ignore_stdout= */ true, /* ignore_stderr= */ false);
	if (verbose)
		printf("%s => %d\n", cmd.c_str(), rc);

	return rc == 0 && readFile(temp_output.path.c_str(), result);
}

bool checkKtx(bool verbose)
{
	const char* toktx_path = readenv("TOKTX_PATH");
	std::string cmd = toktx_path ? toktx_path : "toktx";

	cmd += " --version";

	int rc = execute(cmd.c_str(), /* ignore_stdout= */ true, /* ignore_stderr= */ true);
	if (verbose)
		printf("%s => %d\n", cmd.c_str(), rc);

	return rc == 0;
}

static int readInt16(const std::string& data, size_t offset)
{
	return (unsigned char)data[offset] * 256 + (unsigned char)data[offset + 1];
}

static bool getDimensionsPng(const std::string& data, int& width, int& height)
{
	if (data.size() < 8 + 8 + 13 + 4)
		return false;

	const char* signature = "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a";
	if (data.compare(0, 8, signature) != 0)
		return false;

	if (data.compare(12, 4, "IHDR") != 0)
		return false;

	width = readInt16(data, 18);
	height = readInt16(data, 22);

	return true;
}

static bool getDimensionsJpeg(const std::string& data, int& width, int& height)
{
	size_t offset = 0;

	// note, this can stop parsing before reaching the end but we stop at SOF anyway
	while (offset + 4 <= data.size())
	{
		if (data[offset] != '\xff')
			return false;

		char marker = data[offset + 1];

		if (marker == '\xff')
		{
			offset++;
			continue; // padding
		}

		// d0..d9 correspond to SOI, RSTn, EOI
		if (marker == 0 || unsigned(marker - '\xd0') <= 9)
		{
			offset += 2;
			continue; // no payload
		}

		// c0..c1 correspond to SOF0, SOF1
		if (marker == '\xc0' || marker == '\xc2')
		{
			if (offset + 10 > data.size())
				return false;

			width = readInt16(data, offset + 7);
			height = readInt16(data, offset + 5);

			return true;
		}

		offset += 2 + readInt16(data, offset + 2);
	}

	return false;
}

static bool getDimensions(const std::string& data, const char* mime_type, int& width, int& height)
{
	if (strcmp(mime_type, "image/png") == 0)
		return getDimensionsPng(data, width, height);
	if (strcmp(mime_type, "image/jpeg") == 0)
		return getDimensionsJpeg(data, width, height);

	return false;
}

static int roundPow2(int value)
{
	int result = 1;

	while (result < value)
		result <<= 1;

	// to prevent odd texture sizes from increasing the size too much, we round to nearest power of 2 above a certain size
	if (value > 128 && result * 3 / 4 > value)
		result >>= 1;

	return result;
}

static int roundBlock(int value)
{
	return (value + 3) & ~3;
}

bool encodeKtx(const std::string& data, const char* mime_type, std::string& result, bool normal_map, bool srgb, int quality, float scale, bool pow2, bool uastc, bool verbose)
{
	int width = 0, height = 0;
	if (!getDimensions(data, mime_type, width, height))
		return false;

	TempFile temp_input(mimeExtension(mime_type));
	TempFile temp_output(".ktx2");

	if (!writeFile(temp_input.path.c_str(), data))
		return false;

	const char* toktx_path = readenv("TOKTX_PATH");
	std::string cmd = toktx_path ? toktx_path : "toktx";

	const BasisSettings& bs = kBasisSettings[quality <= 0 ? 0 : quality > 9 ? 9 : quality - 1];

	cmd += " --t2";
	cmd += " --2d";
	cmd += " --genmipmap";
	cmd += " --nowarn";

	int newWidth = roundBlock(int(width * scale)), newHeight = roundBlock(int(height * scale));

	if (pow2)
	{
		newWidth = roundPow2(newWidth);
		newHeight = roundPow2(newHeight);
	}

	if (newWidth != width || newHeight != height)
	{
		char wh[128];
		sprintf(wh, " --resize %dx%d", newWidth, newHeight);
		cmd += wh;
	}

	if (uastc)
	{
		char settings[128];
		sprintf(settings, " %d --uastc_rdo_q %.2f", bs.uastc_l, bs.uastc_q);

		cmd += " --uastc";
		cmd += settings;
		cmd += " --uastc_rdo_d 1024";
		cmd += " --zcmp 9";
	}
	else
	{
		char settings[128];
		sprintf(settings, " --clevel %d --qlevel %d", bs.etc1s_l, bs.etc1s_q);

		cmd += " --bcmp";
		cmd += settings;

		// for optimal quality we should specify separate_rg_to_color_alpha but this requires renderer awareness
		if (normal_map)
			cmd += " --normal_map";
	}

	if (srgb)
		cmd += " --srgb";
	else
		cmd += " --linear";

	cmd += " -- ";
	cmd += temp_output.path;
	cmd += " ";
	cmd += temp_input.path;

	int rc = execute(cmd.c_str(), /* ignore_stdout= */ false, /* ignore_stderr= */ false);
	if (verbose)
		printf("%s => %d\n", cmd.c_str(), rc);

	return rc == 0 && readFile(temp_output.path.c_str(), result);
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 by Philip N. Garner
 *
 * See the file COPYING for the licence associated with this software.
 *
 * Author(s):
 *   Phil Garner, March 2016
 */

#include <istream>
#include <cctype>
#include <cstring>
#include <lube/var.h>

namespace libube
{
    /**
     * Ad-hoc JSON (JavaScript Object Notation) parser; (see http://json.org/)
     */
    class JSON
    {
    public:
        var operator ()(std::istream& iStream);
    private:
        char peek(std::istream& iStream);
        var doValue(std::istream& iStream);
        var doObject(std::istream& iStream);
        var doArray(std::istream& iStream);
        var doString(std::istream& iStream);
        var doRaw(std::istream& iStream);
    };
}

using namespace libube;
using namespace std;

/** Find the next character after whitespace */
char JSON::peek(std::istream& iStream)
{
    char c = iStream.peek();
    while (isspace(c))
    {
        iStream.get();
        c = iStream.peek();
    }
    return c;
}

var JSON::doValue(std::istream& iStream)
{
    var val;
    switch (peek(iStream))
    {
    case '{':
        val = doObject(iStream);
        break;
    case '[':
        val = doArray(iStream);
        break;
    case '"':
        val = doString(iStream);
        break;
    default:
        val = doRaw(iStream);
    }
    return val;
}

var JSON::doObject(std::istream& iStream)
{
    var obj;
    obj[nil];
    var key;
    char got = iStream.get();
    if (got != '{')
        throw error("JSON object doesn't start with {");
    while (iStream)
    {
        switch (peek(iStream))
        {
        case '}':
            iStream.get();
            return obj;
            break;
        case '"':
            key = doString(iStream);
            break;
        case ':':
            iStream.get();
            obj[key] = doValue(iStream);
            break;
        case ',':
            iStream.get();
            key = nil;
            break;
        default:
            throw error("Unknown character");
        }
    }
    return obj;
}

var JSON::doArray(std::istream& iStream)
{
    var arr;
    char got = iStream.get();
    if (got != '[')
        throw error("JSON array doesn't start with [");
    while (iStream)
    {
        switch (peek(iStream))
        {
        case ']':
            iStream.get();
            return arr;
            break;
        case ',':
            iStream.get();
            break;
        default:
            arr.push(doValue(iStream));
        }
    }
    return arr;
}

var JSON::doString(std::istream& iStream)
{
    var str = "";
    char got = iStream.get();
    if (got != '\"')
        throw error("JSON string doesn't start with \"");
    while (iStream.get(got))
        switch (got)
        {
        case '\"':
            return str;
        case '\\':
            // Escape character
            got = iStream.get();
            switch (got)
            {
            case '\"':
                str.push(got);
                break;
            default:
                throw error("Unrecognised JSON string escape");
            }
            break;
        default:
            str.push(got);
        }
    throw error("Unterminated JSON string");
    return nil;
}

var JSON::doRaw(std::istream& iStream)
{
    // Read up to a space or JSON delimiter
    var val = "";
    char c;
    while (iStream.get(c))
    {
        if (isspace(c))
            break;
        if (strchr(",[]{}", c))
            break;
        val.push(c);
    }

    // These values have special meanings
    if (val == "true")
        return 1;
    else if (val == "false")
        return 0;
    else if (val == "null")
        return nil;

    // It's probably a number, but return a string for the moment
    return val;
}

/**
 * For proper JSON we should call doObject(), mandating it be in braces.  In
 * calling doValue() we're allowing just raw values which is OK for lube but
 * not strictly JSON.
 */
var JSON::operator()(std::istream& iStream)
{
    var val = doValue(iStream);
    return val;
}

std::istream& libube::operator >>(std::istream& iStream, var& ioVar)
{
    JSON json;
    var in = json(iStream);
    ioVar = in;
    return iStream;
}
<commit_msg>Parse JSON raw values to numbers (long or double)<commit_after>/*
 * Copyright 2016 by Philip N. Garner
 *
 * See the file COPYING for the licence associated with this software.
 *
 * Author(s):
 *   Phil Garner, March 2016
 */

#include <istream>
#include <cctype>
#include <cstring>
#include <lube/var.h>

namespace libube
{
    /**
     * Ad-hoc JSON (JavaScript Object Notation) parser; (see http://json.org/)
     */
    class JSON
    {
    public:
        var operator ()(std::istream& iStream);
    private:
        char peek(std::istream& iStream);
        var doValue(std::istream& iStream);
        var doObject(std::istream& iStream);
        var doArray(std::istream& iStream);
        var doString(std::istream& iStream);
        var doRaw(std::istream& iStream);
    };
}

using namespace libube;
using namespace std;

/** Find the next character after whitespace */
char JSON::peek(std::istream& iStream)
{
    char c = iStream.peek();
    while (isspace(c))
    {
        iStream.get();
        c = iStream.peek();
    }
    return c;
}

var JSON::doValue(std::istream& iStream)
{
    var val;
    switch (peek(iStream))
    {
    case '{':
        val = doObject(iStream);
        break;
    case '[':
        val = doArray(iStream);
        break;
    case '"':
        val = doString(iStream);
        break;
    default:
        val = doRaw(iStream);
    }
    return val;
}

var JSON::doObject(std::istream& iStream)
{
    var obj;
    obj[nil];
    var key;
    char got = iStream.get();
    if (got != '{')
        throw error("JSON object doesn't start with {");
    while (iStream)
    {
        switch (peek(iStream))
        {
        case '}':
            iStream.get();
            return obj;
            break;
        case '"':
            key = doString(iStream);
            break;
        case ':':
            iStream.get();
            obj[key] = doValue(iStream);
            break;
        case ',':
            iStream.get();
            key = nil;
            break;
        default:
            throw error("Unknown character");
        }
    }
    return obj;
}

var JSON::doArray(std::istream& iStream)
{
    var arr;
    char got = iStream.get();
    if (got != '[')
        throw error("JSON array doesn't start with [");
    while (iStream)
    {
        switch (peek(iStream))
        {
        case ']':
            iStream.get();
            return arr;
            break;
        case ',':
            iStream.get();
            break;
        default:
            arr.push(doValue(iStream));
        }
    }
    return arr;
}

var JSON::doString(std::istream& iStream)
{
    var str = "";
    char got = iStream.get();
    if (got != '\"')
        throw error("JSON string doesn't start with \"");
    while (iStream.get(got))
        switch (got)
        {
        case '\"':
            return str;
        case '\\':
            // Escape character
            got = iStream.get();
            switch (got)
            {
            case '\"':
                str.push(got);
                break;
            default:
                throw error("Unrecognised JSON string escape");
            }
            break;
        default:
            str.push(got);
        }
    throw error("Unterminated JSON string");
    return nil;
}

var toNumber(var iStr)
{
    // Try for integer
    varstream ls(iStr.copy());
    long l;
    ls >> l;
    if (!ls.fail())
        return l;

    // Try for double
    varstream ds(iStr.copy());
    double d;
    ds >> d;
    if (!ds.fail())
        return d;

    // Failed
    return nil;
}

var JSON::doRaw(std::istream& iStream)
{
    // Read up to a space or JSON delimiter
    var val = "";
    char c;
    while (iStream.get(c))
    {
        if (isspace(c))
            break;
        if (strchr(",[]{}", c))
            break;
        val.push(c);
    }

    // These values have special meanings
    if (val == "true")
        return 1;
    else if (val == "false")
        return 0;
    else if (val == "null")
        return nil;

    // It should be a number
    var num = toNumber(val);
    if (num)
        return num;

    // Broke
    var e = "Unrecognised JSON raw value: ";
    e.append(val);
    throw error(e);
}

/**
 * For proper JSON we should call doObject(), mandating it be in braces.  In
 * calling doValue() we're allowing just raw values which is OK for lube but
 * not strictly JSON.
 */
var JSON::operator()(std::istream& iStream)
{
    var val = doValue(iStream);
    return val;
}

std::istream& libube::operator >>(std::istream& iStream, var& ioVar)
{
    JSON json;
    var in = json(iStream);
    ioVar = in;
    return iStream;
}
<|endoftext|>


<commit_before>/*
 * Copyright 2011 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

/* Tests text rendering with LCD and subpixel rendering turned on and off.
 */

#include "gm.h"
#include "sk_tool_utils.h"
#include "SkCanvas.h"
#include "SkPicture.h"
#include "SkPictureImageFilter.h"
#include "SkPictureRecorder.h"
#include "SkSurface.h"

class LcdTextGM : public skiagm::GM {
public:
    LcdTextGM() {
        const int pointSize = 36;
        textHeight = SkIntToScalar(pointSize);
    }

protected:

    SkString onShortName() {
        SkString name("lcdtext");
        name.append(sk_tool_utils::major_platform_os_name());
        return name;
    }

    SkISize onISize() { return SkISize::Make(640, 480); }

    virtual void onDraw(SkCanvas* canvas) {

        y = textHeight;
        drawText(canvas, SkString("TEXT: SubpixelTrue LCDRenderTrue"),
                 true,  true);
        drawText(canvas, SkString("TEXT: SubpixelTrue LCDRenderFalse"),
                 true,  false);
        drawText(canvas, SkString("TEXT: SubpixelFalse LCDRenderTrue"),
                 false, true);
        drawText(canvas, SkString("TEXT: SubpixelFalse LCDRenderFalse"),
                 false, false);
    }

    void drawText(SkCanvas* canvas, const SkString& string,
                  bool subpixelTextEnabled, bool lcdRenderTextEnabled) {
        SkPaint paint;
        paint.setColor(SK_ColorBLACK);
        paint.setDither(true);
        paint.setAntiAlias(true);
        paint.setSubpixelText(subpixelTextEnabled);
        paint.setLCDRenderText(lcdRenderTextEnabled);
        paint.setTextSize(textHeight);

        canvas->drawString(string, 0, y, paint);
        y += textHeight;
    }

private:
    typedef skiagm::GM INHERITED;
    SkScalar y, textHeight;
};

/*
 *  Skia will automatically disable LCD requests if the total size exceeds some limit
 *  (hard coded in this test for now, as it is now avaiable as an API)
 *
 *  Test this both by changing "textsize" and by changing the computed size (textsize * CTM)
 */
class LcdTextSizeGM : public skiagm::GM {
    enum {
        kLCDTextSizeLimit = 48
    };

    static void ScaleAbout(SkCanvas* canvas, SkScalar sx, SkScalar sy, SkScalar px, SkScalar py) {
        SkMatrix m;
        m.setScale(sx, sy, px, py);
        canvas->concat(m);
    }

public:
    LcdTextSizeGM() {}

protected:
    SkString onShortName() {
        return SkString("lcdtextsize");
    }

    SkISize onISize() { return SkISize::Make(320, 120); }

    virtual void onDraw(SkCanvas* canvas) {
        const char* lcd_text = "LCD";
        const char* gray_text = "GRAY";

        SkPaint paint;
        paint.setAntiAlias(true);
        paint.setLCDRenderText(true);

        const struct {
            SkPoint     fLoc;
            SkScalar    fTextSize;
            SkScalar    fScale;
            const char* fText;
        } rec[] = {
            { {  10,  50 }, kLCDTextSizeLimit - 1,     1,  lcd_text },
            { { 160,  50 }, kLCDTextSizeLimit + 1,     1,  gray_text },
            { {  10, 100 }, kLCDTextSizeLimit / 2, 1.99f,  lcd_text },
            { { 160, 100 }, kLCDTextSizeLimit / 2, 2.01f,  gray_text },
        };

        for (size_t i = 0; i < SK_ARRAY_COUNT(rec); ++i) {
            const SkPoint loc = rec[i].fLoc;
            SkAutoCanvasRestore acr(canvas, true);

            paint.setTextSize(rec[i].fTextSize);
            ScaleAbout(canvas, rec[i].fScale, rec[i].fScale, loc.x(), loc.y());
            canvas->drawString(rec[i].fText, loc.x(), loc.y(), paint);
        }
    }

private:
    typedef skiagm::GM INHERITED;
};
DEF_GM( return new LcdTextGM; )
DEF_GM( return new LcdTextSizeGM; )

///////////////////////////////////////////////////////////////////////////////////////////////////

DEF_SIMPLE_GM(savelayer_lcdtext, canvas, 620, 260) {
    SkPaint paint;
    paint.setAntiAlias(true);
    paint.setLCDRenderText(true);
    paint.setTextSize(20);

    canvas->drawString("Hamburgefons", 30, 30, paint);

    const bool gPreserveLCDText[] = { false, true };

    canvas->translate(0, 20);
    for (auto preserve : gPreserveLCDText) {
        preserve ? canvas->saveLayerPreserveLCDTextRequests(nullptr, nullptr)
                 : canvas->saveLayer(nullptr, nullptr);

        canvas->drawString("Hamburgefons", 30, 60, paint);

        SkPaint p;
        p.setColor(0xFFCCCCCC);
        canvas->drawRect(SkRect::MakeLTRB(25, 70, 200, 100), p);
        canvas->drawString("Hamburgefons", 30, 90, paint);

        canvas->restore();
        canvas->translate(0, 80);
    }
}
<commit_msg>Legacy raster still doesn't handle transparent LCD<commit_after>/*
 * Copyright 2011 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

/* Tests text rendering with LCD and subpixel rendering turned on and off.
 */

#include "gm.h"
#include "sk_tool_utils.h"
#include "SkCanvas.h"
#include "SkPicture.h"
#include "SkPictureImageFilter.h"
#include "SkPictureRecorder.h"
#include "SkSurface.h"

class LcdTextGM : public skiagm::GM {
public:
    LcdTextGM() {
        const int pointSize = 36;
        textHeight = SkIntToScalar(pointSize);
    }

protected:

    SkString onShortName() {
        SkString name("lcdtext");
        name.append(sk_tool_utils::major_platform_os_name());
        return name;
    }

    SkISize onISize() { return SkISize::Make(640, 480); }

    virtual void onDraw(SkCanvas* canvas) {

        y = textHeight;
        drawText(canvas, SkString("TEXT: SubpixelTrue LCDRenderTrue"),
                 true,  true);
        drawText(canvas, SkString("TEXT: SubpixelTrue LCDRenderFalse"),
                 true,  false);
        drawText(canvas, SkString("TEXT: SubpixelFalse LCDRenderTrue"),
                 false, true);
        drawText(canvas, SkString("TEXT: SubpixelFalse LCDRenderFalse"),
                 false, false);
    }

    void drawText(SkCanvas* canvas, const SkString& string,
                  bool subpixelTextEnabled, bool lcdRenderTextEnabled) {
        SkPaint paint;
        paint.setColor(SK_ColorBLACK);
        paint.setDither(true);
        paint.setAntiAlias(true);
        paint.setSubpixelText(subpixelTextEnabled);
        paint.setLCDRenderText(lcdRenderTextEnabled);
        paint.setTextSize(textHeight);

        canvas->drawString(string, 0, y, paint);
        y += textHeight;
    }

private:
    typedef skiagm::GM INHERITED;
    SkScalar y, textHeight;
};

/*
 *  Skia will automatically disable LCD requests if the total size exceeds some limit
 *  (hard coded in this test for now, as it is now avaiable as an API)
 *
 *  Test this both by changing "textsize" and by changing the computed size (textsize * CTM)
 */
class LcdTextSizeGM : public skiagm::GM {
    enum {
        kLCDTextSizeLimit = 48
    };

    static void ScaleAbout(SkCanvas* canvas, SkScalar sx, SkScalar sy, SkScalar px, SkScalar py) {
        SkMatrix m;
        m.setScale(sx, sy, px, py);
        canvas->concat(m);
    }

public:
    LcdTextSizeGM() {}

protected:
    SkString onShortName() {
        return SkString("lcdtextsize");
    }

    SkISize onISize() { return SkISize::Make(320, 120); }

    virtual void onDraw(SkCanvas* canvas) {
        const char* lcd_text = "LCD";
        const char* gray_text = "GRAY";

        SkPaint paint;
        paint.setAntiAlias(true);
        paint.setLCDRenderText(true);

        const struct {
            SkPoint     fLoc;
            SkScalar    fTextSize;
            SkScalar    fScale;
            const char* fText;
        } rec[] = {
            { {  10,  50 }, kLCDTextSizeLimit - 1,     1,  lcd_text },
            { { 160,  50 }, kLCDTextSizeLimit + 1,     1,  gray_text },
            { {  10, 100 }, kLCDTextSizeLimit / 2, 1.99f,  lcd_text },
            { { 160, 100 }, kLCDTextSizeLimit / 2, 2.01f,  gray_text },
        };

        for (size_t i = 0; i < SK_ARRAY_COUNT(rec); ++i) {
            const SkPoint loc = rec[i].fLoc;
            SkAutoCanvasRestore acr(canvas, true);

            paint.setTextSize(rec[i].fTextSize);
            ScaleAbout(canvas, rec[i].fScale, rec[i].fScale, loc.x(), loc.y());
            canvas->drawString(rec[i].fText, loc.x(), loc.y(), paint);
        }
    }

private:
    typedef skiagm::GM INHERITED;
};
DEF_GM( return new LcdTextGM; )
DEF_GM( return new LcdTextSizeGM; )

///////////////////////////////////////////////////////////////////////////////////////////////////

DEF_SIMPLE_GM(savelayer_lcdtext, canvas, 620, 260) {
    SkPaint paint;
    paint.setAntiAlias(true);
    paint.setLCDRenderText(true);
    paint.setTextSize(20);

    canvas->drawString("Hamburgefons", 30, 30, paint);

    const bool gPreserveLCDText[] = { false, true };

    canvas->translate(0, 20);
    for (auto preserve : gPreserveLCDText) {
        preserve ? canvas->saveLayerPreserveLCDTextRequests(nullptr, nullptr)
                 : canvas->saveLayer(nullptr, nullptr);
        if (preserve && !canvas->imageInfo().colorSpace()) {
            SkPaint noLCD = paint;
            noLCD.setLCDRenderText(false);
            canvas->drawString("LCD not supported", 30, 60, noLCD);
        } else {
            canvas->drawString("Hamburgefons", 30, 60, paint);
        }

        SkPaint p;
        p.setColor(0xFFCCCCCC);
        canvas->drawRect(SkRect::MakeLTRB(25, 70, 200, 100), p);
        canvas->drawString("Hamburgefons", 30, 90, paint);

        canvas->restore();
        canvas->translate(0, 80);
    }
}
<|endoftext|>


<commit_before><commit_msg>add simmetrix to threshold.cc<commit_after><|endoftext|>


<commit_before>#include <iostream>
#include <string>
#include <H5Cpp.h>
#include <H5File.h>
#include <H5DataSet.h>

#include "cp-neural.h"

typedef struct {
    int index;               // index of the current object
} cp_mnist_iter_info;

herr_t cp_mnist_get_all_groups(hid_t loc_id, const char *name, void *opdata);

H5::H5File *cp_Mnist_pfile;
map<string, MatrixN *> cpMnistData;

herr_t cp_mnist_get_all_groups(hid_t loc_id, const char *name, void *opdata)
{
    cp_mnist_iter_info *info=(cp_mnist_iter_info *)opdata;

    // Here you can do whatever with the name...
    cout << name << " ";

    // you can use this call to select just the groups
    // H5G_LINK    0  Object is a symbolic link.
    // H5G_GROUP   1  Object is a group.
    // H5G_DATASET 2  Object is a dataset.
    // H5G_TYPE    3  Object is a named datatype.
    int obj_type = H5Gget_objtype_by_idx(loc_id, info->index);
    //if(obj_type == H5G_GROUP)
	   //cout << "        is a group" << endl;
    if (obj_type == H5G_DATASET) {
        // cout << " is a dataset" << endl;
        H5::DataSet dataset = cp_Mnist_pfile->openDataSet(name);
        /*
         * Get filespace for rank and dimension
         */
        H5::DataSpace filespace = dataset.getSpace();
        /*
         * Get number of dimensions in the file dataspace
         */
        int rank = filespace.getSimpleExtentNdims();
        // cout << "rank: " << rank << endl;
        /*
         * Get and print the dimension sizes of the file dataspace
         */
        hsize_t dims[2];    // dataset dimensions
        rank = filespace.getSimpleExtentDims( dims );
        // cout << "dataset rank = " << rank << ", dimensions; ";
        // for (int j=0; j<rank; j++) {
        //    cout << (unsigned long)(dims[j]);
        //    if (j<rank-1) cout << " x ";
        //}
        //cout << endl;
        if (rank==1) dims[1]=1;
        cpMnistData[name] = new MatrixN(dims[0],dims[1]);
        MatrixN *pM = cpMnistData[name];
        /*
         * Define the memory space to read dataset.
         */
        H5::DataSpace mspace1(rank, dims);
        /*
         * Read dataset back and display.
         */
         auto dataClass = dataset.getTypeClass();

         if(dataClass == H5T_FLOAT)
         {
             auto floatType = dataset.getFloatType();

             size_t byteSize = floatType.getSize();

             if(byteSize == 4)
             {
                  float *pf = (float *)malloc(sizeof(float) * dims[0] * dims[1]);
                  if (pf) {
                      dataset.read(pf, H5::PredType::NATIVE_FLOAT, mspace1, filespace );
                      for (int y=0; y<dims[0]; y++) {
                          for (int x=0; x<dims[1]; x++) {
                              (*pM)(y,x)=(floatN)pf[y*dims[1]+x];
                          }
                      }
                      free(pf);
                  }
             }
             else if(byteSize == 8)
             {
                 double *pd = (double *)malloc(sizeof(double) * dims[0] * dims[1]);
                 if (pd) {
                     dataset.read(pd, H5::PredType::NATIVE_DOUBLE, mspace1, filespace );
                     for (int y=0; y<dims[0]; y++) {
                         for (int x=0; x<dims[1]; x++) {
                             (*pM)(y,x)=(floatN)pd[y*dims[1]+x];
                         }
                     }
                     free(pd);
                 }
             }
         } else if (dataClass == H5T_INTEGER) {
             int *pi = (int *)malloc(sizeof(int) * dims[0] * dims[1]);
             if (pi) {
                 dataset.read(pi, H5::PredType::NATIVE_INT, mspace1, filespace );
                 for (int y=0; y<dims[0]; y++) {
                     for (int x=0; x<dims[1]; x++) {
                         (*pM)(y,x)=(floatN)pi[y*dims[1]+x];
                     }
                 }
                 free(pi);
             }
         }
    }
    (info->index)++;
    return 0;
 }

bool  getMnistData(string filepath) {
    if (cpMnistData.size() > 0) {
        cout << "cpMnistData contains already elements, not reloading." << endl;
        return true;
    }
    H5::H5File fmn((H5std_string)filepath, H5F_ACC_RDONLY);
    cp_Mnist_pfile=&fmn;
    int nr=fmn.getNumObjs();
    //cout << nr << endl;
    cp_mnist_iter_info info;
    info.index=0;
    cout << "Reading: ";
    fmn.iterateElems("/", NULL, cp_mnist_get_all_groups, &info);
    cout << endl;
    return true;
}

 int main(int argc, char *argv[]) {
     if (argc!=2) {
         cout << "mnisttest <path-mnist.h5-file>" << endl;
         exit(-1);
     }
     getMnistData(argv[1]);
     for (auto it : cpMnistData) {
         cout << it.first << " " << shape(*(it.second)) << endl;
     }

     /*
     vector<int> ins{0,4,16,25,108, 256,777};
     for (auto in : ins) {
         cout << "-------------------------" << endl;
         cout << "Index: " << in << endl;
         for (int cy=0; cy<28; cy++) {
             for (int cx=0; cx<28; cx++) {
                 floatN pt=(*cpMnistData["x_test"])(in, cy*28+cx);
                 if (pt<0.5) cout << " ";
                 else cout << "*";
             }
             cout << endl;
         }
         cout << (*cpMnistData["t_test"])(in,0) << endl;
     }
     */
    TwoLayerNet tl({784,1024,10});
    MatrixN X=*(cpMnistData["x_train"]);
    MatrixN y=*(cpMnistData["t_train"]);
    MatrixN Xv=*(cpMnistData["x_valid"]);
    MatrixN yv=*(cpMnistData["t_valid"]);
    MatrixN Xt=*(cpMnistData["x_test"]);
    MatrixN yt=*(cpMnistData["t_test"]);
    cp_t_params<int> pi;
    cp_t_params<floatN> pf;
    pi["verbose"]=1;
    pi["epochs"]=10;
    pi["batch_size"]=100;
    pf["learning_rate"]=4.0e-1;
    tl.train(X, y, Xv, yv, "sdg", pi, pf);
    floatN final_err=tl.test(Xt, yt);
    cout << "Final error on test-set:" << final_err << endl;
    for (auto it : cpMnistData) {
         free(it.second);
     }
     return 0;
 }
<commit_msg>more epochs<commit_after>#include <iostream>
#include <string>
#include <H5Cpp.h>
#include <H5File.h>
#include <H5DataSet.h>

#include "cp-neural.h"

typedef struct {
    int index;               // index of the current object
} cp_mnist_iter_info;

herr_t cp_mnist_get_all_groups(hid_t loc_id, const char *name, void *opdata);

H5::H5File *cp_Mnist_pfile;
map<string, MatrixN *> cpMnistData;

herr_t cp_mnist_get_all_groups(hid_t loc_id, const char *name, void *opdata)
{
    cp_mnist_iter_info *info=(cp_mnist_iter_info *)opdata;

    // Here you can do whatever with the name...
    cout << name << " ";

    // you can use this call to select just the groups
    // H5G_LINK    0  Object is a symbolic link.
    // H5G_GROUP   1  Object is a group.
    // H5G_DATASET 2  Object is a dataset.
    // H5G_TYPE    3  Object is a named datatype.
    int obj_type = H5Gget_objtype_by_idx(loc_id, info->index);
    //if(obj_type == H5G_GROUP)
	   //cout << "        is a group" << endl;
    if (obj_type == H5G_DATASET) {
        // cout << " is a dataset" << endl;
        H5::DataSet dataset = cp_Mnist_pfile->openDataSet(name);
        /*
         * Get filespace for rank and dimension
         */
        H5::DataSpace filespace = dataset.getSpace();
        /*
         * Get number of dimensions in the file dataspace
         */
        int rank = filespace.getSimpleExtentNdims();
        // cout << "rank: " << rank << endl;
        /*
         * Get and print the dimension sizes of the file dataspace
         */
        hsize_t dims[2];    // dataset dimensions
        rank = filespace.getSimpleExtentDims( dims );
        // cout << "dataset rank = " << rank << ", dimensions; ";
        // for (int j=0; j<rank; j++) {
        //    cout << (unsigned long)(dims[j]);
        //    if (j<rank-1) cout << " x ";
        //}
        //cout << endl;
        if (rank==1) dims[1]=1;
        cpMnistData[name] = new MatrixN(dims[0],dims[1]);
        MatrixN *pM = cpMnistData[name];
        /*
         * Define the memory space to read dataset.
         */
        H5::DataSpace mspace1(rank, dims);
        /*
         * Read dataset back and display.
         */
         auto dataClass = dataset.getTypeClass();

         if(dataClass == H5T_FLOAT)
         {
             auto floatType = dataset.getFloatType();

             size_t byteSize = floatType.getSize();

             if(byteSize == 4)
             {
                  float *pf = (float *)malloc(sizeof(float) * dims[0] * dims[1]);
                  if (pf) {
                      dataset.read(pf, H5::PredType::NATIVE_FLOAT, mspace1, filespace );
                      for (int y=0; y<dims[0]; y++) {
                          for (int x=0; x<dims[1]; x++) {
                              (*pM)(y,x)=(floatN)pf[y*dims[1]+x];
                          }
                      }
                      free(pf);
                  }
             }
             else if(byteSize == 8)
             {
                 double *pd = (double *)malloc(sizeof(double) * dims[0] * dims[1]);
                 if (pd) {
                     dataset.read(pd, H5::PredType::NATIVE_DOUBLE, mspace1, filespace );
                     for (int y=0; y<dims[0]; y++) {
                         for (int x=0; x<dims[1]; x++) {
                             (*pM)(y,x)=(floatN)pd[y*dims[1]+x];
                         }
                     }
                     free(pd);
                 }
             }
         } else if (dataClass == H5T_INTEGER) {
             int *pi = (int *)malloc(sizeof(int) * dims[0] * dims[1]);
             if (pi) {
                 dataset.read(pi, H5::PredType::NATIVE_INT, mspace1, filespace );
                 for (int y=0; y<dims[0]; y++) {
                     for (int x=0; x<dims[1]; x++) {
                         (*pM)(y,x)=(floatN)pi[y*dims[1]+x];
                     }
                 }
                 free(pi);
             }
         }
    }
    (info->index)++;
    return 0;
 }

bool  getMnistData(string filepath) {
    if (cpMnistData.size() > 0) {
        cout << "cpMnistData contains already elements, not reloading." << endl;
        return true;
    }
    H5::H5File fmn((H5std_string)filepath, H5F_ACC_RDONLY);
    cp_Mnist_pfile=&fmn;
    int nr=fmn.getNumObjs();
    //cout << nr << endl;
    cp_mnist_iter_info info;
    info.index=0;
    cout << "Reading: ";
    fmn.iterateElems("/", NULL, cp_mnist_get_all_groups, &info);
    cout << endl;
    return true;
}

 int main(int argc, char *argv[]) {
     if (argc!=2) {
         cout << "mnisttest <path-mnist.h5-file>" << endl;
         exit(-1);
     }
     getMnistData(argv[1]);
     for (auto it : cpMnistData) {
         cout << it.first << " " << shape(*(it.second)) << endl;
     }

     /*
     vector<int> ins{0,4,16,25,108, 256,777};
     for (auto in : ins) {
         cout << "-------------------------" << endl;
         cout << "Index: " << in << endl;
         for (int cy=0; cy<28; cy++) {
             for (int cx=0; cx<28; cx++) {
                 floatN pt=(*cpMnistData["x_test"])(in, cy*28+cx);
                 if (pt<0.5) cout << " ";
                 else cout << "*";
             }
             cout << endl;
         }
         cout << (*cpMnistData["t_test"])(in,0) << endl;
     }
     */
    TwoLayerNet tl({784,1024,10});
    MatrixN X=*(cpMnistData["x_train"]);
    MatrixN y=*(cpMnistData["t_train"]);
    MatrixN Xv=*(cpMnistData["x_valid"]);
    MatrixN yv=*(cpMnistData["t_valid"]);
    MatrixN Xt=*(cpMnistData["x_test"]);
    MatrixN yt=*(cpMnistData["t_test"]);
    cp_t_params<int> pi;
    cp_t_params<floatN> pf;
    pi["verbose"]=1;
    pi["epochs"]=100;
    pi["batch_size"]=100;
    pf["learning_rate"]=1.0e-2;
    tl.train(X, y, Xv, yv, "sdg", pi, pf);
    floatN final_err=tl.test(Xt, yt);
    cout << "Final error on test-set:" << final_err << endl;
    for (auto it : cpMnistData) {
         free(it.second);
     }
     return 0;
 }
<|endoftext|>


<commit_before>#include <vector>
#include <iostream>
#include <algorithm>

int main() {
    std::string s; std::cin >> s;
    int k; std::cin >> k;

    for (auto& c : s) {
        if (c >= 'a' && c <= 'z') {

            c = ((c - 'a' + k) % 26) + 'a';

        } else if (c >= 'A' && c <= 'Z') {

            c = ((c - 'A' + k) % 26) + 'A';

        }
    }

    std::cout << s << std::endl;
    return 0;
}<commit_msg>change cin to getline cpp<commit_after>#include <vector>
#include <string>
#include <iostream>

int main() {
    std::string s; std::getline(std::cin, s);
    int k; std::cin >> k;

    for (auto& c : s) {
        if (c >= 'a' && c <= 'z') {

            c = ((c - 'a' + k) % 26) + 'a';

        } else if (c >= 'A' && c <= 'Z') {

            c = ((c - 'A' + k) % 26) + 'A';

        }
    }

    std::cout << s << std::endl;
    return 0;
}<|endoftext|>


<commit_before>/** \brief Utility for counting references to GND numbers.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2017 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <stdexcept>
#include <unordered_map>
#include <cstdlib>
#include "FileUtil.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "StringUtil.h"
#include "util.h"

namespace {

void Usage() __attribute__((noreturn));

void Usage() {
    std::cerr << "Usage: " << ::progname << " [--control-number-list=list_filename] gnd_number_list marc_data counts\n"
              << "       If a control-number-list filename has been specifiec only references of records\n"
              << "       matching in entries in that file will be counted.\n\n";
    std::exit(EXIT_FAILURE);
}

void LoadGNDNumbers(File * const input, std::unordered_map<std::string, unsigned> * const gnd_numbers_and_counts) {
    while (not input->eof()) {
        std::string line;
        if (input->getline(&line) > 0)
            gnd_numbers_and_counts->emplace(line, 0);
    }

    std::cout << "Loaded " << gnd_numbers_and_counts->size() << " GND numbers.\n";
}

const std::vector<std::string> GND_REFERENCE_FIELDS{ "100", "600", "689", "700" };

void ProcessRecords(MarcReader * const marc_reader,
                    std::unordered_map<std::string, unsigned> * const gnd_numbers_and_counts)
{
    while (const MarcRecord record = marc_reader->read()) {
        for (const auto &gnd_reference_field : GND_REFERENCE_FIELDS) {
            const std::string field_contents(record.getFieldData(gnd_reference_field));
            if (field_contents.empty())
                continue;

            const Subfields subfields(field_contents);
            const auto begin_end(subfields.getIterators('0'));
            for (auto subfield0(begin_end.first); subfield0 != begin_end.second; ++subfield0) {
                if (not StringUtil::StartsWith(subfield0->value_, "(DE-588)"))
                    continue;

                const auto gnd_number_and_count(gnd_numbers_and_counts->find(subfield0->value_));
                if (gnd_number_and_count != gnd_numbers_and_counts->end())
                    ++gnd_number_and_count->second;
            }
        }
    }
}

void WriteCounts(const std::unordered_map<std::string, unsigned> &gnd_numbers_and_counts, File * const output) {
    for (const auto &gnd_number_and_count : gnd_numbers_and_counts) {
        if (gnd_number_and_count.second > 0)
            (*output) << gnd_number_and_count.first.substr(8) << '|' << gnd_number_and_count.second << '\n';
    }
}

void LoadFilterSet(const std::string &input_filename, std::unordered_set<std::string> * const filter_set) {
    std::unique_ptr<File> input(FileUtil::OpenInputFileOrDie(input_filename));
    while (not input->eof()) {
        std::string line;
        input->getline(&line);
        StringUtil::TrimWhite(&line);
        if (likely(not line.empty()))
            filter_set->emplace(line);
    }

    logger->info("loaded " + std::to_string(filter_set->size()) + " control numbers from \"" + input_filename
                 + "\".");
}

} // unnamed namespace

int main(int argc, char *argv[]) {
    ::progname = argv[0];

    if (argc != 4 and argc != 5)
        Usage();

    std::unordered_set<std::string> filter_set;
    if (argc == 5) {
        if (not StringUtil::StartsWith(argv[1], "--control-number-list="))
            Usage();
        LoadFilterSet(argv[1] + __builtin_strlen("--control-number-list="), &filter_set);
        --argc, ++argv;
    }

    try {
        std::unique_ptr<File> gnd_numbers_and_counts_file(FileUtil::OpenInputFileOrDie(argv[1]));
        std::unordered_map<std::string, unsigned> gnd_numbers_and_counts;
        LoadGNDNumbers(gnd_numbers_and_counts_file.get(), &gnd_numbers_and_counts);

        std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(argv[2]));
        ProcessRecords(marc_reader.get(), &gnd_numbers_and_counts);

        std::unique_ptr<File> counts_file(FileUtil::OpenOutputFileOrDie(argv[3]));
        WriteCounts(gnd_numbers_and_counts, counts_file.get());
    } catch (const std::exception &e) {
        logger->error("Caught exception: " + std::string(e.what()));
    }
}
<commit_msg>Actully do the filtering.<commit_after>/** \brief Utility for counting references to GND numbers.
 *  \author Dr. Johannes Ruscheinski (johannes.ruscheinski@uni-tuebingen.de)
 *
 *  \copyright 2017 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <iostream>
#include <stdexcept>
#include <unordered_map>
#include <cstdlib>
#include "FileUtil.h"
#include "MarcReader.h"
#include "MarcRecord.h"
#include "StringUtil.h"
#include "util.h"

namespace {

void Usage() __attribute__((noreturn));

void Usage() {
    std::cerr << "Usage: " << ::progname << " [--control-number-list=list_filename] gnd_number_list marc_data counts\n"
              << "       If a control-number-list filename has been specifiec only references of records\n"
              << "       matching in entries in that file will be counted.\n\n";
    std::exit(EXIT_FAILURE);
}

void LoadGNDNumbers(File * const input, std::unordered_map<std::string, unsigned> * const gnd_numbers_and_counts) {
    while (not input->eof()) {
        std::string line;
        if (input->getline(&line) > 0)
            gnd_numbers_and_counts->emplace(line, 0);
    }

    std::cout << "Loaded " << gnd_numbers_and_counts->size() << " GND numbers.\n";
}

const std::vector<std::string> GND_REFERENCE_FIELDS{ "100", "600", "689", "700" };

void ProcessRecords(MarcReader * const marc_reader, const std::unordered_set<std::string> &filter_set,
                    std::unordered_map<std::string, unsigned> * const gnd_numbers_and_counts)
{
    while (const MarcRecord record = marc_reader->read()) {
        if (not filter_set.empty()) {
            if (filter_set.find(record.getControlNumber()) == filter_set.cend())
                continue;
        }

        for (const auto &gnd_reference_field : GND_REFERENCE_FIELDS) {
            const std::string field_contents(record.getFieldData(gnd_reference_field));
            if (field_contents.empty())
                continue;

            const Subfields subfields(field_contents);
            const auto begin_end(subfields.getIterators('0'));
            for (auto subfield0(begin_end.first); subfield0 != begin_end.second; ++subfield0) {
                if (not StringUtil::StartsWith(subfield0->value_, "(DE-588)"))
                    continue;

                const auto gnd_number_and_count(gnd_numbers_and_counts->find(subfield0->value_));
                if (gnd_number_and_count != gnd_numbers_and_counts->end())
                    ++gnd_number_and_count->second;
            }
        }
    }
}

void WriteCounts(const std::unordered_map<std::string, unsigned> &gnd_numbers_and_counts, File * const output) {
    for (const auto &gnd_number_and_count : gnd_numbers_and_counts) {
        if (gnd_number_and_count.second > 0)
            (*output) << gnd_number_and_count.first.substr(8) << '|' << gnd_number_and_count.second << '\n';
    }
}

void LoadFilterSet(const std::string &input_filename, std::unordered_set<std::string> * const filter_set) {
    std::unique_ptr<File> input(FileUtil::OpenInputFileOrDie(input_filename));
    while (not input->eof()) {
        std::string line;
        input->getline(&line);
        StringUtil::TrimWhite(&line);
        if (likely(not line.empty()))
            filter_set->emplace(line);
    }

    logger->info("loaded " + std::to_string(filter_set->size()) + " control numbers from \"" + input_filename
                 + "\".");
}

} // unnamed namespace

int main(int argc, char *argv[]) {
    ::progname = argv[0];

    if (argc != 4 and argc != 5)
        Usage();

    std::unordered_set<std::string> filter_set;
    if (argc == 5) {
        if (not StringUtil::StartsWith(argv[1], "--control-number-list="))
            Usage();
        LoadFilterSet(argv[1] + __builtin_strlen("--control-number-list="), &filter_set);
        --argc, ++argv;
    }

    try {
        std::unique_ptr<File> gnd_numbers_and_counts_file(FileUtil::OpenInputFileOrDie(argv[1]));
        std::unordered_map<std::string, unsigned> gnd_numbers_and_counts;
        LoadGNDNumbers(gnd_numbers_and_counts_file.get(), &gnd_numbers_and_counts);

        std::unique_ptr<MarcReader> marc_reader(MarcReader::Factory(argv[2]));
        ProcessRecords(marc_reader.get(), filter_set, &gnd_numbers_and_counts);

        std::unique_ptr<File> counts_file(FileUtil::OpenOutputFileOrDie(argv[3]));
        WriteCounts(gnd_numbers_and_counts, counts_file.get());
    } catch (const std::exception &e) {
        logger->error("Caught exception: " + std::string(e.what()));
    }
}
<|endoftext|>