<commit_before>#include <async.h>
#include <aios.h>
#include <dbfe.h>

enum {
    MODE_ENV = 1,
    MODE_OLD = 2
} modes;

int
main (int argc, char *argv[])
{
  if (argc != 3)
    fatal << "usage: dbdump <-e|-o> <dbfile>\n";

  int mode = MODE_ENV;
  if (strcmp(argv[1], "-o") == 0)
    mode = MODE_OLD;

  int r;
  DB *db = NULL;
  DB_ENV* dbe = NULL;
  char cpath[MAXPATHLEN];

  if (mode == MODE_ENV) {

    r = db_env_create (&dbe, 0);
    assert (!r);
    r = chdir (argv[2]);
    if (r == -1) fatal << "couldn't chdir to " << argv[2] << "\n";

    getcwd (cpath, MAXPATHLEN);
    aout << "opening db in: " << cpath << "\n";
    r = dbe->set_data_dir (dbe, cpath);
    assert (!r);
    r = dbe->set_lg_dir (dbe, cpath);
    assert (!r);
    r = dbe->set_tmp_dir (dbe, cpath);
    assert (!r);

    dbe->set_errfile (dbe, stdout);
    r = dbe->open (dbe, NULL,
		   DB_JOINENV,
		   //		   DB_CREATE| DB_INIT_MPOOL | DB_INIT_LOCK |
		   // DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER | DB_JOINENV,
		   0);
    assert (!r);

  }

  r = db_create(&db, dbe, 0);
  assert (r==0);

  if (mode == MODE_OLD) {
#if ((DB_VERSION_MAJOR < 4) || ((DB_VERSION_MAJOR == 4) && (DB_VERSION_MINOR < 1)))
    r = db->open(db, (const char *)argv[2], NULL, DB_BTREE, DB_RDONLY, 0664);
#else
    r = db->open(db, NULL, (const char *)argv[2], NULL,
		 DB_BTREE, DB_RDONLY, 0664);
#endif

} else {
#if ((DB_VERSION_MAJOR < 4) || ((DB_VERSION_MAJOR == 4) && (DB_VERSION_MINOR < 1)))
  r = db->open(db, "db", NULL, DB_BTREE, DB_RDONLY, 0664);
#else
  r = db->open(db, NULL, "db", NULL, DB_BTREE, DB_RDONLY, 0664);
#endif

}

  assert (r == 0);

  DBC *cursor;
  r = db->cursor(db, NULL, &cursor, 0);
  assert (r==0);

  DBT key, data;
  unsigned totalsz = 0;
  unsigned keys = 0;

  for (int i = 0; ; i++) {
    bzero (&key, sizeof (key));
    bzero (&data, sizeof (data));
    //data.flags = DB_DBT_PARTIAL;
    int err = cursor->c_get(cursor, &key, &data,
			    ((i==0) ? DB_FIRST : DB_NEXT));
    if (err == DB_NOTFOUND) {
      aout << "EOF.\n";
      break;
    } else if (err) {
      fatal << "err: " << err << " " << strerror (err) << "\n";
    }

    bzero (&data, sizeof (data));
    err = db->get (db, NULL, &key, &data, 0);
    if (err)
      warn << "lookup err: " << err << " " << strerror (err) << "\n";

    aout << "key[" << i << "] " << hexdump (key.data, key.size)
	 << " " << data.size << "\n";
    keys++;

    totalsz += data.size;

    err_flush ();
  }

  db->close (db, 0);
  dbe->close (dbe, 0);
  aout << "total keys: " << keys << "\n";
  aout << "total bytes: " << totalsz << "\n";
}

<commit_msg>Flush correctly.<commit_after>#include <async.h>
#include <aios.h>
#include <dbfe.h>

enum {
    MODE_ENV = 1,
    MODE_OLD = 2
} modes;

int
main (int argc, char *argv[])
{
  if (argc != 3)
    fatal << "usage: dbdump <-e|-o> <dbfile>\n";

  int mode = MODE_ENV;
  if (strcmp(argv[1], "-o") == 0)
    mode = MODE_OLD;

  int r;
  DB *db = NULL;
  DB_ENV* dbe = NULL;
  char cpath[MAXPATHLEN];

  if (mode == MODE_ENV) {

    r = db_env_create (&dbe, 0);
    assert (!r);
    r = chdir (argv[2]);
    if (r == -1) fatal << "couldn't chdir to " << argv[2] << "\n";

    getcwd (cpath, MAXPATHLEN);
    aout << "opening db in: " << cpath << "\n";
    r = dbe->set_data_dir (dbe, cpath);
    assert (!r);
    r = dbe->set_lg_dir (dbe, cpath);
    assert (!r);
    r = dbe->set_tmp_dir (dbe, cpath);
    assert (!r);

    dbe->set_errfile (dbe, stdout);
    r = dbe->open (dbe, NULL,
		   DB_JOINENV,
		   //		   DB_CREATE| DB_INIT_MPOOL | DB_INIT_LOCK |
		   // DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER | DB_JOINENV,
		   0);
    assert (!r);

  }

  r = db_create(&db, dbe, 0);
  assert (r==0);

  if (mode == MODE_OLD) {
#if ((DB_VERSION_MAJOR < 4) || ((DB_VERSION_MAJOR == 4) && (DB_VERSION_MINOR < 1)))
    r = db->open(db, (const char *)argv[2], NULL, DB_BTREE, DB_RDONLY, 0664);
#else
    r = db->open(db, NULL, (const char *)argv[2], NULL,
		 DB_BTREE, DB_RDONLY, 0664);
#endif

} else {
#if ((DB_VERSION_MAJOR < 4) || ((DB_VERSION_MAJOR == 4) && (DB_VERSION_MINOR < 1)))
  r = db->open(db, "db", NULL, DB_BTREE, DB_RDONLY, 0664);
#else
  r = db->open(db, NULL, "db", NULL, DB_BTREE, DB_RDONLY, 0664);
#endif

}

  assert (r == 0);

  DBC *cursor;
  r = db->cursor(db, NULL, &cursor, 0);
  assert (r==0);

  DBT key, data;
  unsigned totalsz = 0;
  unsigned keys = 0;

  for (int i = 0; ; i++) {
    bzero (&key, sizeof (key));
    bzero (&data, sizeof (data));
    //data.flags = DB_DBT_PARTIAL;
    int err = cursor->c_get(cursor, &key, &data,
			    ((i==0) ? DB_FIRST : DB_NEXT));
    if (err == DB_NOTFOUND) {
      aout << "EOF.\n";
      break;
    } else if (err) {
      fatal << "err: " << err << " " << strerror (err) << "\n";
    }

    bzero (&data, sizeof (data));
    err = db->get (db, NULL, &key, &data, 0);
    if (err)
      warn << "lookup err: " << err << " " << strerror (err) << "\n";

    aout << "key[" << i << "] " << hexdump (key.data, key.size)
	 << " " << data.size << "\n";
    keys++;

    totalsz += data.size;

    aout->flush ();
  }

  db->close (db, 0);
  dbe->close (dbe, 0);
  aout << "total keys: " << keys << "\n";
  aout << "total bytes: " << totalsz << "\n";
}

<|endoftext|>


<commit_before>// (C) 2013 Cybozu.

#include "config.hpp"
#include "gc.hpp"
#include "replication.hpp"
#include "stats.hpp"

#include <algorithm>
#include <cstdlib>

namespace yrmcds {

void gc_thread::run() {
    if( ! cybozu::has_ip_address(g_config.vip()) ) {
        cybozu::logger::error() << "VIP has been lost.  Exiting quickly...";
        std::quick_exit(2);
    }

    using namespace std::chrono;
    auto t1 = steady_clock::now();
    gc();
    if( ! m_new_slaves.empty() )
        cybozu::logger::info() << "Initial replication completed for "
                               << m_new_slaves.size() << " new slave(s).";
    g_stats.objects.store(m_objects, std::memory_order_relaxed);
    g_stats.objects_under_1k.store(m_objects_under_1k, std::memory_order_relaxed);
    g_stats.objects_under_4k.store(m_objects_under_4k, std::memory_order_relaxed);
    g_stats.objects_under_16k.store(m_objects_under_16k, std::memory_order_relaxed);
    g_stats.objects_under_64k.store(m_objects_under_64k, std::memory_order_relaxed);
    g_stats.objects_under_256k.store(m_objects_under_256k, std::memory_order_relaxed);
    g_stats.objects_under_1m.store(m_objects_under_1m, std::memory_order_relaxed);
    g_stats.objects_under_4m.store(m_objects_under_4m, std::memory_order_relaxed);
    g_stats.objects_huge.store(m_objects_huge, std::memory_order_relaxed);
    g_stats.used_memory.store(m_used_memory, std::memory_order_relaxed);
    g_stats.conflicts.store(m_conflicts, std::memory_order_relaxed);
    g_stats.gc_count.fetch_add(1, std::memory_order_relaxed);
    g_stats.oldest_age.store(m_oldest_age, std::memory_order_relaxed);
    g_stats.largest_object_size.store(m_largest_object_size, std::memory_order_relaxed);
    g_stats.last_expirations.store(m_last_expirations, std::memory_order_relaxed);
    g_stats.last_evictions.store(m_last_evictions, std::memory_order_relaxed);
    g_stats.total_evictions.fetch_add(m_last_evictions, std::memory_order_relaxed);

    auto t2 = steady_clock::now();
    std::uint64_t us = static_cast<std::uint64_t>(
        duration_cast<microseconds>(t2-t1).count() );
    g_stats.last_gc_elapsed.store(us, std::memory_order_relaxed);
    g_stats.total_gc_elapsed.fetch_add(us, std::memory_order_relaxed);
    cybozu::logger::debug() << "GC end: elapsed=" << us
                            << "us, expired=" << m_last_expirations
                            << ", evicted=" << m_last_evictions
                            << ", survived=" << m_objects;
}

void gc_thread::gc() {
    std::time_t t = g_stats.flush_time.load();
    bool flush = (t != 0) && (std::time(nullptr) >= t);
    unsigned int evict_age = 0;
    if( (! flush) &&
        (g_stats.used_memory.load(std::memory_order_relaxed) >
         g_config.memory_limit()) ) {
        unsigned int oldest_age =
            g_stats.oldest_age.load(std::memory_order_relaxed);
        unsigned int one_hour = 3600U / g_config.gc_interval() + 1;
        if( oldest_age < (one_hour * 2) ) {
            evict_age = std::max(1U, oldest_age / 2);
        } else {
            evict_age = oldest_age - one_hour;
        }
        cybozu::logger::warning() << "Evicting object of "
                                  << evict_age << " gc old";
    }

    auto pred =
        [this,flush,evict_age](const cybozu::hash_key& k, object& obj) ->bool {
        if( flush && (! obj.locked()) ) {
            if( ! m_slaves.empty() )
                repl_delete(m_slaves, k);
            return true;
        }
        if( evict_age > 0 && obj.age() >= evict_age && (! obj.locked()) ) {
            ++ m_last_evictions;
            if( ! m_slaves.empty() )
                repl_delete(m_slaves, k);
            return true;
        }
        if( obj.expired() ) {
            ++ m_last_expirations;
            if( ! m_slaves.empty() )
                repl_delete(m_slaves, k);
            return true;
        }

        obj.survive();
        if( ++m_objects_in_bucket == 2 )
            ++ m_conflicts;
        ++ m_objects;
        std::size_t size = obj.size();
        if( size < 1024 ) {
            ++ m_objects_under_1k;
        } else if( size < 4096 ) {
            ++ m_objects_under_4k;
        } else if( size < (16 <<10) ) {
            ++ m_objects_under_16k;
        } else if( size < (64 <<10) ) {
            ++ m_objects_under_64k;
        } else if( size < (256 <<10) ) {
            ++ m_objects_under_256k;
        } else if( size < (1 <<20) ) {
            ++ m_objects_under_1m;
        } else if( size < (4 <<20) ) {
            ++ m_objects_under_4m;
        } else {
            ++ m_objects_huge;
        }
        m_used_memory += sizeof(cybozu::hash_key) + k.length() + sizeof(object);
        if( g_config.heap_data_limit() >= obj.size() )
            m_used_memory += obj.size();
        m_oldest_age = std::max(m_oldest_age, obj.age());
        m_largest_object_size = std::max(m_largest_object_size, obj.size());
        if( ! m_new_slaves.empty() )
            repl_object(m_new_slaves, k, obj, false);
        return false;
    };

    for( auto it = m_hash.begin(); it != m_hash.end(); ++it ) {
        m_objects_in_bucket = 0;
        it->gc(pred);
    }

    g_stats.flush_time.store(0);
}

} // namespace yrmcds
<commit_msg>fixes #4, a bug preventing flush taking place.<commit_after>// (C) 2013 Cybozu.

#include "config.hpp"
#include "gc.hpp"
#include "replication.hpp"
#include "stats.hpp"

#include <algorithm>
#include <cstdlib>

namespace yrmcds {

void gc_thread::run() {
    if( ! cybozu::has_ip_address(g_config.vip()) ) {
        cybozu::logger::error() << "VIP has been lost.  Exiting quickly...";
        std::quick_exit(2);
    }

    using namespace std::chrono;
    auto t1 = steady_clock::now();
    gc();
    if( ! m_new_slaves.empty() )
        cybozu::logger::info() << "Initial replication completed for "
                               << m_new_slaves.size() << " new slave(s).";
    g_stats.objects.store(m_objects, std::memory_order_relaxed);
    g_stats.objects_under_1k.store(m_objects_under_1k, std::memory_order_relaxed);
    g_stats.objects_under_4k.store(m_objects_under_4k, std::memory_order_relaxed);
    g_stats.objects_under_16k.store(m_objects_under_16k, std::memory_order_relaxed);
    g_stats.objects_under_64k.store(m_objects_under_64k, std::memory_order_relaxed);
    g_stats.objects_under_256k.store(m_objects_under_256k, std::memory_order_relaxed);
    g_stats.objects_under_1m.store(m_objects_under_1m, std::memory_order_relaxed);
    g_stats.objects_under_4m.store(m_objects_under_4m, std::memory_order_relaxed);
    g_stats.objects_huge.store(m_objects_huge, std::memory_order_relaxed);
    g_stats.used_memory.store(m_used_memory, std::memory_order_relaxed);
    g_stats.conflicts.store(m_conflicts, std::memory_order_relaxed);
    g_stats.gc_count.fetch_add(1, std::memory_order_relaxed);
    g_stats.oldest_age.store(m_oldest_age, std::memory_order_relaxed);
    g_stats.largest_object_size.store(m_largest_object_size, std::memory_order_relaxed);
    g_stats.last_expirations.store(m_last_expirations, std::memory_order_relaxed);
    g_stats.last_evictions.store(m_last_evictions, std::memory_order_relaxed);
    g_stats.total_evictions.fetch_add(m_last_evictions, std::memory_order_relaxed);

    auto t2 = steady_clock::now();
    std::uint64_t us = static_cast<std::uint64_t>(
        duration_cast<microseconds>(t2-t1).count() );
    g_stats.last_gc_elapsed.store(us, std::memory_order_relaxed);
    g_stats.total_gc_elapsed.fetch_add(us, std::memory_order_relaxed);
    cybozu::logger::debug() << "GC end: elapsed=" << us
                            << "us, expired=" << m_last_expirations
                            << ", evicted=" << m_last_evictions
                            << ", survived=" << m_objects;
}

void gc_thread::gc() {
    std::time_t t = g_stats.flush_time.load();
    bool flush = (t != 0) && (std::time(nullptr) >= t);
    unsigned int evict_age = 0;
    if( (! flush) &&
        (g_stats.used_memory.load(std::memory_order_relaxed) >
         g_config.memory_limit()) ) {
        unsigned int oldest_age =
            g_stats.oldest_age.load(std::memory_order_relaxed);
        unsigned int one_hour = 3600U / g_config.gc_interval() + 1;
        if( oldest_age < (one_hour * 2) ) {
            evict_age = std::max(1U, oldest_age / 2);
        } else {
            evict_age = oldest_age - one_hour;
        }
        cybozu::logger::warning() << "Evicting object of "
                                  << evict_age << " gc old";
    }

    auto pred =
        [this,flush,evict_age](const cybozu::hash_key& k, object& obj) ->bool {
        if( flush && (! obj.locked()) ) {
            if( ! m_slaves.empty() )
                repl_delete(m_slaves, k);
            return true;
        }
        if( evict_age > 0 && obj.age() >= evict_age && (! obj.locked()) ) {
            ++ m_last_evictions;
            if( ! m_slaves.empty() )
                repl_delete(m_slaves, k);
            return true;
        }
        if( obj.expired() ) {
            ++ m_last_expirations;
            if( ! m_slaves.empty() )
                repl_delete(m_slaves, k);
            return true;
        }

        obj.survive();
        if( ++m_objects_in_bucket == 2 )
            ++ m_conflicts;
        ++ m_objects;
        std::size_t size = obj.size();
        if( size < 1024 ) {
            ++ m_objects_under_1k;
        } else if( size < 4096 ) {
            ++ m_objects_under_4k;
        } else if( size < (16 <<10) ) {
            ++ m_objects_under_16k;
        } else if( size < (64 <<10) ) {
            ++ m_objects_under_64k;
        } else if( size < (256 <<10) ) {
            ++ m_objects_under_256k;
        } else if( size < (1 <<20) ) {
            ++ m_objects_under_1m;
        } else if( size < (4 <<20) ) {
            ++ m_objects_under_4m;
        } else {
            ++ m_objects_huge;
        }
        m_used_memory += sizeof(cybozu::hash_key) + k.length() + sizeof(object);
        if( g_config.heap_data_limit() >= obj.size() )
            m_used_memory += obj.size();
        m_oldest_age = std::max(m_oldest_age, obj.age());
        m_largest_object_size = std::max(m_largest_object_size, obj.size());
        if( ! m_new_slaves.empty() )
            repl_object(m_new_slaves, k, obj, false);
        return false;
    };

    for( auto it = m_hash.begin(); it != m_hash.end(); ++it ) {
        m_objects_in_bucket = 0;
        it->gc(pred);
    }

    if( flush )
        g_stats.flush_time.store(0);
}

} // namespace yrmcds
<|endoftext|>


<commit_before>//===- fpcmp.cpp - A fuzzy "cmp" that permits floating point noise --------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file was developed by the LLVM research group and is distributed under
// the University of Illinois Open Source License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// fpcmp is a tool that basically works like the 'cmp' tool, except that it can
// tolerate errors due to floating point noise, with the -r and -a options.
//
//===----------------------------------------------------------------------===//

#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileUtilities.h"
#include <iostream>
#include <cmath>

using namespace llvm;

namespace {
  cl::opt<std::string>
  File1(cl::Positional, cl::desc("<input file #1>"), cl::Required);
  cl::opt<std::string>
  File2(cl::Positional, cl::desc("<input file #2>"), cl::Required);

  cl::opt<double>
  RelTolerance("r", cl::desc("Relative error tolerated"), cl::init(0));
  cl::opt<double>
  AbsTolerance("a", cl::desc("Absolute error tolerated"), cl::init(0));
}

int main(int argc, char **argv) {
  cl::ParseCommandLineOptions(argc, argv);

  std::string ErrorMsg;
  int DF = DiffFilesWithTolerance(File1, File2, AbsTolerance, RelTolerance,
                                  &ErrorMsg);
  if (!ErrorMsg.empty())
    std::cerr << argv[0] << ": " << ErrorMsg << "\n";
  return DF;
}

<commit_msg>Drop dead #include<commit_after>//===- fpcmp.cpp - A fuzzy "cmp" that permits floating point noise --------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file was developed by the LLVM research group and is distributed under
// the University of Illinois Open Source License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// fpcmp is a tool that basically works like the 'cmp' tool, except that it can
// tolerate errors due to floating point noise, with the -r and -a options.
//
//===----------------------------------------------------------------------===//

#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileUtilities.h"
#include <iostream>
using namespace llvm;

namespace {
  cl::opt<std::string>
  File1(cl::Positional, cl::desc("<input file #1>"), cl::Required);
  cl::opt<std::string>
  File2(cl::Positional, cl::desc("<input file #2>"), cl::Required);

  cl::opt<double>
  RelTolerance("r", cl::desc("Relative error tolerated"), cl::init(0));
  cl::opt<double>
  AbsTolerance("a", cl::desc("Absolute error tolerated"), cl::init(0));
}

int main(int argc, char **argv) {
  cl::ParseCommandLineOptions(argc, argv);

  std::string ErrorMsg;
  int DF = DiffFilesWithTolerance(File1, File2, AbsTolerance, RelTolerance,
                                  &ErrorMsg);
  if (!ErrorMsg.empty())
    std::cerr << argv[0] << ": " << ErrorMsg << "\n";
  return DF;
}

<|endoftext|>


<commit_before>/** @file
@brief Save data from database to RAM (or hard drive in future version).
@author Yi Zhao
*/
#include "main.h"

/**
@brief Save data from MYSQL_RES to RAM.
@see localrow
@remark Data saved in linked list.\n
Usage:\n
MYSQL_RES *result=mysql_store_result(mysql_conn);\n
localrow *localresult;\n
int res=make_mysqlres_local(&localresult,result);
*/
int make_mysqlres_local(localrow **localresult,MYSQL_RES *result_t){
    int count=0;
    mysql_data_seek(result_t,0);
    localrow **p=localresult;
    MYSQL_ROW sql_row;
    while((sql_row=mysql_fetch_row(result_t))){
        *p=(localrow*)malloc(sizeof(localrow));
        for(int i=0;i<8;i++){
            strcpy((*p)->row[i],sql_row[i]);
        }
        p=&((*p)->next);
        count++;
    }
    *p=NULL;
    return count;
}
/**
@brief Free data on RAM.
@see localrow
*/
void free_mysqlres_local(localrow *localresult){
    while(localresult){
        localrow *p=localresult;
        if(p->next) printf(">>%d\n",p);
        localresult=p->next;
        free(p);
    }
}
/**
@brief Get number of saved sgRNA-Info on RAM.
@see localrow
*/
int localres_count(localrow *lr){
    int i=0;
    while(lr){
        i++;
        lr=lr->next;
        if(i>1000000) return -1;
    }
    return i;
}
<commit_msg>show less and less<commit_after>/** @file
@brief Save data from database to RAM (or hard drive in future version).
@author Yi Zhao
*/
#include "main.h"

/**
@brief Save data from MYSQL_RES to RAM.
@see localrow
@remark Data saved in linked list.\n
Usage:\n
MYSQL_RES *result=mysql_store_result(mysql_conn);\n
localrow *localresult;\n
int res=make_mysqlres_local(&localresult,result);
*/
int make_mysqlres_local(localrow **localresult,MYSQL_RES *result_t){
    int count=0;
    mysql_data_seek(result_t,0);
    localrow **p=localresult;
    MYSQL_ROW sql_row;
    while((sql_row=mysql_fetch_row(result_t))){
        *p=(localrow*)malloc(sizeof(localrow));
        for(int i=0;i<8;i++){
            strcpy((*p)->row[i],sql_row[i]);
        }
        p=&((*p)->next);
        count++;
    }
    *p=NULL;
    return count;
}
/**
@brief Free data on RAM.
@see localrow
*/
void free_mysqlres_local(localrow *localresult){
    while(localresult){
        localrow *p=localresult;
        if(p->next==NULL) printf(">>%d\n",p);
        localresult=p->next;
        free(p);
    }
}
/**
@brief Get number of saved sgRNA-Info on RAM.
@see localrow
*/
int localres_count(localrow *lr){
    int i=0;
    while(lr){
        i++;
        lr=lr->next;
        if(i>1000000) return -1;
    }
    return i;
}
<|endoftext|>


<commit_before>#include "xml.h"

#include <cstring>
#include <iostream>

#include "../leanify.h"
#include "../utils.h"

Xml::Xml(void *p, size_t s /*= 0*/) : Format(p, s), doc(true)
{
    unsigned char *q = (unsigned char *)p;

    const unsigned char utf8_bom[] = { 0xEF, 0xBB, 0xBF };

    // tinyxml2 does not support utf16
    /*
    const unsigned char utf16be_bom[] = { 0xFE, 0xFF };
    const unsigned char utf16le_bom[] = { 0xFF, 0xFE };
    */
    // skip utf8 bom
    if (!memcmp(q, utf8_bom, sizeof(utf8_bom)))
    {
        q += sizeof(utf8_bom);
    }
    /*      else if (!memcmp(q, utf16le_bom, sizeof(utf16le_bom)) || !memcmp(q, utf16be_bom, sizeof(utf16be_bom)))
    {
    q += sizeof(utf16le_bom);
    }
    */
    // skip spaces
    while (isspace(*q) && q < (unsigned char *)p + s)
    {
        q++;
    }
    // only parse the file if it starts with '<'
    if (*q == '<')
    {
        is_valid = (doc.Parse(fp, size) == 0);
    }
    else
    {
        is_valid = false;
    }
}

size_t Xml::Leanify(size_t size_leanified /*= 0*/)
{
    const char *root_name = doc.RootElement()->Name();
    // if the XML is fb2 file
    if (strcmp(root_name, "FictionBook") == 0)
    {
        if (is_verbose)
        {
            std::cout << "FB2 detected." << std::endl;
        }
        if (depth < max_depth)
        {
            depth++;

            // iterate through all binary element
            for (auto e = doc.RootElement()->FirstChildElement("binary"); e; e = e->NextSiblingElement("binary"))
            {
                for (int i = 1; i < depth; i++)
                {
                    std::cout << "-> ";
                }
                std::cout << e->Attribute("id") << std::endl;

                const char *base64_data = e->GetText();
                if (base64_data == nullptr)
                {
                    std::cout << "No data found." << std::endl;
                    continue;
                }
                size_t base64_len = strlen(base64_data);

                // 4 base64 character contains information of 3 bytes
                size_t binary_len = 3 * base64_len / 4;
                unsigned char *binary_data = new unsigned char[binary_len];

                if (Base64Decode(base64_data, base64_len, binary_data, &binary_len))
                {
                    std::cout << "Base64 decode error." << std::endl;
                    delete[] binary_data;
                    continue;
                }

                // Leanify embedded image
                binary_len = LeanifyFile(binary_data, binary_len);

                // allocate a few more bytes for padding
                size_t new_base64_len = 4 * binary_len / 3 + 4;
                char *new_base64_data = new char[new_base64_len];

                if (Base64Encode(binary_data, binary_len, new_base64_data, new_base64_len))
                {
                    std::cout << "Base64 encode error." << std::endl;
                }
                else if (strlen(new_base64_data) < base64_len)
                {
                    e->SetText(new_base64_data);
                }

                delete[] binary_data;
                delete[] new_base64_data;
            }
            depth--;
        }
    }
    else if (strcmp(root_name, "svg") == 0)
    {
        if (is_verbose)
        {
            std::cout << "SVG detected." << std::endl;
        }

        TraverseElements(doc.RootElement(), [](tinyxml2::XMLElement* e)
        {
            // remove empty attribute
            for (auto attr = e->FirstAttribute(); attr; attr = attr->Next())
            {
                auto value = attr->Value();
                if (value == nullptr || *value == 0)
                {
                    e->DeleteAttribute(attr->Name());
                }
            }

            // remove empty text element and container element
            auto name = e->Name();
            if (e->NoChildren())
            {
                if (strcmp(name, "text") == 0 ||
                    strcmp(name, "tspan") == 0 ||
                    strcmp(name, "a") == 0 ||
                    strcmp(name, "defs") == 0 ||
                    strcmp(name, "g") == 0 ||
                    strcmp(name, "marker") == 0 ||
                    strcmp(name, "mask") == 0 ||
                    strcmp(name, "missing-glyph") == 0 ||
                    strcmp(name, "pattern") == 0 ||
                    strcmp(name, "switch") == 0 ||
                    strcmp(name, "symbol") == 0)
                {
                    e->Parent()->DeleteChild(e);
                    return;
                }
            }

            if (strcmp(name, "tref") == 0 && e->Attribute("xlink:href") == nullptr)
            {
                e->Parent()->DeleteChild(e);
                return;
            }

            if (strcmp(name, "metadata") == 0)
            {
                e->Parent()->DeleteChild(e);
                return;
            }
        });
    }

    // print leanified XML to memory
    tinyxml2::XMLPrinter printer(0, true);
    doc.Print(&printer);

    size_t new_size = printer.CStrSize() - 1;     // -1 for the \0
    fp -= size_leanified;
    if (new_size < size)
    {
        memcpy(fp, printer.CStr(), new_size);
        return new_size;
    }
    else if (size_leanified)
    {
        memmove(fp, fp + size_leanified, size);
    }
    return size;
}

void Xml::TraverseElements(tinyxml2::XMLElement *ele, std::function<void(tinyxml2::XMLElement*)> callback)
{
    for (auto e = ele->FirstChildElement(); e; e = e->NextSiblingElement())
    {
        TraverseElements(e, callback);
    }

    callback(ele);
}

<commit_msg>SVG: remove XML declaration and doctype<commit_after>#include "xml.h"

#include <cstring>
#include <iostream>

#include "../leanify.h"
#include "../utils.h"

Xml::Xml(void *p, size_t s /*= 0*/) : Format(p, s), doc(true)
{
    unsigned char *q = (unsigned char *)p;

    const unsigned char utf8_bom[] = { 0xEF, 0xBB, 0xBF };

    // tinyxml2 does not support utf16
    /*
    const unsigned char utf16be_bom[] = { 0xFE, 0xFF };
    const unsigned char utf16le_bom[] = { 0xFF, 0xFE };
    */
    // skip utf8 bom
    if (!memcmp(q, utf8_bom, sizeof(utf8_bom)))
    {
        q += sizeof(utf8_bom);
    }
    /*      else if (!memcmp(q, utf16le_bom, sizeof(utf16le_bom)) || !memcmp(q, utf16be_bom, sizeof(utf16be_bom)))
    {
    q += sizeof(utf16le_bom);
    }
    */
    // skip spaces
    while (isspace(*q) && q < (unsigned char *)p + s)
    {
        q++;
    }
    // only parse the file if it starts with '<'
    if (*q == '<')
    {
        is_valid = (doc.Parse(fp, size) == 0);
    }
    else
    {
        is_valid = false;
    }
}

size_t Xml::Leanify(size_t size_leanified /*= 0*/)
{
    const char *root_name = doc.RootElement()->Name();
    // if the XML is fb2 file
    if (strcmp(root_name, "FictionBook") == 0)
    {
        if (is_verbose)
        {
            std::cout << "FB2 detected." << std::endl;
        }
        if (depth < max_depth)
        {
            depth++;

            // iterate through all binary element
            for (auto e = doc.RootElement()->FirstChildElement("binary"); e; e = e->NextSiblingElement("binary"))
            {
                for (int i = 1; i < depth; i++)
                {
                    std::cout << "-> ";
                }
                std::cout << e->Attribute("id") << std::endl;

                const char *base64_data = e->GetText();
                if (base64_data == nullptr)
                {
                    std::cout << "No data found." << std::endl;
                    continue;
                }
                size_t base64_len = strlen(base64_data);

                // 4 base64 character contains information of 3 bytes
                size_t binary_len = 3 * base64_len / 4;
                unsigned char *binary_data = new unsigned char[binary_len];

                if (Base64Decode(base64_data, base64_len, binary_data, &binary_len))
                {
                    std::cout << "Base64 decode error." << std::endl;
                    delete[] binary_data;
                    continue;
                }

                // Leanify embedded image
                binary_len = LeanifyFile(binary_data, binary_len);

                // allocate a few more bytes for padding
                size_t new_base64_len = 4 * binary_len / 3 + 4;
                char *new_base64_data = new char[new_base64_len];

                if (Base64Encode(binary_data, binary_len, new_base64_data, new_base64_len))
                {
                    std::cout << "Base64 encode error." << std::endl;
                }
                else if (strlen(new_base64_data) < base64_len)
                {
                    e->SetText(new_base64_data);
                }

                delete[] binary_data;
                delete[] new_base64_data;
            }
            depth--;
        }
    }
    else if (strcmp(root_name, "svg") == 0)
    {
        if (is_verbose)
        {
            std::cout << "SVG detected." << std::endl;
        }

        // remove XML declaration and doctype
        for (auto child = doc.FirstChild(); child; child = child->NextSibling())
        {
            if (child->ToDeclaration() || child->ToUnknown())
            {
                doc.DeleteChild(child);
            }
        }

        TraverseElements(doc.RootElement(), [](tinyxml2::XMLElement* e)
        {
            // remove empty attribute
            for (auto attr = e->FirstAttribute(); attr; attr = attr->Next())
            {
                auto value = attr->Value();
                if (value == nullptr || *value == 0)
                {
                    e->DeleteAttribute(attr->Name());
                }
            }

            // remove empty text element and container element
            auto name = e->Name();
            if (e->NoChildren())
            {
                if (strcmp(name, "text") == 0 ||
                    strcmp(name, "tspan") == 0 ||
                    strcmp(name, "a") == 0 ||
                    strcmp(name, "defs") == 0 ||
                    strcmp(name, "g") == 0 ||
                    strcmp(name, "marker") == 0 ||
                    strcmp(name, "mask") == 0 ||
                    strcmp(name, "missing-glyph") == 0 ||
                    strcmp(name, "pattern") == 0 ||
                    strcmp(name, "switch") == 0 ||
                    strcmp(name, "symbol") == 0)
                {
                    e->Parent()->DeleteChild(e);
                    return;
                }
            }

            if (strcmp(name, "tref") == 0 && e->Attribute("xlink:href") == nullptr)
            {
                e->Parent()->DeleteChild(e);
                return;
            }

            if (strcmp(name, "metadata") == 0)
            {
                e->Parent()->DeleteChild(e);
                return;
            }
        });
    }

    // print leanified XML to memory
    tinyxml2::XMLPrinter printer(0, true);
    doc.Print(&printer);

    size_t new_size = printer.CStrSize() - 1;     // -1 for the \0
    fp -= size_leanified;
    if (new_size < size)
    {
        memcpy(fp, printer.CStr(), new_size);
        return new_size;
    }
    else if (size_leanified)
    {
        memmove(fp, fp + size_leanified, size);
    }
    return size;
}

void Xml::TraverseElements(tinyxml2::XMLElement *ele, std::function<void(tinyxml2::XMLElement*)> callback)
{
    for (auto e = ele->FirstChildElement(); e; e = e->NextSiblingElement())
    {
        TraverseElements(e, callback);
    }

    callback(ele);
}

<|endoftext|>


<commit_before>#include <cstdint>
#include <cassert>

#include <immintrin.h>
#include <x86intrin.h>

#define packed_dword(x) _mm512_set1_epi32(x)
#define packed_byte(x) packed_dword((x) | ((x) << 8) | ((x) << 16) | ((x) << 24))

namespace base64 {

    namespace avx512 {

        namespace precalc {

            static uint32_t lookup_0[256];
            static uint32_t lookup_1[256];
            static uint32_t lookup_2[256];
            static uint32_t lookup_3[256];
        }

        void initalize_lookup() {

            using namespace precalc;

            for (int i=0; i < 256; i++) {
                lookup_0[i] = 0x80000000;
                lookup_1[i] = 0x80000000;
                lookup_2[i] = 0x80000000;
                lookup_3[i] = 0x80000000;
            }

            for (int i=0; i < 64; i++) {
                const uint32_t val = static_cast<uint8_t>(base64::lookup[i]);

                lookup_0[val] = i;
                lookup_1[val] = i << 1*6;
                lookup_2[val] = i << 2*6;
                lookup_3[val] = i << 3*6;
            }
        }

        __m512i lookup_gather(const __m512i input, __mmask16& error) {

            const __m512i b0 = _mm512_and_si512(input, packed_dword(0x000000ff));
            const __m512i b1 = _mm512_and_si512(_mm512_srli_epi32(input, 1*8), packed_dword(0x000000ff));
            const __m512i b2 = _mm512_and_si512(_mm512_srli_epi32(input, 2*8), packed_dword(0x000000ff));
            const __m512i b3 = _mm512_and_si512(_mm512_srli_epi32(input, 3*8), packed_dword(0x000000ff));

            // do lookup
            const __m512i r0 = _mm512_i32gather_epi32(b0, (const int*)precalc::lookup_0, 4);
            const __m512i r1 = _mm512_i32gather_epi32(b1, (const int*)precalc::lookup_1, 4);
            const __m512i r2 = _mm512_i32gather_epi32(b2, (const int*)precalc::lookup_2, 4);
            const __m512i r3 = _mm512_i32gather_epi32(b3, (const int*)precalc::lookup_3, 4);
            const __m512i translated = _mm512_or_si512(r0,
                                       _mm512_or_si512(r1,
                                       _mm512_or_si512(r2, r3)));

            error = _mm512_cmplt_epi32_mask(translated, _mm512_set1_epi32(0));

            return translated;
        }

        // returns packed (a[i] >= lo and a[i] <= hi) ? 0x7f : 0x00;
        // assertion a[i] < 0x80
        __m512i _mm512_range_mask_7bit(const __m512i a, const __m512i lo, const __m512i hi) {

            const __m512i L = _mm512_add_epi32(a, lo);
            const __m512i H = _mm512_add_epi32(a, hi);

            /*
                MSB  H   L  |  R
                 0   0   0  |  0
                 0   0   1  |  0
                 0   1   0  |  0
                 0   1   1  |  0
                 1   0   0  |  0
                 1   0   1  |  1
                 1   1   0  |  0
                 1   1   1  |  0
            */

#if 0
            const __m512i MSB = _mm512_and_epi32(_mm512_andnot_epi32(H, L), packed_byte(0x80));
#else
            const __m512i MSB = _mm512_ternarylogic_epi32(packed_byte(0x80), H, L, 0x20);
#endif
            // MSB | (MSB - (MSB >> 7))
            return _mm512_sub_epi32(MSB, _mm512_srli_epi32(MSB, 7));
        }

        // returns packed (a[i] >= lo and a[i] <= hi) ? 0xff : 0x00;
        // assertion a[i] < 0x80
        __m512i _mm512_range_mask_8bit(const __m512i a, const __m512i lo, const __m512i hi) {

            const __m512i L = _mm512_add_epi32(a, lo);
            const __m512i H = _mm512_add_epi32(a, hi);

            const __m512i MSB = _mm512_ternarylogic_epi32(packed_byte(0x80), H, L, 0x20);

            // MSB | (MSB - (MSB >> 7))
            return _mm512_or_si512(MSB, _mm512_sub_epi32(MSB, _mm512_srli_epi32(MSB, 7)));
        }

        // returns packed (a[i] >= lo and a[i] <= hi) ? 0x04 : 0x00;
        // assertion a[i] < 0x80
        __m512i _mm512_range_3rd_bit(const __m512i a, const __m512i lo, const __m512i hi) {

            const __m512i L = _mm512_add_epi32(a, lo);
            const __m512i H = _mm512_add_epi32(a, hi);

            const __m512i MSB = _mm512_ternarylogic_epi32(packed_byte(0x80), H, L, 0x20);

            return _mm512_srli_epi32(MSB, 5);
        }

        __m512i lookup_comparisons(const __m512i input, __mmask16& error) {

            // we operate on lower 7 bits, as all values with 8th bit set are invalid
            const __m512i in = _mm512_and_si512(input, packed_byte(0x7f));

#define GET_RANGE_SHIFT_7BIT(shift, lo, hi) \
            _mm512_and_si512(packed_byte(uint8_t(shift)), \
                             _mm512_range_mask_7bit(in, packed_byte(0x80 - (lo)), packed_byte(0x80 - (hi))))

#define GET_RANGE_SHIFT_8BIT(shift, lo, hi) \
            _mm512_and_si512(packed_byte(uint8_t(shift)), \
                             _mm512_range_mask_8bit(in, packed_byte(0x80 - (lo)), packed_byte(0x80 - (hi))))

#define GET_RANGE_SHIFT_3RD_BIT(lo, hi) \
            _mm512_range_3rd_bit(in, packed_byte(0x80 - (lo)), packed_byte(0x80 - (hi)))

            // shift for range 'A' - 'Z'
            const __m512i range_AZ = GET_RANGE_SHIFT_8BIT(-65, 'A', 'Z' + 1);

            // shift for range 'a' - 'z'
            const __m512i range_az = GET_RANGE_SHIFT_8BIT(-71, 'a', 'z' + 1);

            // shift for range '0' - '9'
            const __m512i range_09 = GET_RANGE_SHIFT_3RD_BIT('0', '9' + 1); // shift = 4

            // shift for character '+'
            const __m512i char_plus = GET_RANGE_SHIFT_7BIT(19, '+', '+' + 1);

            // shift for character '/'
            const __m512i char_slash = GET_RANGE_SHIFT_7BIT(16, '/', '/' + 1);

            // merge partial results
            /*
                a b c a|b|c
                0 0 0   0
                0 0 1   1
                0 1 0   1
                0 1 1   1
                1 0 0   1
                1 0 1   1
                1 1 0   1
                1 1 1   1
            */
#if 0
            const __m512i shift = _mm512_or_si512(range_AZ,
                                  _mm512_or_si512(range_az,
                                  _mm512_or_si512(range_09,
                                  _mm512_or_si512(char_plus, char_slash))));
#else
            const __m512i tmp   = _mm512_ternarylogic_epi32(range_AZ, range_az, range_09, 0xfe);
            const __m512i shift = _mm512_ternarylogic_epi32(char_plus, char_slash, tmp, 0xfe);
#endif

            // a paddb equivalent
            const __m512i shift06 = _mm512_and_si512(shift, packed_byte(0x7f));
            const __m512i shift7  = _mm512_and_si512(shift, packed_byte(0x80));
            const __m512i result  = _mm512_xor_si512(_mm512_add_epi32(in, shift06), shift7);

            // validation
            const __m512i non_zero_7lower = _mm512_add_epi32(shift06, packed_byte(0x7f));
#if 1
            /*  7 lower bits of shift are non-zero
                | 7th bit of shift is non-zero
                | | 7th bit of inoput is non-zero (extended ASCII)
                | | |
                a b c r
                0 0 0 1
                0 0 1 1
                0 1 0 0
                0 1 1 1
                1 0 0 0
                1 0 1 1
                1 1 0 0
                1 1 1 1

                expression = ~(a | b) | c

            */
            // we're use 7th bit of each byte
            const __m512i valid = _mm512_ternarylogic_epi32(in, shift, non_zero_7lower, 0xab);

#else       // a reference implementation
            // non-zero byte in shift -- 7th bit is set
            const __m512i non_zero = _mm512_or_si512(shift, non_zero_7lower);

            // 7th bit marks error - either shift is zero or byte outside standard ASCII
            const __m512i valid = _mm512_or_si512(input, _mm512_xor_si512(non_zero, packed_byte(0x80)));
#endif
            error = _mm512_test_epi32_mask(valid, packed_byte(0x80));

            return result;
        }

    } // namespace avx512bw

} // namespace base64

#undef packed_dword
<commit_msg>decode: AVX512 gather-based proc slightly faster<commit_after>#include <cstdint>
#include <cassert>

#include <immintrin.h>
#include <x86intrin.h>

#define packed_dword(x) _mm512_set1_epi32(x)
#define packed_byte(x) packed_dword((x) | ((x) << 8) | ((x) << 16) | ((x) << 24))

namespace base64 {

    namespace avx512 {

        namespace precalc {

            static uint32_t lookup_0[256];
            static uint32_t lookup_1[256];
            static uint32_t lookup_2[256];
            static uint32_t lookup_3[256];
        }

        void initalize_lookup() {

            using namespace precalc;

            for (int i=0; i < 256; i++) {
                lookup_0[i] = 0x80000000;
                lookup_1[i] = 0x80000000;
                lookup_2[i] = 0x80000000;
                lookup_3[i] = 0x80000000;
            }

            for (int i=0; i < 64; i++) {
                const uint32_t val = static_cast<uint8_t>(base64::lookup[i]);

                lookup_0[val] = i;
                lookup_1[val] = i << 1*6;
                lookup_2[val] = i << 2*6;
                lookup_3[val] = i << 3*6;
            }
        }

        const uint8_t OR_ALL = 0xfe;

        __m512i lookup_gather(const __m512i input, __mmask16& error) {

            const __m512i b0 = _mm512_and_si512(input, packed_dword(0x000000ff));
            const __m512i b1 = _mm512_and_si512(_mm512_srli_epi32(input, 1*8), packed_dword(0x000000ff));
            const __m512i b2 = _mm512_and_si512(_mm512_srli_epi32(input, 2*8), packed_dword(0x000000ff));
            const __m512i b3 = _mm512_srli_epi32(input, 3*8);

            // do lookup
            const __m512i r0 = _mm512_i32gather_epi32(b0, (const int*)precalc::lookup_0, 4);
            const __m512i r1 = _mm512_i32gather_epi32(b1, (const int*)precalc::lookup_1, 4);
            const __m512i r2 = _mm512_i32gather_epi32(b2, (const int*)precalc::lookup_2, 4);
            const __m512i r3 = _mm512_i32gather_epi32(b3, (const int*)precalc::lookup_3, 4);

            // r0 | r1 | r2 | r3
            const __m512i translated = _mm512_or_si512(r0, _mm512_ternarylogic_epi32(r1, r2, r3, OR_ALL));

            error = _mm512_cmplt_epi32_mask(translated, _mm512_set1_epi32(0));

            return translated;
        }

        // returns packed (a[i] >= lo and a[i] <= hi) ? 0x7f : 0x00;
        // assertion a[i] < 0x80
        __m512i _mm512_range_mask_7bit(const __m512i a, const __m512i lo, const __m512i hi) {

            const __m512i L = _mm512_add_epi32(a, lo);
            const __m512i H = _mm512_add_epi32(a, hi);

            /*
                MSB  H   L  |  R
                 0   0   0  |  0
                 0   0   1  |  0
                 0   1   0  |  0
                 0   1   1  |  0
                 1   0   0  |  0
                 1   0   1  |  1
                 1   1   0  |  0
                 1   1   1  |  0
            */

#if 0
            const __m512i MSB = _mm512_and_epi32(_mm512_andnot_epi32(H, L), packed_byte(0x80));
#else
            const __m512i MSB = _mm512_ternarylogic_epi32(packed_byte(0x80), H, L, 0x20);
#endif
            // MSB | (MSB - (MSB >> 7))
            return _mm512_sub_epi32(MSB, _mm512_srli_epi32(MSB, 7));
        }

        // returns packed (a[i] >= lo and a[i] <= hi) ? 0xff : 0x00;
        // assertion a[i] < 0x80
        __m512i _mm512_range_mask_8bit(const __m512i a, const __m512i lo, const __m512i hi) {

            const __m512i L = _mm512_add_epi32(a, lo);
            const __m512i H = _mm512_add_epi32(a, hi);

            const __m512i MSB = _mm512_ternarylogic_epi32(packed_byte(0x80), H, L, 0x20);

            // MSB | (MSB - (MSB >> 7))
            return _mm512_or_si512(MSB, _mm512_sub_epi32(MSB, _mm512_srli_epi32(MSB, 7)));
        }

        // returns packed (a[i] >= lo and a[i] <= hi) ? 0x04 : 0x00;
        // assertion a[i] < 0x80
        __m512i _mm512_range_3rd_bit(const __m512i a, const __m512i lo, const __m512i hi) {

            const __m512i L = _mm512_add_epi32(a, lo);
            const __m512i H = _mm512_add_epi32(a, hi);

            const __m512i MSB = _mm512_ternarylogic_epi32(packed_byte(0x80), H, L, 0x20);

            return _mm512_srli_epi32(MSB, 5);
        }

        __m512i lookup_comparisons(const __m512i input, __mmask16& error) {

            // we operate on lower 7 bits, as all values with 8th bit set are invalid
            const __m512i in = _mm512_and_si512(input, packed_byte(0x7f));

#define GET_RANGE_SHIFT_7BIT(shift, lo, hi) \
            _mm512_and_si512(packed_byte(uint8_t(shift)), \
                             _mm512_range_mask_7bit(in, packed_byte(0x80 - (lo)), packed_byte(0x80 - (hi))))

#define GET_RANGE_SHIFT_8BIT(shift, lo, hi) \
            _mm512_and_si512(packed_byte(uint8_t(shift)), \
                             _mm512_range_mask_8bit(in, packed_byte(0x80 - (lo)), packed_byte(0x80 - (hi))))

#define GET_RANGE_SHIFT_3RD_BIT(lo, hi) \
            _mm512_range_3rd_bit(in, packed_byte(0x80 - (lo)), packed_byte(0x80 - (hi)))

            // shift for range 'A' - 'Z'
            const __m512i range_AZ = GET_RANGE_SHIFT_8BIT(-65, 'A', 'Z' + 1);

            // shift for range 'a' - 'z'
            const __m512i range_az = GET_RANGE_SHIFT_8BIT(-71, 'a', 'z' + 1);

            // shift for range '0' - '9'
            const __m512i range_09 = GET_RANGE_SHIFT_3RD_BIT('0', '9' + 1); // shift = 4

            // shift for character '+'
            const __m512i char_plus = GET_RANGE_SHIFT_7BIT(19, '+', '+' + 1);

            // shift for character '/'
            const __m512i char_slash = GET_RANGE_SHIFT_7BIT(16, '/', '/' + 1);

            // merge partial results
            /*
                a b c a|b|c
                0 0 0   0
                0 0 1   1
                0 1 0   1
                0 1 1   1
                1 0 0   1
                1 0 1   1
                1 1 0   1
                1 1 1   1
            */
#if 0
            const __m512i shift = _mm512_or_si512(range_AZ,
                                  _mm512_or_si512(range_az,
                                  _mm512_or_si512(range_09,
                                  _mm512_or_si512(char_plus, char_slash))));
#else
            const __m512i tmp   = _mm512_ternarylogic_epi32(range_AZ, range_az, range_09, 0xfe);
            const __m512i shift = _mm512_ternarylogic_epi32(char_plus, char_slash, tmp, 0xfe);
#endif

            // a paddb equivalent
            const __m512i shift06 = _mm512_and_si512(shift, packed_byte(0x7f));
            const __m512i shift7  = _mm512_and_si512(shift, packed_byte(0x80));
            const __m512i result  = _mm512_xor_si512(_mm512_add_epi32(in, shift06), shift7);

            // validation
            const __m512i non_zero_7lower = _mm512_add_epi32(shift06, packed_byte(0x7f));
#if 1
            /*  7 lower bits of shift are non-zero
                | 7th bit of shift is non-zero
                | | 7th bit of inoput is non-zero (extended ASCII)
                | | |
                a b c r
                0 0 0 1
                0 0 1 1
                0 1 0 0
                0 1 1 1
                1 0 0 0
                1 0 1 1
                1 1 0 0
                1 1 1 1

                expression = ~(a | b) | c

            */
            // we're use 7th bit of each byte
            const __m512i valid = _mm512_ternarylogic_epi32(in, shift, non_zero_7lower, 0xab);

#else       // a reference implementation
            // non-zero byte in shift -- 7th bit is set
            const __m512i non_zero = _mm512_or_si512(shift, non_zero_7lower);

            // 7th bit marks error - either shift is zero or byte outside standard ASCII
            const __m512i valid = _mm512_or_si512(input, _mm512_xor_si512(non_zero, packed_byte(0x80)));
#endif
            error = _mm512_test_epi32_mask(valid, packed_byte(0x80));

            return result;
        }

    } // namespace avx512bw

} // namespace base64

#undef packed_dword
<|endoftext|>


<commit_before>//
//  SamplerVoice.cpp
//  AudioKit Core
//
//  Created by Shane Dunne, revision history on Github.
//  Copyright © 2018 AudioKit. All rights reserved.
//

#include "SamplerVoice.hpp"
#include <stdio.h>

namespace AudioKitCore
{
    void SamplerVoice::init(double sampleRate)
    {
        samplingRate = float(sampleRate);
        leftFilter.init(sampleRate);
        rightFilter.init(sampleRate);
        adsrEnvelope.init();
        filterEnvelope.init();
    }

    void SamplerVoice::start(unsigned note, float sampleRate, float frequency, float volume, SampleBuffer* buffer)
    {
        sampleBuffer = buffer;
        oscillator.indexPoint = buffer->startPoint;
        oscillator.increment = (buffer->sampleRate / sampleRate) * (frequency / buffer->noteFrequency);
        oscillator.multiplier = 1.0;
        oscillator.isLooping = buffer->isLooping;

        noteVolume = volume;
        adsrEnvelope.start();

        samplingRate = sampleRate;
        leftFilter.updateSampleRate(double(samplingRate));
        rightFilter.updateSampleRate(double(samplingRate));
        filterEnvelope.start();

        glideSemitones = 0.0f;
        if (*glideSecPerOctave != 0.0f && noteFrequency != 0.0 && noteFrequency != frequency)
        {
            // prepare to glide
            glideSemitones = -12.0f * log2f(frequency / noteFrequency);
            if (fabsf(glideSemitones) < 0.01f) glideSemitones = 0.0f;
        }
        noteFrequency = frequency;
        noteNumber = note;
    }

    void SamplerVoice::restart(unsigned note, float sampleRate, float frequency)
    {
        oscillator.increment = (sampleBuffer->sampleRate / sampleRate) * (frequency / sampleBuffer->noteFrequency);
        glideSemitones = 0.0f;
        if (*glideSecPerOctave != 0.0f && noteFrequency != 0.0 && noteFrequency != frequency)
        {
            // prepare to glide
            glideSemitones = -12.0f * log2f(frequency / noteFrequency);
            if (fabsf(glideSemitones) < 0.01f) glideSemitones = 0.0f;
        }
        noteFrequency = frequency;
        noteNumber = note;
    }

    void SamplerVoice::restart(float volume, SampleBuffer* buffer)
    {
        tempNoteVolume = noteVolume;
        newSampleBuffer = buffer;
        adsrEnvelope.restart();
        noteVolume = volume;
        filterEnvelope.start();
    }

    void SamplerVoice::release(bool loopThruRelease)
    {
        if (!loopThruRelease) oscillator.isLooping = false;
        adsrEnvelope.release();
        filterEnvelope.release();
    }

    void SamplerVoice::stop()
    {
        noteNumber = -1;
        adsrEnvelope.reset();
        filterEnvelope.reset();
    }

    bool SamplerVoice::prepToGetSamples(int sampleCount, float masterVolume, float pitchOffset,
                                        float cutoffMultiple, float cutoffEnvelopeStrength,
                                        float resLinear)
    {
        if (adsrEnvelope.isIdle()) return true;

        if (adsrEnvelope.isPreStarting())
        {
            tempGain = masterVolume * tempNoteVolume * adsrEnvelope.getSample();
            if (!adsrEnvelope.isPreStarting())
            {
                tempGain = masterVolume * noteVolume * adsrEnvelope.getSample();
                sampleBuffer = newSampleBuffer;
                oscillator.indexPoint = sampleBuffer->startPoint;
                oscillator.isLooping = sampleBuffer->isLooping;
            }
        }
        else
            tempGain = masterVolume * noteVolume * adsrEnvelope.getSample();

        if (*glideSecPerOctave != 0.0f && glideSemitones != 0.0f)
        {
            float seconds = sampleCount / samplingRate;
            float semitones = 12.0f * seconds / *glideSecPerOctave;
            if (glideSemitones < 0.0f)
            {
                glideSemitones += 12.0f * semitones;
                if (glideSemitones > 0.0f) glideSemitones = 0.0f;
            }
            else
            {
                glideSemitones -= 12.0f * semitones;
                if (glideSemitones < 0.0f) glideSemitones = 0.0f;
            }
        }
        oscillator.setPitchOffsetSemitones(pitchOffset + glideSemitones);

        // negative value of cutoffMultiple means filters are disabled
        if (cutoffMultiple < 0.0f)
        {
            isFilterEnabled = false;
        }
        else
        {
            isFilterEnabled = true;
            double cutoffHz = noteFrequency * (1.0f + cutoffMultiple + cutoffEnvelopeStrength * filterEnvelope.getSample());
            leftFilter.setParams(cutoffHz, resLinear);
            rightFilter.setParams(cutoffHz, resLinear);
        }

        return false;
    }

    bool SamplerVoice::getSamples(int sampleCount, float* leftOutput, float* rightOutput)
    {
        for (int i=0; i < sampleCount; i++)
        {
            float leftSample, rightSample;
            if (oscillator.getSamplePair(sampleBuffer, sampleCount, &leftSample, &rightSample, tempGain)) return true;
            if (isFilterEnabled)
            {
                *leftOutput++ += leftFilter.process(leftSample);
                *rightOutput++ += rightFilter.process(rightSample);
            }
            else
            {
                *leftOutput++ += leftSample;
                *rightOutput++ += rightSample;
            }
        }
        return false;
    }

}
<commit_msg>AKSampler: fix glide timing<commit_after>//
//  SamplerVoice.cpp
//  AudioKit Core
//
//  Created by Shane Dunne, revision history on Github.
//  Copyright © 2018 AudioKit. All rights reserved.
//

#include "SamplerVoice.hpp"
#include <stdio.h>

namespace AudioKitCore
{
    void SamplerVoice::init(double sampleRate)
    {
        samplingRate = float(sampleRate);
        leftFilter.init(sampleRate);
        rightFilter.init(sampleRate);
        adsrEnvelope.init();
        filterEnvelope.init();
    }

    void SamplerVoice::start(unsigned note, float sampleRate, float frequency, float volume, SampleBuffer* buffer)
    {
        sampleBuffer = buffer;
        oscillator.indexPoint = buffer->startPoint;
        oscillator.increment = (buffer->sampleRate / sampleRate) * (frequency / buffer->noteFrequency);
        oscillator.multiplier = 1.0;
        oscillator.isLooping = buffer->isLooping;

        noteVolume = volume;
        adsrEnvelope.start();

        samplingRate = sampleRate;
        leftFilter.updateSampleRate(double(samplingRate));
        rightFilter.updateSampleRate(double(samplingRate));
        filterEnvelope.start();

        glideSemitones = 0.0f;
        if (*glideSecPerOctave != 0.0f && noteFrequency != 0.0 && noteFrequency != frequency)
        {
            // prepare to glide
            glideSemitones = -12.0f * log2f(frequency / noteFrequency);
            if (fabsf(glideSemitones) < 0.01f) glideSemitones = 0.0f;
        }
        noteFrequency = frequency;
        noteNumber = note;
    }

    void SamplerVoice::restart(unsigned note, float sampleRate, float frequency)
    {
        oscillator.increment = (sampleBuffer->sampleRate / sampleRate) * (frequency / sampleBuffer->noteFrequency);
        glideSemitones = 0.0f;
        if (*glideSecPerOctave != 0.0f && noteFrequency != 0.0 && noteFrequency != frequency)
        {
            // prepare to glide
            glideSemitones = -12.0f * log2f(frequency / noteFrequency);
            if (fabsf(glideSemitones) < 0.01f) glideSemitones = 0.0f;
        }
        noteFrequency = frequency;
        noteNumber = note;
    }

    void SamplerVoice::restart(float volume, SampleBuffer* buffer)
    {
        tempNoteVolume = noteVolume;
        newSampleBuffer = buffer;
        adsrEnvelope.restart();
        noteVolume = volume;
        filterEnvelope.start();
    }

    void SamplerVoice::release(bool loopThruRelease)
    {
        if (!loopThruRelease) oscillator.isLooping = false;
        adsrEnvelope.release();
        filterEnvelope.release();
    }

    void SamplerVoice::stop()
    {
        noteNumber = -1;
        adsrEnvelope.reset();
        filterEnvelope.reset();
    }

    bool SamplerVoice::prepToGetSamples(int sampleCount, float masterVolume, float pitchOffset,
                                        float cutoffMultiple, float cutoffEnvelopeStrength,
                                        float resLinear)
    {
        if (adsrEnvelope.isIdle()) return true;

        if (adsrEnvelope.isPreStarting())
        {
            tempGain = masterVolume * tempNoteVolume * adsrEnvelope.getSample();
            if (!adsrEnvelope.isPreStarting())
            {
                tempGain = masterVolume * noteVolume * adsrEnvelope.getSample();
                sampleBuffer = newSampleBuffer;
                oscillator.indexPoint = sampleBuffer->startPoint;
                oscillator.isLooping = sampleBuffer->isLooping;
            }
        }
        else
            tempGain = masterVolume * noteVolume * adsrEnvelope.getSample();

        if (*glideSecPerOctave != 0.0f && glideSemitones != 0.0f)
        {
            float seconds = sampleCount / samplingRate;
            float semitones = 12.0f * seconds / *glideSecPerOctave;
            if (glideSemitones < 0.0f)
            {
                glideSemitones += semitones;
                if (glideSemitones > 0.0f) glideSemitones = 0.0f;
            }
            else
            {
                glideSemitones -= semitones;
                if (glideSemitones < 0.0f) glideSemitones = 0.0f;
            }
        }
        oscillator.setPitchOffsetSemitones(pitchOffset + glideSemitones);

        // negative value of cutoffMultiple means filters are disabled
        if (cutoffMultiple < 0.0f)
        {
            isFilterEnabled = false;
        }
        else
        {
            isFilterEnabled = true;
            double cutoffHz = noteFrequency * (1.0f + cutoffMultiple + cutoffEnvelopeStrength * filterEnvelope.getSample());
            leftFilter.setParams(cutoffHz, resLinear);
            rightFilter.setParams(cutoffHz, resLinear);
        }

        return false;
    }

    bool SamplerVoice::getSamples(int sampleCount, float* leftOutput, float* rightOutput)
    {
        for (int i=0; i < sampleCount; i++)
        {
            float leftSample, rightSample;
            if (oscillator.getSamplePair(sampleBuffer, sampleCount, &leftSample, &rightSample, tempGain)) return true;
            if (isFilterEnabled)
            {
                *leftOutput++ += leftFilter.process(leftSample);
                *rightOutput++ += rightFilter.process(rightSample);
            }
            else
            {
                *leftOutput++ += leftSample;
                *rightOutput++ += rightSample;
            }
        }
        return false;
    }

}
<|endoftext|>


<commit_before>//
// This file is part of the Marble Virtual Globe.
//
// This program is free software licensed under the GNU LGPL. You can
// find a copy of this license in LICENSE.txt in the top directory of
// the source code.
//
// Copyright 2013   Utku Aydın <utkuaydin34@gmail.com>
//

#include "MergeItem.h"

#include "GeoDataPlacemark.h"

namespace Marble {

MergeItem::MergeItem()
{
}

QString MergeItem::pathA()
{
    return m_pathA;
}

void MergeItem::setPathA( const QString &path )
{
    m_pathA = path;
    pathAChanged();
}

QString MergeItem::pathB()
{
    return m_pathB;
}

void MergeItem::setPathB( const QString &path )
{
    m_pathB = path;
    pathBChanged();
}

GeoDataPlacemark MergeItem::placemarkA()
{
    return m_placemarkA;
}

void MergeItem::setPlacemarkA( const GeoDataPlacemark &placemark )
{
    m_placemarkA = placemark;
    emit placemarkAChanged();
    emit nameAChanged();
    emit descriptionAChanged();
}

GeoDataPlacemark MergeItem::placemarkB()
{
    return m_placemarkB;
}

void MergeItem::setPlacemarkB( const GeoDataPlacemark &placemark )
{
    m_placemarkB = placemark;
    emit placemarkBChanged();
    emit nameBChanged();
    emit descriptionBChanged();
}

QString MergeItem::nameA()
{
    return m_placemarkA.name();
}

QString MergeItem::nameB()
{
    return m_placemarkB.name();
}

QString MergeItem::descriptionA()
{
    return m_placemarkA.description();
}

QString MergeItem::descriptionB()
{
    return m_placemarkB.description();
}

MergeItem::Action MergeItem::actionA()
{
    return m_actionA;
}

void MergeItem::setActionA( MergeItem::Action action )
{
    m_actionA = action;
}

MergeItem::Action MergeItem::actionB()
{
    return m_actionB;
}

void MergeItem::setActionB( MergeItem::Action action )
{
    m_actionB = action;
}

MergeItem::Resolution MergeItem::resolution()
{
    return m_resolution;
}

void MergeItem::setResolution( MergeItem::Resolution resolution )
{
    m_resolution = resolution;
    emit resolutionChanged();
}

}

#include "MergeItem.moc"
<commit_msg>Add missing emit keywords<commit_after>//
// This file is part of the Marble Virtual Globe.
//
// This program is free software licensed under the GNU LGPL. You can
// find a copy of this license in LICENSE.txt in the top directory of
// the source code.
//
// Copyright 2013   Utku Aydın <utkuaydin34@gmail.com>
//

#include "MergeItem.h"

#include "GeoDataPlacemark.h"

namespace Marble {

MergeItem::MergeItem()
{
}

QString MergeItem::pathA()
{
    return m_pathA;
}

void MergeItem::setPathA( const QString &path )
{
    m_pathA = path;
    emit pathAChanged();
}

QString MergeItem::pathB()
{
    return m_pathB;
}

void MergeItem::setPathB( const QString &path )
{
    m_pathB = path;
    emit pathBChanged();
}

GeoDataPlacemark MergeItem::placemarkA()
{
    return m_placemarkA;
}

void MergeItem::setPlacemarkA( const GeoDataPlacemark &placemark )
{
    m_placemarkA = placemark;
    emit placemarkAChanged();
    emit nameAChanged();
    emit descriptionAChanged();
}

GeoDataPlacemark MergeItem::placemarkB()
{
    return m_placemarkB;
}

void MergeItem::setPlacemarkB( const GeoDataPlacemark &placemark )
{
    m_placemarkB = placemark;
    emit placemarkBChanged();
    emit nameBChanged();
    emit descriptionBChanged();
}

QString MergeItem::nameA()
{
    return m_placemarkA.name();
}

QString MergeItem::nameB()
{
    return m_placemarkB.name();
}

QString MergeItem::descriptionA()
{
    return m_placemarkA.description();
}

QString MergeItem::descriptionB()
{
    return m_placemarkB.description();
}

MergeItem::Action MergeItem::actionA()
{
    return m_actionA;
}

void MergeItem::setActionA( MergeItem::Action action )
{
    m_actionA = action;
}

MergeItem::Action MergeItem::actionB()
{
    return m_actionB;
}

void MergeItem::setActionB( MergeItem::Action action )
{
    m_actionB = action;
}

MergeItem::Resolution MergeItem::resolution()
{
    return m_resolution;
}

void MergeItem::setResolution( MergeItem::Resolution resolution )
{
    m_resolution = resolution;
    emit resolutionChanged();
}

}

#include "MergeItem.moc"
<|endoftext|>


<commit_before>/*
 *
 *    Copyright (c) 2020 Project CHIP Authors
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

#include "Server.h"

#include <errno.h>
#include <utility>

#include <mdns/minimal/core/DnsHeader.h>

namespace mdns {
namespace Minimal {
namespace {

class ShutdownOnError
{
public:
    ShutdownOnError(ServerBase * s) : mServer(s) {}
    ~ShutdownOnError()
    {
        if (mServer != nullptr)
        {
            mServer->Shutdown();
        }
    }

    CHIP_ERROR ReturnSuccess()
    {
        mServer = nullptr;
        return CHIP_NO_ERROR;
    }

private:
    ServerBase * mServer;
};

} // namespace

namespace BroadcastIpAddresses {

// Get standard mDNS Broadcast addresses

void GetIpv6Into(chip::Inet::IPAddress & dest)
{
    if (!chip::Inet::IPAddress::FromString("FF02::FB", dest))
    {
        ChipLogError(Discovery, "Failed to parse standard IPv6 broadcast address");
    }
}

void GetIpv4Into(chip::Inet::IPAddress & dest)
{
    if (!chip::Inet::IPAddress::FromString("224.0.0.251", dest))
    {
        ChipLogError(Discovery, "Failed to parse standard IPv4 broadcast address");
    }
}

} // namespace BroadcastIpAddresses

ServerBase::~ServerBase()
{
    Shutdown();
}

void ServerBase::Shutdown()
{
    for (size_t i = 0; i < mEndpointCount; i++)
    {
        if (mEndpoints[i].udp != nullptr)
        {
            mEndpoints[i].udp->Close();
            mEndpoints[i].udp = nullptr;
        }
    }
}

CHIP_ERROR ServerBase::Listen(chip::Inet::InetLayer * inetLayer, ListenIterator * it, uint16_t port)
{
    Shutdown(); // ensure everything starts fresh

    size_t endpointIndex                = 0;
    chip::Inet::InterfaceId interfaceId = INET_NULL_INTERFACEID;
    chip::Inet::IPAddressType addressType;

    ShutdownOnError autoShutdown(this);

    while (it->Next(&interfaceId, &addressType))
    {
        if (endpointIndex >= mEndpointCount)
        {
            return CHIP_ERROR_NO_MEMORY;
        }

        EndpointInfo * info = &mEndpoints[endpointIndex];
        info->addressType   = addressType;
        info->interfaceId   = interfaceId;

        CHIP_ERROR err = inetLayer->NewUDPEndPoint(&info->udp);
        if (err != CHIP_NO_ERROR)
        {
            return err;
        }

        info->udp->Bind(addressType, chip::Inet::IPAddress::Any, port, interfaceId);
        if (err != CHIP_NO_ERROR)
        {
            return err;
        }

        info->udp->AppState          = static_cast<void *>(this);
        info->udp->OnMessageReceived = OnUdpPacketReceived;

        err = info->udp->Listen();
        if (err != CHIP_NO_ERROR)
        {
            return err;
        }

        endpointIndex++;
    }

    return autoShutdown.ReturnSuccess();
}

CHIP_ERROR ServerBase::DirectSend(chip::System::PacketBufferHandle && data, const chip::Inet::IPAddress & addr, uint16_t port,
                                  chip::Inet::InterfaceId interface)
{
    for (size_t i = 0; i < mEndpointCount; i++)
    {
        EndpointInfo * info = &mEndpoints[i];
        if (info->udp == nullptr)
        {
            continue;
        }

        if (info->addressType != addr.Type())
        {
            continue;
        }

        chip::Inet::InterfaceId boundIf = info->udp->GetBoundInterface();

        if ((boundIf != INET_NULL_INTERFACEID) && (boundIf != interface))
        {
            continue;
        }

        return info->udp->SendTo(addr, port, std::move(data));
    }

    return CHIP_ERROR_NOT_CONNECTED;
}

CHIP_ERROR ServerBase::BroadcastSend(chip::System::PacketBufferHandle data, uint16_t port, chip::Inet::InterfaceId interface)
{
    for (size_t i = 0; i < mEndpointCount; i++)
    {
        EndpointInfo * info = &mEndpoints[i];

        if (info->udp == nullptr)
        {
            continue;
        }

        if ((info->udp->GetBoundInterface() != interface) && (info->udp->GetBoundInterface() != INET_NULL_INTERFACEID))
        {
            continue;
        }

        CHIP_ERROR err;

        /// The same packet needs to be sent over potentially multiple interfaces.
        /// LWIP does not like having a pbuf sent over serparate interfaces, hence we create a copy
        /// TODO: this wastes one copy of the data and that could be optimized away
        chip::System::PacketBufferHandle copy = data.CloneData();

        if (info->addressType == chip::Inet::kIPAddressType_IPv6)
        {
            err = info->udp->SendTo(mIpv6BroadcastAddress, port, info->udp->GetBoundInterface(), std::move(copy));
        }
#if INET_CONFIG_ENABLE_IPV4
        else if (info->addressType == chip::Inet::kIPAddressType_IPv4)
        {
            err = info->udp->SendTo(mIpv4BroadcastAddress, port, info->udp->GetBoundInterface(), std::move(copy));
        }
#endif
        else
        {
            return CHIP_ERROR_INCORRECT_STATE;
        }

        if (err != CHIP_NO_ERROR)
        {
            return err;
        }
    }

    return CHIP_NO_ERROR;
}

CHIP_ERROR ServerBase::BroadcastSend(chip::System::PacketBufferHandle data, uint16_t port)
{
    for (size_t i = 0; i < mEndpointCount; i++)
    {
        EndpointInfo * info = &mEndpoints[i];

        if (info->udp == nullptr)
        {
            continue;
        }

        CHIP_ERROR err;

        /// The same packet needs to be sent over potentially multiple interfaces.
        /// LWIP does not like having a pbuf sent over serparate interfaces, hence we create a copy
        /// TODO: this wastes one copy of the data and that could be optimized away
        chip::System::PacketBufferHandle copy = data.CloneData();

        if (info->addressType == chip::Inet::kIPAddressType_IPv6)
        {
            err = info->udp->SendTo(mIpv6BroadcastAddress, port, info->udp->GetBoundInterface(), std::move(copy));
        }
#if INET_CONFIG_ENABLE_IPV4
        else if (info->addressType == chip::Inet::kIPAddressType_IPv4)
        {
            err = info->udp->SendTo(mIpv4BroadcastAddress, port, info->udp->GetBoundInterface(), std::move(copy));
        }
#endif
        else
        {
            return CHIP_ERROR_INCORRECT_STATE;
        }

        if (err == chip::System::MapErrorPOSIX(ENETUNREACH))
        {
            // Send attempted to an unreachable network. Generally should not happen if
            // interfaces are configured properly, however such a failure to broadcast
            // may not be critical either.
            ChipLogError(Discovery, "Attempt to mDNS broadcast to an unreachable destination.");
        }
        else if (err != CHIP_NO_ERROR)
        {
            return err;
        }
    }

    return CHIP_NO_ERROR;
}

void ServerBase::OnUdpPacketReceived(chip::Inet::IPEndPointBasis * endPoint, chip::System::PacketBufferHandle buffer,
                                     const chip::Inet::IPPacketInfo * info)
{
    ServerBase * srv = static_cast<ServerBase *>(endPoint->AppState);
    if (!srv->mDelegate)
    {
        return;
    }

    mdns::Minimal::BytesRange data(buffer->Start(), buffer->Start() + buffer->DataLength());
    if (data.Size() < HeaderRef::kSizeBytes)
    {
        ChipLogError(Discovery, "Packet to small for mDNS data: %d bytes", static_cast<int>(data.Size()));
        return;
    }

    if (HeaderRef(const_cast<uint8_t *>(data.Start())).GetFlags().IsQuery())
    {
        srv->mDelegate->OnQuery(data, info);
    }
    else
    {
        srv->mDelegate->OnResponse(data, info);
    }
}

} // namespace Minimal
} // namespace mdns
<commit_msg>Join multicast groups for mDNS. (#4974)<commit_after>/*
 *
 *    Copyright (c) 2020 Project CHIP Authors
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

#include "Server.h"

#include <errno.h>
#include <utility>

#include <mdns/minimal/core/DnsHeader.h>
#include <support/ReturnMacros.h>

namespace mdns {
namespace Minimal {
namespace {

class ShutdownOnError
{
public:
    ShutdownOnError(ServerBase * s) : mServer(s) {}
    ~ShutdownOnError()
    {
        if (mServer != nullptr)
        {
            mServer->Shutdown();
        }
    }

    CHIP_ERROR ReturnSuccess()
    {
        mServer = nullptr;
        return CHIP_NO_ERROR;
    }

private:
    ServerBase * mServer;
};

} // namespace

namespace BroadcastIpAddresses {

// Get standard mDNS Broadcast addresses

void GetIpv6Into(chip::Inet::IPAddress & dest)
{
    if (!chip::Inet::IPAddress::FromString("FF02::FB", dest))
    {
        ChipLogError(Discovery, "Failed to parse standard IPv6 broadcast address");
    }
}

void GetIpv4Into(chip::Inet::IPAddress & dest)
{
    if (!chip::Inet::IPAddress::FromString("224.0.0.251", dest))
    {
        ChipLogError(Discovery, "Failed to parse standard IPv4 broadcast address");
    }
}

} // namespace BroadcastIpAddresses

namespace {

CHIP_ERROR JoinMulticastGroup(chip::Inet::InterfaceId interfaceId, chip::Inet::UDPEndPoint * endpoint,
                              chip::Inet::IPAddressType addressType)
{

    chip::Inet::IPAddress address;

    if (addressType == chip::Inet::IPAddressType::kIPAddressType_IPv6)
    {
        BroadcastIpAddresses::GetIpv6Into(address);
#if INET_CONFIG_ENABLE_IPV4
    }
    else if (addressType == chip::Inet::IPAddressType::kIPAddressType_IPv4)
    {
        BroadcastIpAddresses::GetIpv4Into(address);
#endif // INET_CONFIG_ENABLE_IPV4
    }
    else
    {
        return CHIP_ERROR_INVALID_ARGUMENT;
    }

    return endpoint->JoinMulticastGroup(interfaceId, address);
}

const char * AddressTypeStr(chip::Inet::IPAddressType addressType)
{
    switch (addressType)
    {
    case chip::Inet::IPAddressType::kIPAddressType_IPv6:
        return "IPv6";
#if INET_CONFIG_ENABLE_IPV4
    case chip::Inet::IPAddressType::kIPAddressType_IPv4:
        return "IPv4";
#endif // INET_CONFIG_ENABLE_IPV4
    default:
        return "UNKNOWN";
    }
}

} // namespace

ServerBase::~ServerBase()
{
    Shutdown();
}

void ServerBase::Shutdown()
{
    for (size_t i = 0; i < mEndpointCount; i++)
    {
        if (mEndpoints[i].udp != nullptr)
        {
            mEndpoints[i].udp->Close();
            mEndpoints[i].udp = nullptr;
        }
    }
}

CHIP_ERROR ServerBase::Listen(chip::Inet::InetLayer * inetLayer, ListenIterator * it, uint16_t port)
{
    Shutdown(); // ensure everything starts fresh

    size_t endpointIndex                = 0;
    chip::Inet::InterfaceId interfaceId = INET_NULL_INTERFACEID;
    chip::Inet::IPAddressType addressType;

    ShutdownOnError autoShutdown(this);

    while (it->Next(&interfaceId, &addressType))
    {
        ReturnErrorCodeIf(endpointIndex >= mEndpointCount, CHIP_ERROR_NO_MEMORY);

        EndpointInfo * info = &mEndpoints[endpointIndex];
        info->addressType   = addressType;
        info->interfaceId   = interfaceId;

        ReturnErrorOnFailure(inetLayer->NewUDPEndPoint(&info->udp));

        ReturnErrorOnFailure(info->udp->Bind(addressType, chip::Inet::IPAddress::Any, port, interfaceId));

        info->udp->AppState          = static_cast<void *>(this);
        info->udp->OnMessageReceived = OnUdpPacketReceived;

        ReturnErrorOnFailure(info->udp->Listen());

        CHIP_ERROR err = JoinMulticastGroup(interfaceId, info->udp, addressType);
        if (err != CHIP_NO_ERROR)
        {
            char interfaceName[chip::Inet::InterfaceIterator::kMaxIfNameLength];
            chip::Inet::GetInterfaceName(interfaceId, interfaceName, sizeof(interfaceName));

            // Log only as non-fatal error. Failure to join will mean we reply to unicast queries only.
            ChipLogError(DeviceLayer, "MDNS failed to join multicast group on %s for address type %s: %s", interfaceName,
                         AddressTypeStr(addressType), chip::ErrorStr(err));
        }

        endpointIndex++;
    }

    return autoShutdown.ReturnSuccess();
}

CHIP_ERROR ServerBase::DirectSend(chip::System::PacketBufferHandle && data, const chip::Inet::IPAddress & addr, uint16_t port,
                                  chip::Inet::InterfaceId interface)
{
    for (size_t i = 0; i < mEndpointCount; i++)
    {
        EndpointInfo * info = &mEndpoints[i];
        if (info->udp == nullptr)
        {
            continue;
        }

        if (info->addressType != addr.Type())
        {
            continue;
        }

        chip::Inet::InterfaceId boundIf = info->udp->GetBoundInterface();

        if ((boundIf != INET_NULL_INTERFACEID) && (boundIf != interface))
        {
            continue;
        }

        return info->udp->SendTo(addr, port, std::move(data));
    }

    return CHIP_ERROR_NOT_CONNECTED;
}

CHIP_ERROR ServerBase::BroadcastSend(chip::System::PacketBufferHandle data, uint16_t port, chip::Inet::InterfaceId interface)
{
    for (size_t i = 0; i < mEndpointCount; i++)
    {
        EndpointInfo * info = &mEndpoints[i];

        if (info->udp == nullptr)
        {
            continue;
        }

        if ((info->udp->GetBoundInterface() != interface) && (info->udp->GetBoundInterface() != INET_NULL_INTERFACEID))
        {
            continue;
        }

        CHIP_ERROR err;

        /// The same packet needs to be sent over potentially multiple interfaces.
        /// LWIP does not like having a pbuf sent over serparate interfaces, hence we create a copy
        /// TODO: this wastes one copy of the data and that could be optimized away
        chip::System::PacketBufferHandle copy = data.CloneData();

        if (info->addressType == chip::Inet::kIPAddressType_IPv6)
        {
            err = info->udp->SendTo(mIpv6BroadcastAddress, port, info->udp->GetBoundInterface(), std::move(copy));
        }
#if INET_CONFIG_ENABLE_IPV4
        else if (info->addressType == chip::Inet::kIPAddressType_IPv4)
        {
            err = info->udp->SendTo(mIpv4BroadcastAddress, port, info->udp->GetBoundInterface(), std::move(copy));
        }
#endif
        else
        {
            return CHIP_ERROR_INCORRECT_STATE;
        }

        if (err != CHIP_NO_ERROR)
        {
            return err;
        }
    }

    return CHIP_NO_ERROR;
}

CHIP_ERROR ServerBase::BroadcastSend(chip::System::PacketBufferHandle data, uint16_t port)
{
    for (size_t i = 0; i < mEndpointCount; i++)
    {
        EndpointInfo * info = &mEndpoints[i];

        if (info->udp == nullptr)
        {
            continue;
        }

        CHIP_ERROR err;

        /// The same packet needs to be sent over potentially multiple interfaces.
        /// LWIP does not like having a pbuf sent over serparate interfaces, hence we create a copy
        /// TODO: this wastes one copy of the data and that could be optimized away
        chip::System::PacketBufferHandle copy = data.CloneData();

        if (info->addressType == chip::Inet::kIPAddressType_IPv6)
        {
            err = info->udp->SendTo(mIpv6BroadcastAddress, port, info->udp->GetBoundInterface(), std::move(copy));
        }
#if INET_CONFIG_ENABLE_IPV4
        else if (info->addressType == chip::Inet::kIPAddressType_IPv4)
        {
            err = info->udp->SendTo(mIpv4BroadcastAddress, port, info->udp->GetBoundInterface(), std::move(copy));
        }
#endif
        else
        {
            return CHIP_ERROR_INCORRECT_STATE;
        }

        if (err == chip::System::MapErrorPOSIX(ENETUNREACH))
        {
            // Send attempted to an unreachable network. Generally should not happen if
            // interfaces are configured properly, however such a failure to broadcast
            // may not be critical either.
            ChipLogError(Discovery, "Attempt to mDNS broadcast to an unreachable destination.");
        }
        else if (err != CHIP_NO_ERROR)
        {
            return err;
        }
    }

    return CHIP_NO_ERROR;
}

void ServerBase::OnUdpPacketReceived(chip::Inet::IPEndPointBasis * endPoint, chip::System::PacketBufferHandle buffer,
                                     const chip::Inet::IPPacketInfo * info)
{
    ServerBase * srv = static_cast<ServerBase *>(endPoint->AppState);
    if (!srv->mDelegate)
    {
        return;
    }

    mdns::Minimal::BytesRange data(buffer->Start(), buffer->Start() + buffer->DataLength());
    if (data.Size() < HeaderRef::kSizeBytes)
    {
        ChipLogError(Discovery, "Packet to small for mDNS data: %d bytes", static_cast<int>(data.Size()));
        return;
    }

    if (HeaderRef(const_cast<uint8_t *>(data.Start())).GetFlags().IsQuery())
    {
        srv->mDelegate->OnQuery(data, info);
    }
    else
    {
        srv->mDelegate->OnResponse(data, info);
    }
}

} // namespace Minimal
} // namespace mdns
<|endoftext|>


<commit_before>#include "optimizer.hpp"

#include <memory>
#include <unordered_set>

#include "cost_estimation/cost_estimator_logical.hpp"
#include "expression/expression_utils.hpp"
#include "expression/lqp_subquery_expression.hpp"
#include "logical_query_plan/join_node.hpp"
#include "logical_query_plan/logical_plan_root_node.hpp"
#include "logical_query_plan/lqp_utils.hpp"
#include "logical_query_plan/predicate_node.hpp"
#include "strategy/between_composition_rule.hpp"
#include "strategy/chunk_pruning_rule.hpp"
#include "strategy/column_pruning_rule.hpp"
#include "strategy/expression_reduction_rule.hpp"
#include "strategy/in_expression_rewrite_rule.hpp"
#include "strategy/index_scan_rule.hpp"
#include "strategy/insert_limit_in_exists_rule.hpp"
#include "strategy/join_ordering_rule.hpp"
#include "strategy/join_predicate_ordering_rule.hpp"
#include "strategy/predicate_placement_rule.hpp"
#include "strategy/predicate_reordering_rule.hpp"
#include "strategy/predicate_split_up_rule.hpp"
#include "strategy/subquery_to_join_rule.hpp"
#include "utils/performance_warning.hpp"

/**
 * IMPORTANT NOTES ON OPTIMIZING SUBQUERY LQPS
 *
 * Multiple Expressions in different nodes might reference the same LQP. Most commonly this will be the case for a
 * ProjectionNode computing a subquery and a subsequent PredicateNode filtering based on it.
 * We do not WANT to optimize the LQP twice (optimization takes time after all) and we CANNOT optimize it twice, since,
 * e.g., a non-deterministic rule, could produce two different LQPs while optimizing and then the SubqueryExpression
 * in the PredicateNode couldn't be resolved to a column anymore. There are more subtle ways LQPs might break in this
 * scenario, and frankly, this is one of the weak links in the expression system...
 *
 * ...long story short:
 * !!!
 * EACH UNIQUE SUB-LQP IS ONLY OPTIMIZED ONCE, EVEN IF IT OCCURS IN DIFFERENT NODES/EXPRESSIONS.
 * !!!
 *
 * -> collect_subquery_expressions_by_lqp() identifies unique LQPs and the (multiple) SubqueryExpressions referencing
 *                                          each of these unique LQPs.
 *
 * -> Optimizer::_apply_rule()              optimizes each unique LQP exactly once and assigns the optimized LQPs back
 *                                          to the SubqueryExpressions referencing them.
 */

namespace {

using namespace opossum;  // NOLINT

// All SubqueryExpressions referencing the same LQP
using SubqueryExpressionsByLQP =
    std::vector<std::pair<std::shared_ptr<AbstractLQPNode>, std::vector<std::shared_ptr<LQPSubqueryExpression>>>>;

// See comment at the top of file for the purpose of this.
void collect_subquery_expressions_by_lqp(SubqueryExpressionsByLQP& subquery_expressions_by_lqp,
                                         const std::shared_ptr<AbstractLQPNode>& node,
                                         std::unordered_set<std::shared_ptr<AbstractLQPNode>>& visited_nodes) {
  if (!node) return;
  if (!visited_nodes.emplace(node).second) return;

  for (const auto& expression : node->node_expressions) {
    visit_expression(expression, [&](const auto& sub_expression) {
      const auto subquery_expression = std::dynamic_pointer_cast<LQPSubqueryExpression>(sub_expression);
      if (!subquery_expression) return ExpressionVisitation::VisitArguments;

      for (auto& [lqp, subquery_expressions] : subquery_expressions_by_lqp) {
        if (*lqp == *subquery_expression->lqp) {
          subquery_expressions.emplace_back(subquery_expression);
          return ExpressionVisitation::DoNotVisitArguments;
        }
      }
      subquery_expressions_by_lqp.emplace_back(subquery_expression->lqp, std::vector{subquery_expression});

      return ExpressionVisitation::DoNotVisitArguments;
    });
  }

  collect_subquery_expressions_by_lqp(subquery_expressions_by_lqp, node->left_input(), visited_nodes);
  collect_subquery_expressions_by_lqp(subquery_expressions_by_lqp, node->right_input(), visited_nodes);
}

}  // namespace

namespace opossum {

std::shared_ptr<Optimizer> Optimizer::create_default_optimizer() {
  const auto optimizer = std::make_shared<Optimizer>();

  optimizer->add_rule(std::make_unique<ExpressionReductionRule>());

  optimizer->add_rule(std::make_unique<PredicateSplitUpRule>());

  optimizer->add_rule(std::make_unique<ColumnPruningRule>());

  optimizer->add_rule(std::make_unique<ChunkPruningRule>());

  // Run before SubqueryToJoinRule, since the Semi/Anti Joins it introduces are opaque to the JoinOrderingRule
  optimizer->add_rule(std::make_unique<JoinOrderingRule>());

  optimizer->add_rule(std::make_unique<BetweenCompositionRule>());

  // Position the predicates after the JoinOrderingRule ran. The JOR manipulates predicate placement as well, but
  // for now we want the PredicateReorderingRule to have the final say on predicate positions
  optimizer->add_rule(std::make_unique<PredicatePlacementRule>());

  optimizer->add_rule(std::make_unique<SubqueryToJoinRule>());

  optimizer->add_rule(std::make_unique<JoinPredicateOrderingRule>());

  optimizer->add_rule(std::make_unique<InsertLimitInExistsRule>());

  // Bring predicates into the desired order once the PredicatePlacementRule has positioned them as desired
  optimizer->add_rule(std::make_unique<PredicateReorderingRule>());

  // Before the IN predicate is rewritten, it should have been moved to a good position. Also, while the IN predicate
  // might become a join, it is semantically more similar to a predicate. If we run this rule too early, it might
  // hinder other optimizations that stop at joins. For example, the join ordering currently does not know about semi
  // joins and would not recognize such a rewritten predicate.
  optimizer->add_rule(std::make_unique<InExpressionRewriteRule>());

  optimizer->add_rule(std::make_unique<IndexScanRule>());

  return optimizer;
}

Optimizer::Optimizer(const std::shared_ptr<AbstractCostEstimator>& cost_estimator) : _cost_estimator(cost_estimator) {}

void Optimizer::add_rule(std::unique_ptr<AbstractRule> rule) {
  rule->cost_estimator = _cost_estimator;
  _rules.emplace_back(std::move(rule));
}

std::shared_ptr<AbstractLQPNode> Optimizer::optimize(const std::shared_ptr<AbstractLQPNode>& input) const {
  // Add explicit root node, so the rules can freely change the tree below it without having to maintain a root node
  // to return to the Optimizer
  const auto root_node = LogicalPlanRootNode::make(input);

  for (const auto& rule : _rules) {
    _apply_rule(*rule, root_node);
  }

  // Remove LogicalPlanRootNode
  const auto optimized_node = root_node->left_input();
  root_node->set_left_input(nullptr);

  return optimized_node;
}

void Optimizer::_apply_rule(const AbstractRule& rule, const std::shared_ptr<AbstractLQPNode>& root_node) const {
  rule.apply_to(root_node);

  /**
   * Optimize Subqueries
   */
  auto subquery_expressions_by_lqp = SubqueryExpressionsByLQP{};
  auto visited_nodes = std::unordered_set<std::shared_ptr<AbstractLQPNode>>{};
  collect_subquery_expressions_by_lqp(subquery_expressions_by_lqp, root_node, visited_nodes);

  for (const auto& [lqp, subquery_expressions] : subquery_expressions_by_lqp) {
    const auto local_root_node = LogicalPlanRootNode::make(lqp);
    _apply_rule(rule, local_root_node);
    for (const auto& subquery_expression : subquery_expressions) {
      subquery_expression->lqp = local_root_node->left_input();
    }
  }
}

}  // namespace opossum
<commit_msg>Run chunk pruning after between composition (#1855)<commit_after>#include "optimizer.hpp"

#include <memory>
#include <unordered_set>

#include "cost_estimation/cost_estimator_logical.hpp"
#include "expression/expression_utils.hpp"
#include "expression/lqp_subquery_expression.hpp"
#include "logical_query_plan/join_node.hpp"
#include "logical_query_plan/logical_plan_root_node.hpp"
#include "logical_query_plan/lqp_utils.hpp"
#include "logical_query_plan/predicate_node.hpp"
#include "strategy/between_composition_rule.hpp"
#include "strategy/chunk_pruning_rule.hpp"
#include "strategy/column_pruning_rule.hpp"
#include "strategy/expression_reduction_rule.hpp"
#include "strategy/in_expression_rewrite_rule.hpp"
#include "strategy/index_scan_rule.hpp"
#include "strategy/insert_limit_in_exists_rule.hpp"
#include "strategy/join_ordering_rule.hpp"
#include "strategy/join_predicate_ordering_rule.hpp"
#include "strategy/predicate_placement_rule.hpp"
#include "strategy/predicate_reordering_rule.hpp"
#include "strategy/predicate_split_up_rule.hpp"
#include "strategy/subquery_to_join_rule.hpp"
#include "utils/performance_warning.hpp"

/**
 * IMPORTANT NOTES ON OPTIMIZING SUBQUERY LQPS
 *
 * Multiple Expressions in different nodes might reference the same LQP. Most commonly this will be the case for a
 * ProjectionNode computing a subquery and a subsequent PredicateNode filtering based on it.
 * We do not WANT to optimize the LQP twice (optimization takes time after all) and we CANNOT optimize it twice, since,
 * e.g., a non-deterministic rule, could produce two different LQPs while optimizing and then the SubqueryExpression
 * in the PredicateNode couldn't be resolved to a column anymore. There are more subtle ways LQPs might break in this
 * scenario, and frankly, this is one of the weak links in the expression system...
 *
 * ...long story short:
 * !!!
 * EACH UNIQUE SUB-LQP IS ONLY OPTIMIZED ONCE, EVEN IF IT OCCURS IN DIFFERENT NODES/EXPRESSIONS.
 * !!!
 *
 * -> collect_subquery_expressions_by_lqp() identifies unique LQPs and the (multiple) SubqueryExpressions referencing
 *                                          each of these unique LQPs.
 *
 * -> Optimizer::_apply_rule()              optimizes each unique LQP exactly once and assigns the optimized LQPs back
 *                                          to the SubqueryExpressions referencing them.
 */

namespace {

using namespace opossum;  // NOLINT

// All SubqueryExpressions referencing the same LQP
using SubqueryExpressionsByLQP =
    std::vector<std::pair<std::shared_ptr<AbstractLQPNode>, std::vector<std::shared_ptr<LQPSubqueryExpression>>>>;

// See comment at the top of file for the purpose of this.
void collect_subquery_expressions_by_lqp(SubqueryExpressionsByLQP& subquery_expressions_by_lqp,
                                         const std::shared_ptr<AbstractLQPNode>& node,
                                         std::unordered_set<std::shared_ptr<AbstractLQPNode>>& visited_nodes) {
  if (!node) return;
  if (!visited_nodes.emplace(node).second) return;

  for (const auto& expression : node->node_expressions) {
    visit_expression(expression, [&](const auto& sub_expression) {
      const auto subquery_expression = std::dynamic_pointer_cast<LQPSubqueryExpression>(sub_expression);
      if (!subquery_expression) return ExpressionVisitation::VisitArguments;

      for (auto& [lqp, subquery_expressions] : subquery_expressions_by_lqp) {
        if (*lqp == *subquery_expression->lqp) {
          subquery_expressions.emplace_back(subquery_expression);
          return ExpressionVisitation::DoNotVisitArguments;
        }
      }
      subquery_expressions_by_lqp.emplace_back(subquery_expression->lqp, std::vector{subquery_expression});

      return ExpressionVisitation::DoNotVisitArguments;
    });
  }

  collect_subquery_expressions_by_lqp(subquery_expressions_by_lqp, node->left_input(), visited_nodes);
  collect_subquery_expressions_by_lqp(subquery_expressions_by_lqp, node->right_input(), visited_nodes);
}

}  // namespace

namespace opossum {

std::shared_ptr<Optimizer> Optimizer::create_default_optimizer() {
  const auto optimizer = std::make_shared<Optimizer>();

  optimizer->add_rule(std::make_unique<ExpressionReductionRule>());

  optimizer->add_rule(std::make_unique<PredicateSplitUpRule>());

  optimizer->add_rule(std::make_unique<ColumnPruningRule>());

  // Run before SubqueryToJoinRule, since the Semi/Anti Joins it introduces are opaque to the JoinOrderingRule
  optimizer->add_rule(std::make_unique<JoinOrderingRule>());

  optimizer->add_rule(std::make_unique<BetweenCompositionRule>());

  // Position the predicates after the JoinOrderingRule ran. The JOR manipulates predicate placement as well, but
  // for now we want the PredicateReorderingRule to have the final say on predicate positions
  optimizer->add_rule(std::make_unique<PredicatePlacementRule>());

  optimizer->add_rule(std::make_unique<SubqueryToJoinRule>());

  optimizer->add_rule(std::make_unique<JoinPredicateOrderingRule>());

  optimizer->add_rule(std::make_unique<InsertLimitInExistsRule>());

  // Prune chunks after the BetweenCompositionRule ran, as `a >= 5 AND a <= 7` may not be prunable predicates while
  // `a BETWEEN 5 and 7` is. Also, run it after the PredicatePlacementRule, so that predicates are as close to the
  // StoredTableNode as possible where the ChunkPruningRule can work with them.
  optimizer->add_rule(std::make_unique<ChunkPruningRule>());

  // Bring predicates into the desired order once the PredicatePlacementRule has positioned them as desired
  optimizer->add_rule(std::make_unique<PredicateReorderingRule>());

  // Before the IN predicate is rewritten, it should have been moved to a good position. Also, while the IN predicate
  // might become a join, it is semantically more similar to a predicate. If we run this rule too early, it might
  // hinder other optimizations that stop at joins. For example, the join ordering currently does not know about semi
  // joins and would not recognize such a rewritten predicate.
  optimizer->add_rule(std::make_unique<InExpressionRewriteRule>());

  optimizer->add_rule(std::make_unique<IndexScanRule>());

  return optimizer;
}

Optimizer::Optimizer(const std::shared_ptr<AbstractCostEstimator>& cost_estimator) : _cost_estimator(cost_estimator) {}

void Optimizer::add_rule(std::unique_ptr<AbstractRule> rule) {
  rule->cost_estimator = _cost_estimator;
  _rules.emplace_back(std::move(rule));
}

std::shared_ptr<AbstractLQPNode> Optimizer::optimize(const std::shared_ptr<AbstractLQPNode>& input) const {
  // Add explicit root node, so the rules can freely change the tree below it without having to maintain a root node
  // to return to the Optimizer
  const auto root_node = LogicalPlanRootNode::make(input);

  for (const auto& rule : _rules) {
    _apply_rule(*rule, root_node);
  }

  // Remove LogicalPlanRootNode
  const auto optimized_node = root_node->left_input();
  root_node->set_left_input(nullptr);

  return optimized_node;
}

void Optimizer::_apply_rule(const AbstractRule& rule, const std::shared_ptr<AbstractLQPNode>& root_node) const {
  rule.apply_to(root_node);

  /**
   * Optimize Subqueries
   */
  auto subquery_expressions_by_lqp = SubqueryExpressionsByLQP{};
  auto visited_nodes = std::unordered_set<std::shared_ptr<AbstractLQPNode>>{};
  collect_subquery_expressions_by_lqp(subquery_expressions_by_lqp, root_node, visited_nodes);

  for (const auto& [lqp, subquery_expressions] : subquery_expressions_by_lqp) {
    const auto local_root_node = LogicalPlanRootNode::make(lqp);
    _apply_rule(rule, local_root_node);
    for (const auto& subquery_expression : subquery_expressions) {
      subquery_expression->lqp = local_root_node->left_input();
    }
  }
}

}  // namespace opossum
<|endoftext|>