<commit_before>//
// This file is part of the Marble Desktop Globe.
//
// This program is free software licensed under the GNU LGPL. You can
// find a copy of this license in LICENSE.txt in the top directory of
// the source code.
//
// Copyright 2007      Carlos Licea     <carlos _licea@hotmail.com>
//

#include"FlatScanlineTextureMapper.h"

#include <cmath>

#include <QtCore/QDebug>

#include "GeoDataPoint.h"
#include "GeoPolygon.h"
#include "MarbleDirs.h"
#include "TextureTile.h"
#include "TileLoader.h"

FlatScanlineTextureMapper::FlatScanlineTextureMapper( TileLoader *tileLoader, QObject * parent )
    : AbstractScanlineTextureMapper( tileLoader, parent )
{
    m_oldCenterLon   = 0.0;
    m_oldYPaintedTop = 0;
}

void FlatScanlineTextureMapper::mapTexture( ViewParams *viewParams )
{
    QImage* canvasImage = viewParams->m_canvasImage;
    const int radius = viewParams->m_radius;

   // Initialize needed variables:
    double lon = 0.0;
    double lat = 0.0;

    m_tilePosX = 65535;
    m_tilePosY = 65535;

    m_toTileCoordinatesLon = (double)(m_tileLoader->globalWidth( m_tileLevel )
                             / 2 - m_tilePosX);
    m_toTileCoordinatesLat = (double)(m_tileLoader->globalHeight( m_tileLevel )
                             / 2 - m_tilePosY);

    int yTop;
    int yPaintedTop;
    int yPaintedBottom;

    // Reset backend
    m_tileLoader->resetTilehash();
    selectTileLevel( viewParams );

    // Calculate translation of center point
    double centerLon, centerLat;
    viewParams->centerCoordinates( centerLon, centerLat );

    int yCenterOffset =  (int)((float)( 2 * radius / M_PI) * centerLat );

    //Calculate y-range the represented by the center point, yTop and yBottom
    //and what actually can be painted
    yPaintedTop    = yTop = m_imageHeight / 2 - 2 * radius + yCenterOffset;
    yPaintedBottom        = m_imageHeight / 2 + 2 * radius + yCenterOffset;

    if (yPaintedTop < 0)                yPaintedTop = 0;
    if (yPaintedTop > m_imageHeight)    yPaintedTop = m_imageHeight;
    if (yPaintedBottom < 0)             yPaintedBottom = 0;
    if (yPaintedBottom > m_imageHeight) yPaintedBottom = m_imageHeight;

    // Calculate how many degrees are being represented per pixel.
    const float rad2Pixel = M_PI / (float)( 2 * radius );

    float leftLon = + centerLon - ( rad2Pixel * m_imageWidth / 2 );
    while ( leftLon < -M_PI ) leftLon += 2 * M_PI;
    while ( leftLon >  M_PI ) leftLon -= 2 * M_PI;
    // Paint the map.
    for ( int y = yPaintedTop ;y < yPaintedBottom; ++y ) {
        if( viewParams->m_projection == Equirectangular )
            lat = M_PI/2 - (y - yTop )* rad2Pixel;
        else if( viewParams->m_projection == Mercator )
            lat = atan( sinh( ((m_imageHeight / 2 + yCenterOffset) - y) / (double)(2 * radius) * M_PI ) );
        m_scanLine = (QRgb*)( canvasImage->scanLine( y ) );
        lon = leftLon;

        QRgb * scanLineBegin = m_scanLine;
        const QRgb * scanLineEnd   = m_scanLine + m_imageWidth;

        for ( QRgb * scanLine = scanLineBegin; scanLine < scanLineEnd;
              ++scanLine ) {
            lon += rad2Pixel;
            if ( lon < -M_PI ) lon += 2 * M_PI;
            if ( lon >  M_PI ) lon -= 2 * M_PI;
            pixelValue( lon, lat, scanLine );
        }
    }

    // Remove unused lines
    const int clearStart = ( yPaintedTop - m_oldYPaintedTop <= 0 ) ? yPaintedBottom : 0;
    const int clearStop  = ( yPaintedTop - m_oldYPaintedTop <= 0 ) ? m_imageHeight  : yTop;

    for ( int y = clearStart; y < clearStop; ++y ) {
        m_scanLine = (QRgb*)( canvasImage->scanLine( y ) );
        for ( int x = 0; x < m_imageWidth; ++x ) {
            *(m_scanLine + x) = 0;
        }
    }
    m_oldYPaintedTop = yPaintedTop;

    m_tileLoader->cleanupTilehash();
}

#include "FlatScanlineTextureMapper.moc"
<commit_msg>- Fixing last commit<commit_after>//
// This file is part of the Marble Desktop Globe.
//
// This program is free software licensed under the GNU LGPL. You can
// find a copy of this license in LICENSE.txt in the top directory of
// the source code.
//
// Copyright 2007      Carlos Licea     <carlos _licea@hotmail.com>
//

#include"FlatScanlineTextureMapper.h"

#include <cmath>

#include <QtCore/QDebug>

#include "GeoDataPoint.h"
#include "GeoPolygon.h"
#include "MarbleDirs.h"
#include "TextureTile.h"
#include "TileLoader.h"

FlatScanlineTextureMapper::FlatScanlineTextureMapper( TileLoader *tileLoader, QObject * parent )
    : AbstractScanlineTextureMapper( tileLoader, parent )
{
    m_oldCenterLon   = 0.0;
    m_oldYPaintedTop = 0;
}

void FlatScanlineTextureMapper::mapTexture( ViewParams *viewParams )
{
    QImage* canvasImage = viewParams->m_canvasImage;
    const int radius = viewParams->m_radius;

   // Initialize needed variables:
    double lon = 0.0;
    double lat = 0.0;

    m_tilePosX = 65535;
    m_tilePosY = 65535;

    m_toTileCoordinatesLon = (double)(m_tileLoader->globalWidth( m_tileLevel )
                             / 2 - m_tilePosX);
    m_toTileCoordinatesLat = (double)(m_tileLoader->globalHeight( m_tileLevel )
                             / 2 - m_tilePosY);

    int yTop;
    int yPaintedTop;
    int yPaintedBottom;

    // Reset backend
    m_tileLoader->resetTilehash();
    selectTileLevel( viewParams );

    // Calculate translation of center point
    double centerLon, centerLat;
    viewParams->centerCoordinates( centerLon, centerLat );

    int yCenterOffset =  (int)((float)( 2 * radius / M_PI) * centerLat );

    //Calculate y-range the represented by the center point, yTop and yBottom
    //and what actually can be painted
    if( viewParams->m_projection == Equirectangular ) {
        yPaintedTop    = yTop = m_imageHeight / 2 - radius + yCenterOffset;
        yPaintedBottom        = m_imageHeight / 2 + radius + yCenterOffset;
    }
    else if( viewParams->m_projection == Mercator ) {
        yPaintedTop    = yTop = m_imageHeight / 2 - 2 * radius + yCenterOffset;
        yPaintedBottom        = m_imageHeight / 2 + 2 * radius + yCenterOffset;
    }

    if (yPaintedTop < 0)                yPaintedTop = 0;
    if (yPaintedTop > m_imageHeight)    yPaintedTop = m_imageHeight;
    if (yPaintedBottom < 0)             yPaintedBottom = 0;
    if (yPaintedBottom > m_imageHeight) yPaintedBottom = m_imageHeight;

    // Calculate how many degrees are being represented per pixel.
    const float rad2Pixel = M_PI / (float)( 2 * radius );

    float leftLon = + centerLon - ( rad2Pixel * m_imageWidth / 2 );
    while ( leftLon < -M_PI ) leftLon += 2 * M_PI;
    while ( leftLon >  M_PI ) leftLon -= 2 * M_PI;
    // Paint the map.
    for ( int y = yPaintedTop ;y < yPaintedBottom; ++y ) {
        if( viewParams->m_projection == Equirectangular )
            lat = M_PI/2 - (y - yTop )* rad2Pixel;
        else if( viewParams->m_projection == Mercator )
            lat = atan( sinh( ((m_imageHeight / 2 + yCenterOffset) - y) / (double)(2 * radius) * M_PI ) );
        m_scanLine = (QRgb*)( canvasImage->scanLine( y ) );
        lon = leftLon;

        QRgb * scanLineBegin = m_scanLine;
        const QRgb * scanLineEnd   = m_scanLine + m_imageWidth;

        for ( QRgb * scanLine = scanLineBegin; scanLine < scanLineEnd;
              ++scanLine ) {
            lon += rad2Pixel;
            if ( lon < -M_PI ) lon += 2 * M_PI;
            if ( lon >  M_PI ) lon -= 2 * M_PI;
            pixelValue( lon, lat, scanLine );
        }
    }

    // Remove unused lines
    const int clearStart = ( yPaintedTop - m_oldYPaintedTop <= 0 ) ? yPaintedBottom : 0;
    const int clearStop  = ( yPaintedTop - m_oldYPaintedTop <= 0 ) ? m_imageHeight  : yTop;

    for ( int y = clearStart; y < clearStop; ++y ) {
        m_scanLine = (QRgb*)( canvasImage->scanLine( y ) );
        for ( int x = 0; x < m_imageWidth; ++x ) {
            *(m_scanLine + x) = 0;
        }
    }
    m_oldYPaintedTop = yPaintedTop;

    m_tileLoader->cleanupTilehash();
}

#include "FlatScanlineTextureMapper.moc"
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-base/util/SimpleRateLimit.h"
#include "fnord-base/InternMap.h"
#include "fnord-json/json.h"
#include "fnord-mdb/MDB.h"
#include "fnord-mdb/MDBUtil.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-sstable/sstablewriter.h"
#include "fnord-sstable/SSTableColumnSchema.h"
#include "fnord-sstable/SSTableColumnReader.h"
#include "fnord-sstable/SSTableColumnWriter.h"
#include "common.h"
#include "PackedExample.h"

using namespace fnord;

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_data_file",
      fnord::cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "output file",
      "<filename>");

  flags.defineFlag(
      "output_meta_file",
      fnord::cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "output file",
      "<filename>");

  flags.defineFlag(
      "min_observations",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "100",
      "minimum nuber of feature observations",
      "<num>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto sstables = flags.getArgv();
  auto tbl_cnt = sstables.size();
  if (tbl_cnt == 0) {
    fnord::logCritical("cm.featuredump", "No input tables, exiting");
    return 1;
  }

  /* open output files */
  auto output_data_file_path = flags.getString("output_data_file");
  auto output_meta_file_path = flags.getString("output_meta_file");

  fnord::logInfo(
      "cm.featureprep",
      "Writing features to: $0",
      output_data_file_path);

  fnord::logInfo(
      "cm.featureprep",
      "Writing feature metadata to: $0",
      output_meta_file_path);

  if (FileUtil::exists(output_meta_file_path + "~")) {
    fnord::logInfo(
        "cm.featuredump",
        "Deleting orphaned tmp file: $0",
        output_meta_file_path + "~");

    FileUtil::rm(output_meta_file_path + "~");
  }

  if (FileUtil::exists(output_data_file_path + "~")) {
    fnord::logInfo(
        "cm.featuredump",
        "Deleting orphaned tmp file: $0",
        output_data_file_path + "~");

    FileUtil::rm(output_data_file_path + "~");
  }

  auto output_data_file = File::openFile(
      output_data_file_path + "~",
      File::O_READ | File::O_WRITE | File::O_CREATE);

  auto output_meta_file = File::openFile(
      output_meta_file_path + "~",
      File::O_READ | File::O_WRITE | File::O_CREATE);

  /* read input meta tables and count features */
  HashMap<String, uint64_t> feature_counts;
  int row_idx = 0;
  for (int tbl_idx = 0; tbl_idx < sstables.size(); ++tbl_idx) {
    String sstable = sstables[tbl_idx];
    StringUtil::replaceAll(&sstable, ".sstable", "_meta.sstable");
    fnord::logInfo("cm.featuredump", "Importing meta sstable: $0", sstable);

    /* read sstable heade r*/
    sstable::SSTableReader reader(File::openFile(sstable, File::O_READ));
    sstable::SSTableColumnSchema schema;
    schema.loadIndex(&reader);

    if (reader.bodySize() == 0) {
      fnord::logCritical("cm.featuredump", "unfinished sstable: $0", sstable);
      exit(1);
    }

    /* get sstable cursor */
    auto cursor = reader.getCursor();
    auto body_size = reader.bodySize();

    /* status line */
    util::SimpleRateLimitedFn status_line(kMicrosPerSecond, [&] () {
      auto p = (tbl_idx / (double) tbl_cnt) +
          ((cursor->position() / (double) body_size)) / (double) tbl_cnt;

      fnord::logInfo(
          "cm.featuredump",
          "[$0%] Reading meta sstables... rows=$1 features=$2",
          (size_t) (p * 100),
          row_idx,
          feature_counts.size());
    });

    /* read sstable rows */
    for (; cursor->valid(); ++row_idx) {
      status_line.runMaybe();

      auto key = cursor->getKeyString();
      auto val = cursor->getDataBuffer();

      sstable::SSTableColumnReader cols(&schema, val);
      auto cnt = cols.getUInt64Column(schema.columnID("count"));

      feature_counts[key] += cnt;

      if (!cursor->next()) {
        break;
      }
    }

    status_line.runForce();
  }

  /* select features for export and assign indexes*/
  HashMap<String, uint64_t> feature_idx;
  uint64_t cur_idx = 0;
  auto orig_count = feature_counts.size();
  auto min_count = flags.getInt("min_observations");
  for (auto iter = feature_counts.begin(); iter != feature_counts.end(); ) {
    if (iter->second > min_count) {
      feature_idx.emplace(iter->first, ++cur_idx);
    }

    iter = feature_counts.erase(iter);
  }

  feature_counts.clear();

  fnord::logInfo(
      "cm.featuredump",
      "Exporting $1/$0 features",
      orig_count,
      feature_idx.size());

  /* write feature idx meta file */
  for (const auto& f : feature_idx) {
    output_meta_file.write(StringUtil::format("$1 $0\n", f.first, f.second));
  }

  /* read input data tables */
  int rows_read = 0;
  int rows_written = 0;
  for (int tbl_idx = 0; tbl_idx < sstables.size(); ++tbl_idx) {
    const auto& sstable = sstables[tbl_idx];
    fnord::logInfo("cm.featuredump", "Importing data sstable: $0", sstable);

    /* read sstable heade r*/
    sstable::SSTableReader reader(File::openFile(sstable, File::O_READ));
    sstable::SSTableColumnSchema schema;
    schema.loadIndex(&reader);

    if (reader.bodySize() == 0) {
      fnord::logCritical("cm.featuredump", "unfinished sstable: $0", sstable);
      exit(1);
    }

    /* get sstable cursor */
    auto cursor = reader.getCursor();
    auto body_size = reader.bodySize();

    /* status line */
    util::SimpleRateLimitedFn status_line(kMicrosPerSecond, [&] () {
      auto p = (tbl_idx / (double) tbl_cnt) +
          ((cursor->position() / (double) body_size)) / (double) tbl_cnt;

      fnord::logInfo(
          "cm.featuredump",
          "[$0%] Reading data sstables... rows_read=$1 rows_written=$2",
          (size_t) (p * 100),
          rows_read,
          rows_written);
    });

    /* read data sstables and copy features */
    for (; cursor->valid(); ++rows_read) {
      status_line.runMaybe();

      auto val = cursor->getDataBuffer();
      sstable::SSTableColumnReader cols(&schema, val);

      cm::Example ex;
      ex.label = cols.getFloatColumn(schema.columnID("label"));
      auto features = cols.getStringColumns(schema.columnID("feature"));
      for (const auto& f : features) {
        auto idx = feature_idx.find(f);
        if (idx == feature_idx.end()) {
          continue;
        }

        ex.features.emplace_back(idx->second, 1.0);
      }

      if (ex.features.size() > 0) {
        auto ex_str = exampleToSVMLight(ex);
        output_data_file.write(ex_str + "\n");
        ++rows_written;
      }

      if (!cursor->next()) {
        break;
      }
    }

    status_line.runForce();
  }

  FileUtil::mv(output_meta_file_path + "~", output_meta_file_path);
  FileUtil::mv(output_data_file_path + "~", output_data_file_path);

  return 0;
}

<commit_msg>cm-featuredump --output_lightsvm --output_meta<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-base/util/SimpleRateLimit.h"
#include "fnord-base/InternMap.h"
#include "fnord-json/json.h"
#include "fnord-mdb/MDB.h"
#include "fnord-mdb/MDBUtil.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-sstable/sstablewriter.h"
#include "fnord-sstable/SSTableColumnSchema.h"
#include "fnord-sstable/SSTableColumnReader.h"
#include "fnord-sstable/SSTableColumnWriter.h"
#include "common.h"
#include "PackedExample.h"

using namespace fnord;

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_lightsvm",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output lightsvm data",
      "<filename>");

  flags.defineFlag(
      "output_meta",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "output feature metadata",
      "<filename>");

  flags.defineFlag(
      "min_observations",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "100",
      "minimum nuber of feature observations",
      "<num>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto sstables = flags.getArgv();
  auto tbl_cnt = sstables.size();
  if (tbl_cnt == 0) {
    fnord::logCritical("cm.featuredump", "No input tables, exiting");
    return 1;
  }

  /* open output: meta file */
  Option<File> output_meta_file;
  String output_meta_file_path;

  if (flags.isSet("output_meta")) {
    output_meta_file_path = flags.getString("output_meta");

    fnord::logInfo(
        "cm.featureprep",
        "Writing feature metadata to: $0",
        output_meta_file_path);

    if (FileUtil::exists(output_meta_file_path + "~")) {
      fnord::logInfo(
          "cm.featuredump",
          "Deleting orphaned tmp file: $0",
          output_meta_file_path + "~");

      FileUtil::rm(output_meta_file_path + "~");
    }

    output_meta_file = std::move(Option<File>(std::move(
        File::openFile(
            output_meta_file_path + "~",
            File::O_READ | File::O_WRITE | File::O_CREATE))));
  }

  /* open output: lightsvm file */
  Option<File> output_lightsvm_file;
  String output_lightsvm_file_path;

  if (flags.isSet("output_lightsvm")) {
    output_lightsvm_file_path = flags.getString("output_lightsvm");

    fnord::logInfo(
        "cm.featureprep",
        "Writing lightSVM features to: $0",
        output_lightsvm_file_path);

    if (FileUtil::exists(output_lightsvm_file_path + "~")) {
      fnord::logInfo(
          "cm.featuredump",
          "Deleting orphaned tmp file: $0",
          output_lightsvm_file_path + "~");

      FileUtil::rm(output_lightsvm_file_path + "~");
    }

    output_lightsvm_file = std::move(Option<File>(std::move(
        File::openFile(
            output_lightsvm_file_path + "~",
            File::O_READ | File::O_WRITE | File::O_CREATE))));
  }

  /* read input meta tables and count features */
  HashMap<String, uint64_t> feature_counts;
  int row_idx = 0;
  for (int tbl_idx = 0; tbl_idx < sstables.size(); ++tbl_idx) {
    String sstable = sstables[tbl_idx];
    StringUtil::replaceAll(&sstable, ".sstable", "_meta.sstable");
    fnord::logInfo("cm.featuredump", "Importing meta sstable: $0", sstable);

    /* read sstable heade r*/
    sstable::SSTableReader reader(File::openFile(sstable, File::O_READ));
    sstable::SSTableColumnSchema schema;
    schema.loadIndex(&reader);

    if (reader.bodySize() == 0) {
      fnord::logCritical("cm.featuredump", "unfinished sstable: $0", sstable);
      exit(1);
    }

    /* get sstable cursor */
    auto cursor = reader.getCursor();
    auto body_size = reader.bodySize();

    /* status line */
    util::SimpleRateLimitedFn status_line(kMicrosPerSecond, [&] () {
      auto p = (tbl_idx / (double) tbl_cnt) +
          ((cursor->position() / (double) body_size)) / (double) tbl_cnt;

      fnord::logInfo(
          "cm.featuredump",
          "[$0%] Reading meta sstables... rows=$1 features=$2",
          (size_t) (p * 100),
          row_idx,
          feature_counts.size());
    });

    /* read sstable rows */
    for (; cursor->valid(); ++row_idx) {
      status_line.runMaybe();

      auto key = cursor->getKeyString();
      auto val = cursor->getDataBuffer();

      sstable::SSTableColumnReader cols(&schema, val);
      auto cnt = cols.getUInt64Column(schema.columnID("count"));

      feature_counts[key] += cnt;

      if (!cursor->next()) {
        break;
      }
    }

    status_line.runForce();
  }

  /* select features for export and assign indexes*/
  HashMap<String, uint64_t> feature_idx;
  uint64_t cur_idx = 0;
  auto orig_count = feature_counts.size();
  auto min_count = flags.getInt("min_observations");
  for (auto iter = feature_counts.begin(); iter != feature_counts.end(); ) {
    if (iter->second > min_count) {
      feature_idx.emplace(iter->first, ++cur_idx);
    }

    iter = feature_counts.erase(iter);
  }

  feature_counts.clear();

  fnord::logInfo(
      "cm.featuredump",
      "Exporting $1/$0 features",
      orig_count,
      feature_idx.size());

  /* write feature idx meta file */
  if (!output_meta_file.isEmpty()) {
    for (const auto& f : feature_idx) {
      auto line = StringUtil::format("$1 $0\n", f.first, f.second);
      output_meta_file.get().write(line);
    }
  }

  /* read input data tables */
  int rows_read = 0;
  int rows_written = 0;
  for (int tbl_idx = 0; tbl_idx < sstables.size(); ++tbl_idx) {
    const auto& sstable = sstables[tbl_idx];
    fnord::logInfo("cm.featuredump", "Importing data sstable: $0", sstable);

    /* read sstable heade r*/
    sstable::SSTableReader reader(File::openFile(sstable, File::O_READ));
    sstable::SSTableColumnSchema schema;
    schema.loadIndex(&reader);

    if (reader.bodySize() == 0) {
      fnord::logCritical("cm.featuredump", "unfinished sstable: $0", sstable);
      exit(1);
    }

    /* get sstable cursor */
    auto cursor = reader.getCursor();
    auto body_size = reader.bodySize();

    /* status line */
    util::SimpleRateLimitedFn status_line(kMicrosPerSecond, [&] () {
      auto p = (tbl_idx / (double) tbl_cnt) +
          ((cursor->position() / (double) body_size)) / (double) tbl_cnt;

      fnord::logInfo(
          "cm.featuredump",
          "[$0%] Reading data sstables... rows_read=$1 rows_written=$2",
          (size_t) (p * 100),
          rows_read,
          rows_written);
    });

    /* read data sstables and copy features */
    for (; cursor->valid(); ++rows_read) {
      status_line.runMaybe();

      auto val = cursor->getDataBuffer();
      sstable::SSTableColumnReader cols(&schema, val);

      cm::Example ex;
      ex.label = cols.getFloatColumn(schema.columnID("label"));
      auto features = cols.getStringColumns(schema.columnID("feature"));
      for (const auto& f : features) {
        auto idx = feature_idx.find(f);
        if (idx == feature_idx.end()) {
          continue;
        }

        ex.features.emplace_back(idx->second, 1.0);
      }

      if (!output_lightsvm_file.isEmpty() && ex.features.size() > 0) {
        auto ex_str = exampleToSVMLight(ex);
        output_lightsvm_file.get().write(ex_str + "\n");
        ++rows_written;
      }

      if (!cursor->next()) {
        break;
      }
    }

    status_line.runForce();
  }

  if (!output_meta_file.isEmpty()) {
    FileUtil::mv(output_meta_file_path + "~", output_meta_file_path);
  }

  if (!output_lightsvm_file.isEmpty()) {
    FileUtil::mv(output_lightsvm_file_path + "~", output_lightsvm_file_path);
  }

  return 0;
}

<|endoftext|>


<commit_before>/* $Id: issuetests.cpp 54 2013-07-04 12:04:51Z rijswijk $ */

/*
 * Copyright (c) 2014 Antonio de la Piedra
 * Copyright (c) 2013 Roland van Rijswijk-Deij
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*****************************************************************************
 issuetests.cpp

 Test credential issuance
 *****************************************************************************/

#include <stdlib.h>
#include <cppunit/extensions/HelperMacros.h>
#include <string>
#include <gmpxx.h>
#include <time.h>
#include "managertests.h"
#include "silvia_issuer.h"
#include "silvia_types.h"
#include "silvia_parameters.h"
#include "silvia_macros.h"
#include "silvia_irma_issuer.h"
#include "silvia_irma_manager.h"

CPPUNIT_TEST_SUITE_REGISTRATION(manager_tests);

void manager_tests::setUp()
{
}

void manager_tests::tearDown()
{
}

void manager_tests::test_irma_manager()
{

	std::string PIN = "000000";
	silvia_irma_manager irma_manager;
	std::vector<bytestring> commands = irma_manager.get_log_commands(PIN);

	CPPUNIT_ASSERT(commands[0] == "00A4040009F849524D416361726400"); // select
	CPPUNIT_ASSERT(commands[1] == "00200001083030303030300000"); // VERIFY APDU
	CPPUNIT_ASSERT(commands[2] == "803B0000"); // LOG #1
	CPPUNIT_ASSERT(commands[3] == "803B0F00"); // LOG #2
}
<commit_msg>Improved lib/manager unit test<commit_after>/* $Id: issuetests.cpp 54 2013-07-04 12:04:51Z rijswijk $ */

/*
 * Copyright (c) 2014 Antonio de la Piedra
 * Copyright (c) 2013 Roland van Rijswijk-Deij
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*****************************************************************************
 issuetests.cpp

 Test credential issuance
 *****************************************************************************/

#include <stdlib.h>
#include <cppunit/extensions/HelperMacros.h>
#include <string>
#include <gmpxx.h>
#include <time.h>
#include <iostream>
#include <sstream>
#include <iterator>
#include <iomanip>

#include "managertests.h"
#include "silvia_issuer.h"
#include "silvia_types.h"
#include "silvia_parameters.h"
#include "silvia_macros.h"
#include "silvia_irma_issuer.h"
#include "silvia_irma_manager.h"

CPPUNIT_TEST_SUITE_REGISTRATION(manager_tests);

void manager_tests::setUp()
{
}

void manager_tests::tearDown()
{
}

void manager_tests::test_irma_manager()
{

        std::stringstream ss;

        // Generate a 6-digit random PIN
        srand (time(NULL));
        ss << rand() % 900000 + 100000;
        std::string PIN = ss.str();

        std::ostringstream PIN_hex;
        PIN_hex << std::setw(2) << std::setfill('0') << std::hex << std::uppercase;
        std::copy(PIN.begin(), PIN.end(), std::ostream_iterator<unsigned int>(PIN_hex));

	silvia_irma_manager irma_manager;

	std::vector<bytestring> commands = irma_manager.get_log_commands(PIN);

	CPPUNIT_ASSERT(commands[0] == "00A4040009F849524D416361726400"); // select

	std::stringstream verify_apdu;
	verify_apdu << "0020000108" << PIN_hex.str() << "0000";

	CPPUNIT_ASSERT(commands[1] == verify_apdu.str().c_str()); // VERIFY APDU
	CPPUNIT_ASSERT(commands[2] == "803B0000"); // LOG #1
	CPPUNIT_ASSERT(commands[3] == "803B0F00"); // LOG #2
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-base/util/SimpleRateLimit.h"
#include "fnord-base/InternMap.h"
#include "fnord-json/json.h"
#include "fnord-mdb/MDB.h"
#include "fnord-mdb/MDBUtil.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-sstable/sstablewriter.h"
#include "fnord-sstable/SSTableColumnSchema.h"
#include "fnord-sstable/SSTableColumnReader.h"
#include "fnord-sstable/SSTableColumnWriter.h"
#include "common.h"

using namespace fnord;

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_file",
      fnord::cli::FlagParser::T_STRING,
      true,
      "o",
      NULL,
      "output file",
      "<filename>");

  flags.defineFlag(
      "min_observations",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "100",
      "minimum nuber of feature observations",
      "<num>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto sstables = flags.getArgv();
  auto tbl_cnt = sstables.size();
  if (tbl_cnt == 0) {
    fnord::logCritical("cm.featuredump", "No input tables, exiting");
    return 1;
  }

  /* read input meta tables and count features */
  HashMap<String, uint64_t> feature_counts;
  for (int tbl_idx = 0; tbl_idx < sstables.size(); ++tbl_idx) {
    String sstable = sstables[tbl_idx];
    StringUtil::replaceAll(&sstable, ".sstable", "_meta.sstable");
    fnord::logInfo("cm.featuredump", "Importing sstable: $0", sstable);

    /* read sstable heade r*/
    sstable::SSTableReader reader(File::openFile(sstable, File::O_READ));
    sstable::SSTableColumnSchema schema;
    schema.loadIndex(&reader);

    if (reader.bodySize() == 0) {
      fnord::logCritical("cm.featuredump", "unfinished sstable: $0", sstable);
      exit(1);
    }

    /* get sstable cursor */
    auto cursor = reader.getCursor();
    auto body_size = reader.bodySize();
    int row_idx = 0;

    /* status line */
    util::SimpleRateLimitedFn status_line(kMicrosPerSecond, [&] () {
      auto p = (tbl_idx / (double) tbl_cnt) +
          ((cursor->position() / (double) body_size)) / (double) tbl_cnt;

      fnord::logInfo(
          "cm.featuredump",
          "[$0%] Reading sstable... rows=$3",
          (size_t) (p * 100), tbl_idx + 1, sstables.size(), row_idx);
    });

    /* read sstable rows */
    for (; cursor->valid(); ++row_idx) {
      status_line.runMaybe();

      auto key = cursor->getKeyString();
      auto val = cursor->getDataBuffer();

      sstable::SSTableColumnReader cols(&schema, val);
      auto cnt = cols.getUInt64Column(schema.columnID("count"));

      feature_counts[key] += cnt;

      if (!cursor->next()) {
        break;
      }
    }

    status_line.runForce();
  }

  /* select features for export */
  Set<String> features;
  auto orig_count = feature_counts.size();
  auto min_count = flags.getInt("min_observations");
  for (auto iter = feature_counts.begin(); iter != feature_counts.end(); ) {
    if (iter->second > min_count) {
      features.emplace(iter->first);
    }

    iter = feature_counts.erase(iter);
  }

  feature_counts.clear();

  fnord::logInfo(
      "cm.featuredump",
      "Exporting $1/$0 features",
      orig_count,
      features.size());

  return 0;
}

<commit_msg>better progress indicator in cm-featuredump<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-base/util/SimpleRateLimit.h"
#include "fnord-base/InternMap.h"
#include "fnord-json/json.h"
#include "fnord-mdb/MDB.h"
#include "fnord-mdb/MDBUtil.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-sstable/sstablewriter.h"
#include "fnord-sstable/SSTableColumnSchema.h"
#include "fnord-sstable/SSTableColumnReader.h"
#include "fnord-sstable/SSTableColumnWriter.h"
#include "common.h"

using namespace fnord;

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "output_file",
      fnord::cli::FlagParser::T_STRING,
      true,
      "o",
      NULL,
      "output file",
      "<filename>");

  flags.defineFlag(
      "min_observations",
      fnord::cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "100",
      "minimum nuber of feature observations",
      "<num>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto sstables = flags.getArgv();
  auto tbl_cnt = sstables.size();
  if (tbl_cnt == 0) {
    fnord::logCritical("cm.featuredump", "No input tables, exiting");
    return 1;
  }

  /* read input meta tables and count features */
  HashMap<String, uint64_t> feature_counts;
  int row_idx = 0;
  for (int tbl_idx = 0; tbl_idx < sstables.size(); ++tbl_idx) {
    String sstable = sstables[tbl_idx];
    StringUtil::replaceAll(&sstable, ".sstable", "_meta.sstable");
    fnord::logInfo("cm.featuredump", "Importing sstable: $0", sstable);

    /* read sstable heade r*/
    sstable::SSTableReader reader(File::openFile(sstable, File::O_READ));
    sstable::SSTableColumnSchema schema;
    schema.loadIndex(&reader);

    if (reader.bodySize() == 0) {
      fnord::logCritical("cm.featuredump", "unfinished sstable: $0", sstable);
      exit(1);
    }

    /* get sstable cursor */
    auto cursor = reader.getCursor();
    auto body_size = reader.bodySize();

    /* status line */
    util::SimpleRateLimitedFn status_line(kMicrosPerSecond, [&] () {
      auto p = (tbl_idx / (double) tbl_cnt) +
          ((cursor->position() / (double) body_size)) / (double) tbl_cnt;

      fnord::logInfo(
          "cm.featuredump",
          "[$0%] Reading sstables... rows=$3",
          (size_t) (p * 100), tbl_idx + 1, sstables.size(), row_idx);
    });

    /* read sstable rows */
    for (; cursor->valid(); ++row_idx) {
      status_line.runMaybe();

      auto key = cursor->getKeyString();
      auto val = cursor->getDataBuffer();

      sstable::SSTableColumnReader cols(&schema, val);
      auto cnt = cols.getUInt64Column(schema.columnID("count"));

      feature_counts[key] += cnt;

      if (!cursor->next()) {
        break;
      }
    }

    status_line.runForce();
  }

  /* select features for export */
  Set<String> features;
  auto orig_count = feature_counts.size();
  auto min_count = flags.getInt("min_observations");
  for (auto iter = feature_counts.begin(); iter != feature_counts.end(); ) {
    if (iter->second > min_count) {
      features.emplace(iter->first);
    }

    iter = feature_counts.erase(iter);
  }

  feature_counts.clear();

  fnord::logInfo(
      "cm.featuredump",
      "Exporting $1/$0 features",
      orig_count,
      features.size());

  return 0;
}

<|endoftext|>


<commit_before>#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"

#include <sys/time.h>

#include <regex>

#include <nlohmann/json.hpp>

using namespace std::string_literals;

namespace nix {

struct HgInfo
{
    Path storePath;
    std::string branch;
    std::string rev;
    uint64_t revCount = 0;
};

std::regex commitHashRegex("^[0-9a-fA-F]{40}$");

HgInfo exportMercurial(ref<Store> store, const std::string & uri,
    std::string rev, const std::string & name)
{
    if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {

        bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";

        if (!clean) {

            /* This is an unclean working tree. So copy all tracked
               files. */

            printTalkative("copying unclean Mercurial working tree '%s'", uri);

            HgInfo hgInfo;
            hgInfo.rev = "0000000000000000000000000000000000000000";
            hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));

            auto files = tokenizeString<std::set<std::string>>(
                runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);

            PathFilter filter = [&](const Path & p) -> bool {
                assert(hasPrefix(p, uri));
                auto st = lstat(p);
                std::string file(p, uri.size() + 1);
                if (file == ".hg") return false;
                // FIXME: filter out directories with no tracked files.
                if (S_ISDIR(st.st_mode)) return true;
                return files.count(file);
            };

            hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);

            return hgInfo;
        }
    }

    if (rev == "") rev = "default";

    Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));

    Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));

    /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
       do so now. */
    time_t now = time(0);
    struct stat st;
    if (stat(stampFile.c_str(), &st) != 0 ||
        st.st_mtime < now - settings.tarballTtl)
    {
        /* Except that if this is a commit hash that we already have,
           we don't have to pull again. */
        if (!(std::regex_match(rev, commitHashRegex)
                && pathExists(cacheDir)
                && runProgram(
                    RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
                    .killStderr(true)).second == "1"))
        {
            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));

            if (pathExists(cacheDir)) {
                runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
            } else {
                createDirs(dirOf(cacheDir));
                runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
            }
        }

        writeFile(stampFile, "");
    }

    auto tokens = tokenizeString<std::vector<std::string>>(
        runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
    assert(tokens.size() == 3);

    HgInfo hgInfo;
    hgInfo.rev = tokens[0];
    hgInfo.revCount = std::stoull(tokens[1]);
    hgInfo.branch = tokens[2];

    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
    Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);

    try {
        auto json = nlohmann::json::parse(readFile(storeLink));

        assert(json["name"] == name && json["rev"] == hgInfo.rev);

        hgInfo.storePath = json["storePath"];

        if (store->isValidPath(hgInfo.storePath)) {
            printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
            return hgInfo;
        }

    } catch (SysError & e) {
        if (e.errNo != ENOENT) throw;
    }

    Path tmpDir = createTempDir();
    AutoDelete delTmpDir(tmpDir, true);

    runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });

    deletePath(tmpDir + "/.hg_archival.txt");

    hgInfo.storePath = store->addToStore(name, tmpDir);

    nlohmann::json json;
    json["storePath"] = hgInfo.storePath;
    json["uri"] = uri;
    json["name"] = name;
    json["branch"] = hgInfo.branch;
    json["rev"] = hgInfo.rev;
    json["revCount"] = hgInfo.revCount;

    writeFile(storeLink, json.dump());

    return hgInfo;
}

static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
    std::string url;
    std::string rev;
    std::string name = "source";
    PathSet context;

    state.forceValue(*args[0]);

    if (args[0]->type == tAttrs) {

        state.forceAttrs(*args[0], pos);

        for (auto & attr : *args[0]->attrs) {
            string n(attr.name);
            if (n == "url")
                url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
            else if (n == "rev")
                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
            else if (n == "name")
                name = state.forceStringNoCtx(*attr.value, *attr.pos);
            else
                throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos);
        }

        if (url.empty())
            throw EvalError(format("'url' argument required, at %1%") % pos);

    } else
        url = state.coerceToString(pos, *args[0], context, false, false);

    if (!isUri(url)) url = absPath(url);

    // FIXME: git externals probably can be used to bypass the URI
    // whitelist. Ah well.
    state.checkURI(url);

    auto hgInfo = exportMercurial(state.store, url, rev, name);

    state.mkAttrs(v, 8);
    mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
    mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
    mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
    v.attrs->sort();
}

static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

}
<commit_msg>fetchMercurial: fix error message<commit_after>#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"

#include <sys/time.h>

#include <regex>

#include <nlohmann/json.hpp>

using namespace std::string_literals;

namespace nix {

struct HgInfo
{
    Path storePath;
    std::string branch;
    std::string rev;
    uint64_t revCount = 0;
};

std::regex commitHashRegex("^[0-9a-fA-F]{40}$");

HgInfo exportMercurial(ref<Store> store, const std::string & uri,
    std::string rev, const std::string & name)
{
    if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {

        bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";

        if (!clean) {

            /* This is an unclean working tree. So copy all tracked
               files. */

            printTalkative("copying unclean Mercurial working tree '%s'", uri);

            HgInfo hgInfo;
            hgInfo.rev = "0000000000000000000000000000000000000000";
            hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));

            auto files = tokenizeString<std::set<std::string>>(
                runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);

            PathFilter filter = [&](const Path & p) -> bool {
                assert(hasPrefix(p, uri));
                auto st = lstat(p);
                std::string file(p, uri.size() + 1);
                if (file == ".hg") return false;
                // FIXME: filter out directories with no tracked files.
                if (S_ISDIR(st.st_mode)) return true;
                return files.count(file);
            };

            hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);

            return hgInfo;
        }
    }

    if (rev == "") rev = "default";

    Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));

    Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));

    /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds,
       do so now. */
    time_t now = time(0);
    struct stat st;
    if (stat(stampFile.c_str(), &st) != 0 ||
        st.st_mtime < now - settings.tarballTtl)
    {
        /* Except that if this is a commit hash that we already have,
           we don't have to pull again. */
        if (!(std::regex_match(rev, commitHashRegex)
                && pathExists(cacheDir)
                && runProgram(
                    RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
                    .killStderr(true)).second == "1"))
        {
            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));

            if (pathExists(cacheDir)) {
                runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
            } else {
                createDirs(dirOf(cacheDir));
                runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
            }
        }

        writeFile(stampFile, "");
    }

    auto tokens = tokenizeString<std::vector<std::string>>(
        runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
    assert(tokens.size() == 3);

    HgInfo hgInfo;
    hgInfo.rev = tokens[0];
    hgInfo.revCount = std::stoull(tokens[1]);
    hgInfo.branch = tokens[2];

    std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
    Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);

    try {
        auto json = nlohmann::json::parse(readFile(storeLink));

        assert(json["name"] == name && json["rev"] == hgInfo.rev);

        hgInfo.storePath = json["storePath"];

        if (store->isValidPath(hgInfo.storePath)) {
            printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
            return hgInfo;
        }

    } catch (SysError & e) {
        if (e.errNo != ENOENT) throw;
    }

    Path tmpDir = createTempDir();
    AutoDelete delTmpDir(tmpDir, true);

    runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });

    deletePath(tmpDir + "/.hg_archival.txt");

    hgInfo.storePath = store->addToStore(name, tmpDir);

    nlohmann::json json;
    json["storePath"] = hgInfo.storePath;
    json["uri"] = uri;
    json["name"] = name;
    json["branch"] = hgInfo.branch;
    json["rev"] = hgInfo.rev;
    json["revCount"] = hgInfo.revCount;

    writeFile(storeLink, json.dump());

    return hgInfo;
}

static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
    std::string url;
    std::string rev;
    std::string name = "source";
    PathSet context;

    state.forceValue(*args[0]);

    if (args[0]->type == tAttrs) {

        state.forceAttrs(*args[0], pos);

        for (auto & attr : *args[0]->attrs) {
            string n(attr.name);
            if (n == "url")
                url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
            else if (n == "rev")
                rev = state.forceStringNoCtx(*attr.value, *attr.pos);
            else if (n == "name")
                name = state.forceStringNoCtx(*attr.value, *attr.pos);
            else
                throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos);
        }

        if (url.empty())
            throw EvalError(format("'url' argument required, at %1%") % pos);

    } else
        url = state.coerceToString(pos, *args[0], context, false, false);

    if (!isUri(url)) url = absPath(url);

    // FIXME: git externals probably can be used to bypass the URI
    // whitelist. Ah well.
    state.checkURI(url);

    auto hgInfo = exportMercurial(state.store, url, rev, name);

    state.mkAttrs(v, 8);
    mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath}));
    mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch);
    mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev);
    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12));
    mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount);
    v.attrs->sort();
}

static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

}
<|endoftext|>


<commit_before>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009-2013 Klaus Post

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "parsers/X3fParser.h"
#include "common/Common.h"                // for uint32, uchar8
#include "decoders/RawDecoderException.h" // for ThrowRDE, RawDecoderException
#include "decoders/X3fDecoder.h"          // for X3fDecoder
#include "io/Buffer.h"                    // for Buffer
#include "io/ByteStream.h"                // for ByteStream
#include "io/Endianness.h"                // for getHostEndianness, Endiann...
#include "io/IOException.h"               // for IOException
#include <cstring>                        // for memset
#include <map>                            // for map, map<>::mapped_type
#include <string>                         // for string, basic_string, oper...
#include <vector>                         // for vector

using namespace std;

namespace RawSpeed {

X3fParser::X3fParser(Buffer* file) : RawParser(file) {
  decoder = nullptr;
  bytes = nullptr;
  uint32 size = file->getSize();
  if (size<104+128)
    ThrowRDE("X3F file too small");

  bytes = new ByteStream(file, 0, size, getHostEndianness() == little);

  try {
    try {
      // Read signature
      if (bytes->getU32() != 0x62564f46)
        ThrowRDE("Not an X3f file (Signature)");

      uint32 version = bytes->getU32();
      if (version < 0x00020000)
        ThrowRDE("File version too old");

      // Skip identifier + mark bits
      bytes->skipBytes(16+4);

      bytes->setPosition(0);
      decoder = new X3fDecoder(file);
      readDirectory();
    } catch (IOException &e) {
      ThrowRDE("IO Error while reading header: %s", e.what());
    }
  } catch (RawDecoderException &e) {
    freeObjects();
    throw e;
  }
}

void X3fParser::freeObjects() {
  delete bytes;
  delete decoder;
  decoder = nullptr;
  bytes = nullptr;
}

X3fParser::~X3fParser() { freeObjects(); }

static string getIdAsString(ByteStream *bytes) {
  uchar8 id[5];
  for (int i = 0; i < 4; i++)
    id[i] = bytes->getByte();
  id[4] = 0;
  return string((const char*)id);
}

void X3fParser::readDirectory()
{
  bytes->setPosition(mInput->getSize() - 4);
  uint32 dir_off = bytes->getU32();
  bytes->setPosition(dir_off);

  // Check signature
  if ("SECd" != getIdAsString(bytes))
    ThrowRDE("Unable to locate directory");

  uint32 version = bytes->getU32();
  if (version < 0x00020000)
    ThrowRDE("File version too old (directory)");

  uint32 n_entries = bytes->getU32();
  for (uint32 i = 0; i < n_entries; i++) {
    X3fDirectory dir(bytes);
    decoder->mDirectory.push_back(dir);
    uint32 old_pos = bytes->getPosition();
    if ("IMA2" == dir.id || "IMAG" == dir.id) {
      decoder->mImages.emplace_back(bytes, dir.offset, dir.length);
    }
    if ("PROP" == dir.id) {
      decoder->mProperties.addProperties(bytes, dir.offset, dir.length);
    }
    bytes->setPosition(old_pos);
  }
}

RawDecoder* X3fParser::getDecoder()
{
  if (nullptr == decoder)
    ThrowRDE("No decoder found!");
  RawDecoder *ret = decoder;
  decoder = nullptr;
  return ret;
}

X3fDirectory::X3fDirectory( ByteStream *bytes )
{
    offset = bytes->getU32();
    length = bytes->getU32();
    id = getIdAsString(bytes);
    uint32 old_pos = bytes->getPosition();
    bytes->setPosition(offset);
    sectionID = getIdAsString(bytes);
    bytes->setPosition(old_pos);
}

X3fImage::X3fImage( ByteStream *bytes, uint32 offset, uint32 length )
{
  bytes->setPosition(offset);
  string id = getIdAsString(bytes);
  if (id != "SECi")
    ThrowRDE("Unknown Image signature");

  uint32 version = bytes->getU32();
  if (version < 0x00020000)
    ThrowRDE("File version too old (image)");

  type = bytes->getU32();
  format = bytes->getU32();
  width = bytes->getU32();
  height = bytes->getU32();
  pitchB = bytes->getU32();
  dataOffset = bytes->getPosition();
  dataSize = length - (dataOffset-offset);
  if (pitchB == dataSize)
    pitchB = 0;
}

/*
* ConvertUTF16toUTF8 function only Copyright:
*
* Copyright 2001-2004 Unicode, Inc.
*
* Disclaimer
*
* This source code is provided as is by Unicode, Inc. No claims are
* made as to fitness for any particular purpose. No warranties of any
* kind are expressed or implied. The recipient agrees to determine
* applicability of information provided. If this file has been
* purchased on magnetic or optical media from Unicode, Inc., the
* sole remedy for any claim will be exchange of defective media
* within 90 days of receipt.
*
* Limitations on Rights to Redistribute This Code
*
* Unicode, Inc. hereby grants the right to freely use the information
* supplied in this file in the creation of products supporting the
* Unicode Standard, and to make copies of this file in any form
* for internal or external distribution as long as this notice
* remains attached.
*/

using UTF32 = unsigned int;    /* at least 32 bits */
using UTF16 = unsigned short;  /* at least 16 bits */
using UTF8 = unsigned char;    /* typically 8 bits */
using Boolean = unsigned char; /* 0 or 1 */

/* Some fundamental constants */
#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
#define UNI_MAX_BMP (UTF32)0x0000FFFF
#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF

#define UNI_MAX_UTF8_BYTES_PER_CODE_POINT 4

#define UNI_UTF16_BYTE_ORDER_MARK_NATIVE  0xFEFF
#define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE

#define UNI_SUR_HIGH_START  (UTF32)0xD800
#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
#define UNI_SUR_LOW_START   (UTF32)0xDC00
#define UNI_SUR_LOW_END     (UTF32)0xDFFF

static const int halfShift  = 10; /* used for shifting by 10 bits */
static const UTF32 halfBase = 0x0010000UL;
static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };

static bool ConvertUTF16toUTF8(const UTF16** sourceStart,
                               const UTF16* sourceEnd, UTF8** targetStart,
                               const UTF8* targetEnd) {
  bool success = true;
  const UTF16* source = *sourceStart;
  UTF8* target = *targetStart;
  while (source < sourceEnd) {
    UTF32 ch;
    unsigned short bytesToWrite = 0;
    const UTF32 byteMask = 0xBF;
    const UTF32 byteMark = 0x80;
    const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
    ch = *source++;
    /* If we have a surrogate pair, convert to UTF32 first. */
    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
      /* If the 16 bits following the high surrogate are in the source buffer... */
      if (source < sourceEnd) {
        UTF32 ch2 = *source;
        /* If it's a low surrogate, convert to UTF32. */
        if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
          ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
            + (ch2 - UNI_SUR_LOW_START) + halfBase;
          ++source;
#if 0
        } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
          --source; /* return to the illegal value itself */
          success = false;
          break;
#endif
        }
      } else { /* We don't have the 16 bits following the high surrogate. */
        --source; /* return to the high surrogate */
        success = false;
        break;
      }
    }
    /* Figure out how many bytes the result will require */
    if (ch < (UTF32)0x80) {      bytesToWrite = 1;
    } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
    } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
    } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
    } else {                            bytesToWrite = 3;
    ch = UNI_REPLACEMENT_CHAR;
    }

    target += bytesToWrite;
    if (target > targetEnd) {
      source = oldSource; /* Back up source pointer! */
      target -= bytesToWrite;
      success = false;
      break;
    }
    switch (bytesToWrite) { /* note: everything falls through. */
            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
            case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
    }
    target += bytesToWrite;
  }
  // Function modified to retain source + target positions
  //  *sourceStart = source;
  //  *targetStart = target;
  return success;
}

string X3fPropertyCollection::getString( ByteStream *bytes ) {
  uint32 max_len = bytes->getRemainSize() / 2;
  const auto *start = (const UTF16 *)bytes->getData(max_len * 2);
  const UTF16* src_end = start;
  uint32 i = 0;
  for (; i < max_len && start == src_end; i++) {
    if (start[i] == 0) {
      src_end = &start[i];
    }
  }
  if (start != src_end) {
    auto *dest = new UTF8[i * 4UL + 1];
    memset(dest, 0, i * 4UL + 1);
    if (ConvertUTF16toUTF8(&start, src_end, &dest, &dest[i * 4 - 1])) {
      string ret((const char*)dest);
      delete[] dest;
      return ret;
    }
    delete[] dest;
  }
  return "";
}

void X3fPropertyCollection::addProperties( ByteStream *bytes, uint32 offset, uint32 length )
{
  bytes->setPosition(offset);
  string id = getIdAsString(bytes);
  if (id != "SECp")
    ThrowRDE("Unknown Property signature");

  uint32 version = bytes->getU32();
  if (version < 0x00020000)
    ThrowRDE("File version too old (properties)");

  uint32 entries = bytes->getU32();
  if (!entries)
    return;

  if (0 != bytes->getU32())
    ThrowRDE("Unknown property character encoding");

  // Skip 4 reserved bytes
  bytes->skipBytes(4);

  // Skip size (not used ATM)
  bytes->skipBytes(4);

  if (entries > 1000)
    ThrowRDE("Unreasonable number of properties: %u", entries);

  uint32 data_start = bytes->getPosition() + entries*8;
  for (uint32 i = 0; i < entries; i++) {
    uint32 key_pos = bytes->getU32();
    uint32 value_pos = bytes->getU32();
    uint32 old_pos = bytes->getPosition();
    try {
      bytes->setPosition(key_pos * 2 + data_start);
      string key = getString(bytes);
      bytes->setPosition(value_pos * 2 + data_start);
      string val = getString(bytes);
      props[key] = val;
    } catch (IOException &) {
    }
    bytes->setPosition(old_pos);
  }
}

} // namespace RawSpeed
<commit_msg>X3fParser::X3fParser(): let C++ rethrow the exception automatically.<commit_after>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009-2013 Klaus Post

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "parsers/X3fParser.h"
#include "common/Common.h"                // for uint32, uchar8
#include "decoders/RawDecoderException.h" // for ThrowRDE, RawDecoderException
#include "decoders/X3fDecoder.h"          // for X3fDecoder
#include "io/Buffer.h"                    // for Buffer
#include "io/ByteStream.h"                // for ByteStream
#include "io/Endianness.h"                // for getHostEndianness, Endiann...
#include "io/IOException.h"               // for IOException
#include <cstring>                        // for memset
#include <map>                            // for map, map<>::mapped_type
#include <string>                         // for string, basic_string, oper...
#include <vector>                         // for vector

using namespace std;

namespace RawSpeed {

X3fParser::X3fParser(Buffer* file) : RawParser(file) {
  decoder = nullptr;
  bytes = nullptr;
  uint32 size = file->getSize();
  if (size<104+128)
    ThrowRDE("X3F file too small");

  bytes = new ByteStream(file, 0, size, getHostEndianness() == little);

  try {
    try {
      // Read signature
      if (bytes->getU32() != 0x62564f46)
        ThrowRDE("Not an X3f file (Signature)");

      uint32 version = bytes->getU32();
      if (version < 0x00020000)
        ThrowRDE("File version too old");

      // Skip identifier + mark bits
      bytes->skipBytes(16+4);

      bytes->setPosition(0);
      decoder = new X3fDecoder(file);
      readDirectory();
    } catch (IOException &e) {
      ThrowRDE("IO Error while reading header: %s", e.what());
    }
  } catch (RawDecoderException &e) {
    freeObjects();
    throw;
  }
}

void X3fParser::freeObjects() {
  delete bytes;
  delete decoder;
  decoder = nullptr;
  bytes = nullptr;
}

X3fParser::~X3fParser() { freeObjects(); }

static string getIdAsString(ByteStream *bytes) {
  uchar8 id[5];
  for (int i = 0; i < 4; i++)
    id[i] = bytes->getByte();
  id[4] = 0;
  return string((const char*)id);
}

void X3fParser::readDirectory()
{
  bytes->setPosition(mInput->getSize() - 4);
  uint32 dir_off = bytes->getU32();
  bytes->setPosition(dir_off);

  // Check signature
  if ("SECd" != getIdAsString(bytes))
    ThrowRDE("Unable to locate directory");

  uint32 version = bytes->getU32();
  if (version < 0x00020000)
    ThrowRDE("File version too old (directory)");

  uint32 n_entries = bytes->getU32();
  for (uint32 i = 0; i < n_entries; i++) {
    X3fDirectory dir(bytes);
    decoder->mDirectory.push_back(dir);
    uint32 old_pos = bytes->getPosition();
    if ("IMA2" == dir.id || "IMAG" == dir.id) {
      decoder->mImages.emplace_back(bytes, dir.offset, dir.length);
    }
    if ("PROP" == dir.id) {
      decoder->mProperties.addProperties(bytes, dir.offset, dir.length);
    }
    bytes->setPosition(old_pos);
  }
}

RawDecoder* X3fParser::getDecoder()
{
  if (nullptr == decoder)
    ThrowRDE("No decoder found!");
  RawDecoder *ret = decoder;
  decoder = nullptr;
  return ret;
}

X3fDirectory::X3fDirectory( ByteStream *bytes )
{
    offset = bytes->getU32();
    length = bytes->getU32();
    id = getIdAsString(bytes);
    uint32 old_pos = bytes->getPosition();
    bytes->setPosition(offset);
    sectionID = getIdAsString(bytes);
    bytes->setPosition(old_pos);
}

X3fImage::X3fImage( ByteStream *bytes, uint32 offset, uint32 length )
{
  bytes->setPosition(offset);
  string id = getIdAsString(bytes);
  if (id != "SECi")
    ThrowRDE("Unknown Image signature");

  uint32 version = bytes->getU32();
  if (version < 0x00020000)
    ThrowRDE("File version too old (image)");

  type = bytes->getU32();
  format = bytes->getU32();
  width = bytes->getU32();
  height = bytes->getU32();
  pitchB = bytes->getU32();
  dataOffset = bytes->getPosition();
  dataSize = length - (dataOffset-offset);
  if (pitchB == dataSize)
    pitchB = 0;
}

/*
* ConvertUTF16toUTF8 function only Copyright:
*
* Copyright 2001-2004 Unicode, Inc.
*
* Disclaimer
*
* This source code is provided as is by Unicode, Inc. No claims are
* made as to fitness for any particular purpose. No warranties of any
* kind are expressed or implied. The recipient agrees to determine
* applicability of information provided. If this file has been
* purchased on magnetic or optical media from Unicode, Inc., the
* sole remedy for any claim will be exchange of defective media
* within 90 days of receipt.
*
* Limitations on Rights to Redistribute This Code
*
* Unicode, Inc. hereby grants the right to freely use the information
* supplied in this file in the creation of products supporting the
* Unicode Standard, and to make copies of this file in any form
* for internal or external distribution as long as this notice
* remains attached.
*/

using UTF32 = unsigned int;    /* at least 32 bits */
using UTF16 = unsigned short;  /* at least 16 bits */
using UTF8 = unsigned char;    /* typically 8 bits */
using Boolean = unsigned char; /* 0 or 1 */

/* Some fundamental constants */
#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
#define UNI_MAX_BMP (UTF32)0x0000FFFF
#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF

#define UNI_MAX_UTF8_BYTES_PER_CODE_POINT 4

#define UNI_UTF16_BYTE_ORDER_MARK_NATIVE  0xFEFF
#define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE

#define UNI_SUR_HIGH_START  (UTF32)0xD800
#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
#define UNI_SUR_LOW_START   (UTF32)0xDC00
#define UNI_SUR_LOW_END     (UTF32)0xDFFF

static const int halfShift  = 10; /* used for shifting by 10 bits */
static const UTF32 halfBase = 0x0010000UL;
static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };

static bool ConvertUTF16toUTF8(const UTF16** sourceStart,
                               const UTF16* sourceEnd, UTF8** targetStart,
                               const UTF8* targetEnd) {
  bool success = true;
  const UTF16* source = *sourceStart;
  UTF8* target = *targetStart;
  while (source < sourceEnd) {
    UTF32 ch;
    unsigned short bytesToWrite = 0;
    const UTF32 byteMask = 0xBF;
    const UTF32 byteMark = 0x80;
    const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
    ch = *source++;
    /* If we have a surrogate pair, convert to UTF32 first. */
    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
      /* If the 16 bits following the high surrogate are in the source buffer... */
      if (source < sourceEnd) {
        UTF32 ch2 = *source;
        /* If it's a low surrogate, convert to UTF32. */
        if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
          ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
            + (ch2 - UNI_SUR_LOW_START) + halfBase;
          ++source;
#if 0
        } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
          --source; /* return to the illegal value itself */
          success = false;
          break;
#endif
        }
      } else { /* We don't have the 16 bits following the high surrogate. */
        --source; /* return to the high surrogate */
        success = false;
        break;
      }
    }
    /* Figure out how many bytes the result will require */
    if (ch < (UTF32)0x80) {      bytesToWrite = 1;
    } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
    } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
    } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
    } else {                            bytesToWrite = 3;
    ch = UNI_REPLACEMENT_CHAR;
    }

    target += bytesToWrite;
    if (target > targetEnd) {
      source = oldSource; /* Back up source pointer! */
      target -= bytesToWrite;
      success = false;
      break;
    }
    switch (bytesToWrite) { /* note: everything falls through. */
            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
            case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
    }
    target += bytesToWrite;
  }
  // Function modified to retain source + target positions
  //  *sourceStart = source;
  //  *targetStart = target;
  return success;
}

string X3fPropertyCollection::getString( ByteStream *bytes ) {
  uint32 max_len = bytes->getRemainSize() / 2;
  const auto *start = (const UTF16 *)bytes->getData(max_len * 2);
  const UTF16* src_end = start;
  uint32 i = 0;
  for (; i < max_len && start == src_end; i++) {
    if (start[i] == 0) {
      src_end = &start[i];
    }
  }
  if (start != src_end) {
    auto *dest = new UTF8[i * 4UL + 1];
    memset(dest, 0, i * 4UL + 1);
    if (ConvertUTF16toUTF8(&start, src_end, &dest, &dest[i * 4 - 1])) {
      string ret((const char*)dest);
      delete[] dest;
      return ret;
    }
    delete[] dest;
  }
  return "";
}

void X3fPropertyCollection::addProperties( ByteStream *bytes, uint32 offset, uint32 length )
{
  bytes->setPosition(offset);
  string id = getIdAsString(bytes);
  if (id != "SECp")
    ThrowRDE("Unknown Property signature");

  uint32 version = bytes->getU32();
  if (version < 0x00020000)
    ThrowRDE("File version too old (properties)");

  uint32 entries = bytes->getU32();
  if (!entries)
    return;

  if (0 != bytes->getU32())
    ThrowRDE("Unknown property character encoding");

  // Skip 4 reserved bytes
  bytes->skipBytes(4);

  // Skip size (not used ATM)
  bytes->skipBytes(4);

  if (entries > 1000)
    ThrowRDE("Unreasonable number of properties: %u", entries);

  uint32 data_start = bytes->getPosition() + entries*8;
  for (uint32 i = 0; i < entries; i++) {
    uint32 key_pos = bytes->getU32();
    uint32 value_pos = bytes->getU32();
    uint32 old_pos = bytes->getPosition();
    try {
      bytes->setPosition(key_pos * 2 + data_start);
      string key = getString(bytes);
      bytes->setPosition(value_pos * 2 + data_start);
      string val = getString(bytes);
      props[key] = val;
    } catch (IOException &) {
    }
    bytes->setPosition(old_pos);
  }
}

} // namespace RawSpeed
<|endoftext|>