<commit_before>
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/AsmParser/Parser.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/CodeGen/CommandFlags.h"
#include "llvm/CodeGen/LinkAllAsmWriterComponents.h"
#include "llvm/CodeGen/LinkAllCodegenComponents.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/IRPrintingPasses.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Verifier.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/Linker/Linker.h"
#include "llvm/MC/SubtargetFeature.h"
#include "llvm/IR/LegacyPassManager.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/ToolOutputFile.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetSubtargetInfo.h"
#include "llvm/Transforms/Scalar.h"

#include <memory>
#include <string>

#include "DynamicCallCounter.h"
#include "StaticCallCounter.h"

#include "config.h"

using namespace llvm;
using std::string;
using std::unique_ptr;
using std::vector;
using llvm::sys::ExecuteAndWait;
using llvm::sys::findProgramByName;
using llvm::legacy::PassManager;

enum class AnalysisType {
  STATIC,
  DYNAMIC,
};

static cl::OptionCategory callCounterCategory{"call counter options"};

static cl::opt<string> inPath{cl::Positional,
                              cl::desc{"<Module to analyze>"},
                              cl::value_desc{"bitcode filename"},
                              cl::init(""),
                              cl::Required,
                              cl::cat{callCounterCategory}};

static cl::opt<AnalysisType> analysisType{
    cl::desc{"Select analyis type:"},
    cl::values(clEnumValN(AnalysisType::STATIC,
                          "static",
                          "Count static direct calls."),
               clEnumValN(AnalysisType::DYNAMIC,
                          "dynamic",
                          "Count dynamic direct calls."),
               clEnumValEnd),
    cl::Required,
    cl::cat{callCounterCategory}};

static cl::opt<string> outFile{"o",
                               cl::desc{"Filename of the instrumented program"},
                               cl::value_desc{"filename"},
                               cl::init(""),
                               cl::cat{callCounterCategory}};

static cl::opt<char> optLevel{
    "O",
    cl::desc{"Optimization level. [-O0, -O1, -O2, or -O3] (default = '-O2')"},
    cl::Prefix,
    cl::ZeroOrMore,
    cl::init('2'),
    cl::cat{callCounterCategory}};

static cl::list<string> libPaths{"L",
                                 cl::Prefix,
                                 cl::desc{"Specify a library search path"},
                                 cl::value_desc{"directory"},
                                 cl::cat{callCounterCategory}};

static cl::list<string> libraries{"l",
                                  cl::Prefix,
                                  cl::desc{"Specify libraries to link against"},
                                  cl::value_desc{"library prefix"},
                                  cl::cat{callCounterCategory}};

static void
compile(Module& m, StringRef outputPath) {
  string err;

  Triple triple        = Triple(m.getTargetTriple());
  Target const* target = TargetRegistry::lookupTarget(MArch, triple, err);
  if (!target) {
    report_fatal_error("Unable to find target:\n " + err);
  }

  CodeGenOpt::Level level = CodeGenOpt::Default;
  switch (optLevel) {
    default:
      report_fatal_error("Invalid optimization level.\n");
    // No fall through
    case '0': level = CodeGenOpt::None; break;
    case '1': level = CodeGenOpt::Less; break;
    case '2': level = CodeGenOpt::Default; break;
    case '3': level = CodeGenOpt::Aggressive; break;
  }

  string FeaturesStr;
  TargetOptions options = InitTargetOptionsFromCodeGenFlags();
  unique_ptr<TargetMachine> machine(
      target->createTargetMachine(triple.getTriple(),
                                  MCPU,
                                  FeaturesStr,
                                  options,
                                  getRelocModel(),
                                  CMModel,
                                  level));
  assert(machine.get() && "Could not allocate target machine!");

  if (FloatABIForCalls != FloatABI::Default) {
    options.FloatABIType = FloatABIForCalls;
  }

  std::error_code errc;
  auto out =
      std::make_unique<tool_output_file>(outputPath, errc, sys::fs::F_None);
  if (!out) {
    report_fatal_error("Unable to create file:\n " + errc.message());
  }

  // Build up all of the passes that we want to do to the module.
  legacy::PassManager pm;

  // Add target specific info and transforms
  TargetLibraryInfoImpl tlii(triple);
  pm.add(new TargetLibraryInfoWrapperPass(tlii));

  m.setDataLayout(machine->createDataLayout());

  {  // Bound this scope
    raw_pwrite_stream* os(&out->os());

    FileType = TargetMachine::CGFT_ObjectFile;
    std::unique_ptr<buffer_ostream> bos;
    if (!out->os().supportsSeeking()) {
      bos = std::make_unique<buffer_ostream>(*os);
      os  = bos.get();
    }

    // Ask the target to add backend passes as necessary.
    if (machine->addPassesToEmitFile(pm, *os, FileType)) {
      report_fatal_error("target does not support generation "
                         "of this file type!\n");
    }

    // Before executing passes, print the final values of the LLVM options.
    cl::PrintOptionValues();

    pm.run(m);
  }

  // Keep the output binary if we've been successful to this point.
  out->keep();
}

static void
link(StringRef objectFile, StringRef outputFile) {
  auto clang = findProgramByName("clang++");
  string opt("-O");
  opt += optLevel;

  if (!clang) {
    report_fatal_error("Unable to find clang.");
  }
  vector<string> args{clang.get(), opt, "-o", outputFile, objectFile};

  for (auto& libPath : libPaths) {
    args.push_back("-L" + libPath);
  }

  for (auto& library : libraries) {
    args.push_back("-l" + library);
  }

  vector<char const*> charArgs;
  for (auto& arg : args) {
    charArgs.push_back(arg.c_str());
  }
  charArgs.push_back(nullptr);

  for (auto& arg : args) {
    outs() << arg.c_str() << " ";
  }
  outs() << "\n";

  string err;
  if (-1 == ExecuteAndWait(
                clang.get(), &charArgs[0], nullptr, nullptr, 0, 0, &err)) {
    report_fatal_error("Unable to link output file.");
  }
}

static void
generateBinary(Module& m, StringRef outputFilename) {
  // Compiling to native should allow things to keep working even when the
  // version of clang on the system and the version of LLVM used to compile
  // the tool don't quite match up.
  string objectFile = outputFilename.str() + ".o";
  compile(m, objectFile);
  link(objectFile, outputFilename);
}

static void
saveModule(Module const& m, StringRef filename) {
  std::error_code errc;
  raw_fd_ostream out(filename.data(), errc, sys::fs::F_None);

  if (errc) {
    report_fatal_error("error saving llvm module to '" + filename + "': \n"
                       + errc.message());
  }
  WriteBitcodeToFile(&m, out);
}

static void
prepareLinkingPaths(SmallString<32> invocationPath) {
  // First search the directory of the binary for the library, in case it is
  // all bundled together.
  sys::path::remove_filename(invocationPath);
  if (!invocationPath.empty()) {
    libPaths.push_back(invocationPath.str());
  }
// If the builder doesn't plan on installing it, we still need to get to the
// runtime library somehow, so just build in the path to the temporary one.
#ifdef CMAKE_INSTALL_PREFIX
  libPaths.push_back(CMAKE_INSTALL_PREFIX "/lib");
#elif defined(CMAKE_TEMP_LIBRARY_PATH)
  libPaths.push_back(CMAKE_TEMP_LIBRARY_PATH);
#elif defined(TEMP_LIBRARY_PATH)
  // This is a bit of a hack
  libPaths.push_back(TEMP_LIBRARY_PATH "/Debug+Asserts/lib/");
  libPaths.push_back(TEMP_LIBRARY_PATH "/Release+Asserts/lib/");
  libPaths.push_back(TEMP_LIBRARY_PATH "/Debug/lib/");
  libPaths.push_back(TEMP_LIBRARY_PATH "/Release/lib/");
#endif
  libraries.push_back(RUNTIME_LIB);
  libraries.push_back("rt");
}

static void
instrumentForDynamicCount(Module& m) {
  InitializeAllTargets();
  InitializeAllTargetMCs();
  InitializeAllAsmPrinters();
  InitializeAllAsmParsers();
  cl::AddExtraVersionPrinter(TargetRegistry::printRegisteredTargetsForVersion);

  if (outFile.getValue() == "") {
    errs() << "-o command line option must be specified.\n";
    exit(-1);
  }

  // Build up all of the passes that we want to run on the module.
  legacy::PassManager pm;
  pm.add(new callcounter::DynamicCallCounter());
  pm.add(createVerifierPass());
  pm.run(m);

  generateBinary(m, outFile);
  saveModule(m, outFile + ".callcounter.bc");
}

struct StaticCountPrinter : public ModulePass {
  static char ID;
  raw_ostream& out;

  explicit StaticCountPrinter(raw_ostream& out) : ModulePass(ID), out(out) {}

  bool
  runOnModule(Module& m) override {
    getAnalysis<callcounter::StaticCallCounter>().print(out, &m);
    return false;
  }

  void
  getAnalysisUsage(AnalysisUsage& au) const override {
    au.addRequired<callcounter::StaticCallCounter>();
    au.setPreservesAll();
  }
};

char StaticCountPrinter::ID = 0;

static void
countStaticCalls(Module& m) {
  // Build up all of the passes that we want to run on the module.
  legacy::PassManager pm;
  pm.add(new callcounter::StaticCallCounter());
  pm.add(new StaticCountPrinter(outs()));
  pm.run(m);
}

int
main(int argc, char** argv) {
  // This boilerplate provides convenient stack traces and clean LLVM exit
  // handling. It also initializes the built in support for convenient
  // command line option handling.
  sys::PrintStackTraceOnErrorSignal(argv[0]);
  llvm::PrettyStackTraceProgram X(argc, argv);
  llvm_shutdown_obj shutdown;
  cl::HideUnrelatedOptions(callCounterCategory);
  cl::ParseCommandLineOptions(argc, argv);

  // Construct an IR file from the filename passed on the command line.
  SMDiagnostic err;
  LLVMContext context;
  unique_ptr<Module> module = parseIRFile(inPath.getValue(), err, context);

  if (!module.get()) {
    errs() << "Error reading bitcode file: " << inPath << "\n";
    err.print(argv[0], errs());
    return -1;
  }

  if (AnalysisType::DYNAMIC == analysisType) {
    prepareLinkingPaths(StringRef(argv[0]));
    instrumentForDynamicCount(*module);
  } else {
    countStaticCalls(*module);
  }

  return 0;
}
<commit_msg>Working with LLVM 4.0<commit_after>
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/AsmParser/Parser.h"
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/CodeGen/CommandFlags.h"
#include "llvm/CodeGen/LinkAllAsmWriterComponents.h"
#include "llvm/CodeGen/LinkAllCodegenComponents.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/IRPrintingPasses.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Verifier.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/Linker/Linker.h"
#include "llvm/MC/SubtargetFeature.h"
#include "llvm/IR/LegacyPassManager.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/ToolOutputFile.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetSubtargetInfo.h"
#include "llvm/Transforms/Scalar.h"

#include <memory>
#include <string>

#include "DynamicCallCounter.h"
#include "StaticCallCounter.h"

#include "config.h"

using namespace llvm;
using std::string;
using std::unique_ptr;
using std::vector;
using llvm::sys::ExecuteAndWait;
using llvm::sys::findProgramByName;
using llvm::legacy::PassManager;

enum class AnalysisType {
  STATIC,
  DYNAMIC,
};

static cl::OptionCategory callCounterCategory{"call counter options"};

static cl::opt<string> inPath{cl::Positional,
                              cl::desc{"<Module to analyze>"},
                              cl::value_desc{"bitcode filename"},
                              cl::init(""),
                              cl::Required,
                              cl::cat{callCounterCategory}};

static cl::opt<AnalysisType> analysisType{
    cl::desc{"Select analyis type:"},
    cl::values(clEnumValN(AnalysisType::STATIC,
                          "static",
                          "Count static direct calls."),
               clEnumValN(AnalysisType::DYNAMIC,
                          "dynamic",
                          "Count dynamic direct calls.")
               ),
    cl::Required,
    cl::cat{callCounterCategory}};

static cl::opt<string> outFile{"o",
                               cl::desc{"Filename of the instrumented program"},
                               cl::value_desc{"filename"},
                               cl::init(""),
                               cl::cat{callCounterCategory}};

static cl::opt<char> optLevel{
    "O",
    cl::desc{"Optimization level. [-O0, -O1, -O2, or -O3] (default = '-O2')"},
    cl::Prefix,
    cl::ZeroOrMore,
    cl::init('2'),
    cl::cat{callCounterCategory}};

static cl::list<string> libPaths{"L",
                                 cl::Prefix,
                                 cl::desc{"Specify a library search path"},
                                 cl::value_desc{"directory"},
                                 cl::cat{callCounterCategory}};

static cl::list<string> libraries{"l",
                                  cl::Prefix,
                                  cl::desc{"Specify libraries to link against"},
                                  cl::value_desc{"library prefix"},
                                  cl::cat{callCounterCategory}};

static void
compile(Module& m, StringRef outputPath) {
  string err;

  Triple triple        = Triple(m.getTargetTriple());
  Target const* target = TargetRegistry::lookupTarget(MArch, triple, err);
  if (!target) {
    report_fatal_error("Unable to find target:\n " + err);
  }

  CodeGenOpt::Level level = CodeGenOpt::Default;
  switch (optLevel) {
    default:
      report_fatal_error("Invalid optimization level.\n");
    // No fall through
    case '0': level = CodeGenOpt::None; break;
    case '1': level = CodeGenOpt::Less; break;
    case '2': level = CodeGenOpt::Default; break;
    case '3': level = CodeGenOpt::Aggressive; break;
  }

  string FeaturesStr;
  TargetOptions options = InitTargetOptionsFromCodeGenFlags();
  unique_ptr<TargetMachine> machine(
      target->createTargetMachine(triple.getTriple(),
                                  MCPU,
                                  FeaturesStr,
                                  options,
                                  getRelocModel(),
                                  CMModel,
                                  level));
  assert(machine.get() && "Could not allocate target machine!");

  if (FloatABIForCalls != FloatABI::Default) {
    options.FloatABIType = FloatABIForCalls;
  }

  std::error_code errc;
  auto out =
      std::make_unique<tool_output_file>(outputPath, errc, sys::fs::F_None);
  if (!out) {
    report_fatal_error("Unable to create file:\n " + errc.message());
  }

  // Build up all of the passes that we want to do to the module.
  legacy::PassManager pm;

  // Add target specific info and transforms
  TargetLibraryInfoImpl tlii(triple);
  pm.add(new TargetLibraryInfoWrapperPass(tlii));

  m.setDataLayout(machine->createDataLayout());

  {  // Bound this scope
    raw_pwrite_stream* os(&out->os());

    FileType = TargetMachine::CGFT_ObjectFile;
    std::unique_ptr<buffer_ostream> bos;
    if (!out->os().supportsSeeking()) {
      bos = std::make_unique<buffer_ostream>(*os);
      os  = bos.get();
    }

    // Ask the target to add backend passes as necessary.
    if (machine->addPassesToEmitFile(pm, *os, FileType)) {
      report_fatal_error("target does not support generation "
                         "of this file type!\n");
    }

    // Before executing passes, print the final values of the LLVM options.
    cl::PrintOptionValues();

    pm.run(m);
  }

  // Keep the output binary if we've been successful to this point.
  out->keep();
}

static void
link(StringRef objectFile, StringRef outputFile) {
  auto clang = findProgramByName("clang++");
  string opt("-O");
  opt += optLevel;

  if (!clang) {
    report_fatal_error("Unable to find clang.");
  }
  vector<string> args{clang.get(), opt, "-o", outputFile, objectFile};

  for (auto& libPath : libPaths) {
    args.push_back("-L" + libPath);
  }

  for (auto& library : libraries) {
    args.push_back("-l" + library);
  }

  vector<char const*> charArgs;
  for (auto& arg : args) {
    charArgs.push_back(arg.c_str());
  }
  charArgs.push_back(nullptr);

  for (auto& arg : args) {
    outs() << arg.c_str() << " ";
  }
  outs() << "\n";

  string err;
  if (-1 == ExecuteAndWait(
                clang.get(), &charArgs[0], nullptr, nullptr, 0, 0, &err)) {
    report_fatal_error("Unable to link output file.");
  }
}

static void
generateBinary(Module& m, StringRef outputFilename) {
  // Compiling to native should allow things to keep working even when the
  // version of clang on the system and the version of LLVM used to compile
  // the tool don't quite match up.
  string objectFile = outputFilename.str() + ".o";
  compile(m, objectFile);
  link(objectFile, outputFilename);
}

static void
saveModule(Module const& m, StringRef filename) {
  std::error_code errc;
  raw_fd_ostream out(filename.data(), errc, sys::fs::F_None);

  if (errc) {
    report_fatal_error("error saving llvm module to '" + filename + "': \n"
                       + errc.message());
  }
  WriteBitcodeToFile(&m, out);
}

static void
prepareLinkingPaths(SmallString<32> invocationPath) {
  // First search the directory of the binary for the library, in case it is
  // all bundled together.
  sys::path::remove_filename(invocationPath);
  if (!invocationPath.empty()) {
    libPaths.push_back(invocationPath.str());
  }
// If the builder doesn't plan on installing it, we still need to get to the
// runtime library somehow, so just build in the path to the temporary one.
#ifdef CMAKE_INSTALL_PREFIX
  libPaths.push_back(CMAKE_INSTALL_PREFIX "/lib");
#elif defined(CMAKE_TEMP_LIBRARY_PATH)
  libPaths.push_back(CMAKE_TEMP_LIBRARY_PATH);
#elif defined(TEMP_LIBRARY_PATH)
  // This is a bit of a hack
  libPaths.push_back(TEMP_LIBRARY_PATH "/Debug+Asserts/lib/");
  libPaths.push_back(TEMP_LIBRARY_PATH "/Release+Asserts/lib/");
  libPaths.push_back(TEMP_LIBRARY_PATH "/Debug/lib/");
  libPaths.push_back(TEMP_LIBRARY_PATH "/Release/lib/");
#endif
  libraries.push_back(RUNTIME_LIB);
  libraries.push_back("rt");
}

static void
instrumentForDynamicCount(Module& m) {
  InitializeAllTargets();
  InitializeAllTargetMCs();
  InitializeAllAsmPrinters();
  InitializeAllAsmParsers();
  cl::AddExtraVersionPrinter(TargetRegistry::printRegisteredTargetsForVersion);

  if (outFile.getValue() == "") {
    errs() << "-o command line option must be specified.\n";
    exit(-1);
  }

  // Build up all of the passes that we want to run on the module.
  legacy::PassManager pm;
  pm.add(new callcounter::DynamicCallCounter());
  pm.add(createVerifierPass());
  pm.run(m);

  generateBinary(m, outFile);
  saveModule(m, outFile + ".callcounter.bc");
}

struct StaticCountPrinter : public ModulePass {
  static char ID;
  raw_ostream& out;

  explicit StaticCountPrinter(raw_ostream& out) : ModulePass(ID), out(out) {}

  bool
  runOnModule(Module& m) override {
    getAnalysis<callcounter::StaticCallCounter>().print(out, &m);
    return false;
  }

  void
  getAnalysisUsage(AnalysisUsage& au) const override {
    au.addRequired<callcounter::StaticCallCounter>();
    au.setPreservesAll();
  }
};

char StaticCountPrinter::ID = 0;

static void
countStaticCalls(Module& m) {
  // Build up all of the passes that we want to run on the module.
  legacy::PassManager pm;
  pm.add(new callcounter::StaticCallCounter());
  pm.add(new StaticCountPrinter(outs()));
  pm.run(m);
}

int
main(int argc, char** argv) {
  // This boilerplate provides convenient stack traces and clean LLVM exit
  // handling. It also initializes the built in support for convenient
  // command line option handling.
  sys::PrintStackTraceOnErrorSignal(argv[0]);
  llvm::PrettyStackTraceProgram X(argc, argv);
  llvm_shutdown_obj shutdown;
  cl::HideUnrelatedOptions(callCounterCategory);
  cl::ParseCommandLineOptions(argc, argv);

  // Construct an IR file from the filename passed on the command line.
  SMDiagnostic err;
  LLVMContext context;
  unique_ptr<Module> module = parseIRFile(inPath.getValue(), err, context);

  if (!module.get()) {
    errs() << "Error reading bitcode file: " << inPath << "\n";
    err.print(argv[0], errs());
    return -1;
  }

  if (AnalysisType::DYNAMIC == analysisType) {
    prepareLinkingPaths(StringRef(argv[0]));
    instrumentForDynamicCount(*module);
  } else {
    countStaticCalls(*module);
  }

  return 0;
}
<|endoftext|>


<commit_before>/*

  This program will add histograms from a list of root files and write them
  to a target root file. The target file is newly created and must not be
  identical to one of the source files.

  Syntax:

       hadd targetfile source1 source2 ...

  if the source files contains histograms and Trees, one can skip
  the Trees with
       hadd -T targetfile source1 source2 ...

  Authors: Rene Brun, Dirk Geppert, Sven A. Schmidt, sven.schmidt@cern.ch

 */

#include "RConfig.h"
#include <string>
#include "TChain.h"
#include "TFile.h"
#include "TH1.h"
#include "TKey.h"
#include "Riostream.h"

TList *FileList;
TFile *Target, *Source;
Bool_t noTrees;

void MergeRootfile( TDirectory *target, TList *sourcelist );

int main( int argc, char **argv ) {

  if ( argc < 4 || "-h" == string(argv[1]) || "--help" == string(argv[1]) ) {
    cout << "Usage: " << argv[0] << " [-f] [-T] targetfile source1 source2 [source3 ...]" << endl;
    cout << "This program will add histograms from a list of root files and write them" << endl;
    cout << "to a target root file. The target file is newly created and must not " << endl;
    cout << "exist, or if -f (\"force\") is given, must not be one of the source files." << endl;
    cout << "Supply at least two source files for this to make sense... ;-)" << endl;
    cout << "If the first argument is -T, Trees are not merged" <<endl;
    return 1;
  }
  FileList = new TList();

  Bool_t force = (!strcmp(argv[1],"-f") || !strcmp(argv[2],"-f"));
  noTrees = (!strcmp(argv[1],"-T") || !strcmp(argv[2],"-T"));

  int ffirst = 2;
  if (force) ffirst++;
  if (noTrees) ffirst++;

  cout << "Target file: " << argv[ffirst-1] << endl;
  Target = TFile::Open( argv[ffirst-1], (force?"RECREATE":"CREATE") );
  if (!Target || Target->IsZombie()) {
     cerr << "Error opening target file (does " << argv[ffirst-1] << " exist?)." << endl;
     cerr << "Pass \"-f\" argument to force re-creation of output file." << endl;
     exit(1);
  }

  // by default hadd can merge Trees in a file that can go up to 100 Gbytes
  Long64_t maxsize = 100000000; //100GB
  maxsize *= 100;  //to bypass some compiler limitations with big constants
  TTree::SetMaxTreeSize(maxsize);

  for ( int i = ffirst; i < argc; i++ ) {
    cout << "Source file " << i-ffirst+1 << ": " << argv[i] << endl;
    Source = TFile::Open( argv[i] );
    FileList->Add(Source);
  }

  MergeRootfile( Target, FileList );

  return 0;
}

void MergeRootfile( TDirectory *target, TList *sourcelist ) {

  //  cout << "Target path: " << target->GetPath() << endl;
  TString path( (char*)strstr( target->GetPath(), ":" ) );
  path.Remove( 0, 2 );

  TFile *first_source = (TFile*)sourcelist->First();
  first_source->cd( path );
  TDirectory *current_sourcedir = gDirectory;

  // loop over all keys in this directory
  TChain *globChain = 0;
  TIter nextkey( current_sourcedir->GetListOfKeys() );
  TKey *key, *oldkey=0;
  //gain time, do not add the objects in the list in memory
  TH1::AddDirectory(kFALSE);

  while ( (key = (TKey*)nextkey())) {

    //keep only the highest cycle number for each key
    if (oldkey && !strcmp(oldkey->GetName(),key->GetName())) continue;

    // read object from first source file
    first_source->cd( path );
    TObject *obj = key->ReadObj();

    if ( obj->IsA()->InheritsFrom( TH1::Class() ) ) {
      // descendant of TH1 -> merge it

      //      cout << "Merging histogram " << obj->GetName() << endl;
      TH1 *h1 = (TH1*)obj;
      TList listH;

      // loop over all source files and add the content of the
      // correspondant histogram to the one pointed to by "h1"
      TFile *nextsource = (TFile*)sourcelist->After( first_source );
      while ( nextsource ) {

        // make sure we are at the correct directory level by cd'ing to path
        nextsource->cd( path );
        TKey *key2 = (TKey*)gDirectory->GetListOfKeys()->FindObject(h1->GetName());
        if (key2) {
           listH.Add(key2->ReadObj());
           h1->Merge(&listH);
           listH.Delete();
        }

        nextsource = (TFile*)sourcelist->After( nextsource );
      }
    }
    else if ( obj->IsA()->InheritsFrom( "TTree" ) ) {

      // loop over all source files create a chain of Trees "globChain"
      if (!noTrees) {
         TString obj_name;
         if (path.Length()) {
            obj_name = path + "/" + obj->GetName();
         } else {
            obj_name = obj->GetName();
         }

         globChain = new TChain(obj_name);
         globChain->Add(first_source->GetName());
         TFile *nextsource = (TFile*)sourcelist->After( first_source );
         //      const char* file_name = nextsource->GetName();
         // cout << "file name  " << file_name << endl;
         while ( nextsource ) {
            globChain->Add(nextsource->GetName());
            nextsource = (TFile*)sourcelist->After( nextsource );
         }
      }

    } else if ( obj->IsA()->InheritsFrom( "TDirectory" ) ) {
      // it's a subdirectory

      cout << "Found subdirectory " << obj->GetName() << endl;

      // create a new subdir of same name and title in the target file
      target->cd();
      TDirectory *newdir = target->mkdir( obj->GetName(), obj->GetTitle() );

      // newdir is now the starting point of another round of merging
      // newdir still knows its depth within the target file via
      // GetPath(), so we can still figure out where we are in the recursion
      MergeRootfile( newdir, sourcelist );

    } else {

      // object is of no type that we know or can handle
      cout << "Unknown object type, name: "
           << obj->GetName() << " title: " << obj->GetTitle() << endl;
    }

    // now write the merged histogram (which is "in" obj) to the target file
    // note that this will just store obj in the current directory level,
    // which is not persistent until the complete directory itself is stored
    // by "target->Write()" below
    if ( obj ) {
       target->cd();

       //!!if the object is a tree, it is stored in globChain...
       if(obj->IsA()->InheritsFrom( "TTree" )) {
          if (!noTrees) {
             globChain->Merge(target->GetFile(),0,"keep");
             delete globChain;
          }
       } else {
          obj->Write( key->GetName() );
       }
    }
    oldkey = key;

  } // while ( ( TKey *key = (TKey*)nextkey() ) )

  // save modifications to target file
  target->SaveSelf(kTRUE);

}
<commit_msg>In the hadd main program we explicit delete the target file (closing it). Otherwise we may have a problem when the file destructor is called from the TROOT destructor and the TStreamerBase dictionary is already gone.<commit_after>/*

  This program will add histograms from a list of root files and write them
  to a target root file. The target file is newly created and must not be
  identical to one of the source files.

  Syntax:

       hadd targetfile source1 source2 ...

  if the source files contains histograms and Trees, one can skip
  the Trees with
       hadd -T targetfile source1 source2 ...

  Authors: Rene Brun, Dirk Geppert, Sven A. Schmidt, sven.schmidt@cern.ch

 */

#include "RConfig.h"
#include <string>
#include "TChain.h"
#include "TFile.h"
#include "TH1.h"
#include "TKey.h"
#include "Riostream.h"

TList *FileList;
TFile *Target, *Source;
Bool_t noTrees;

void MergeRootfile( TDirectory *target, TList *sourcelist );

int main( int argc, char **argv ) {

  if ( argc < 4 || "-h" == string(argv[1]) || "--help" == string(argv[1]) ) {
    cout << "Usage: " << argv[0] << " [-f] [-T] targetfile source1 source2 [source3 ...]" << endl;
    cout << "This program will add histograms from a list of root files and write them" << endl;
    cout << "to a target root file. The target file is newly created and must not " << endl;
    cout << "exist, or if -f (\"force\") is given, must not be one of the source files." << endl;
    cout << "Supply at least two source files for this to make sense... ;-)" << endl;
    cout << "If the first argument is -T, Trees are not merged" <<endl;
    return 1;
  }
  FileList = new TList();

  Bool_t force = (!strcmp(argv[1],"-f") || !strcmp(argv[2],"-f"));
  noTrees = (!strcmp(argv[1],"-T") || !strcmp(argv[2],"-T"));

  int ffirst = 2;
  if (force) ffirst++;
  if (noTrees) ffirst++;

  cout << "Target file: " << argv[ffirst-1] << endl;
  Target = TFile::Open( argv[ffirst-1], (force?"RECREATE":"CREATE") );
  if (!Target || Target->IsZombie()) {
     cerr << "Error opening target file (does " << argv[ffirst-1] << " exist?)." << endl;
     cerr << "Pass \"-f\" argument to force re-creation of output file." << endl;
     exit(1);
  }

  // by default hadd can merge Trees in a file that can go up to 100 Gbytes
  Long64_t maxsize = 100000000; //100GB
  maxsize *= 100;  //to bypass some compiler limitations with big constants
  TTree::SetMaxTreeSize(maxsize);

  for ( int i = ffirst; i < argc; i++ ) {
    cout << "Source file " << i-ffirst+1 << ": " << argv[i] << endl;
    Source = TFile::Open( argv[i] );
    FileList->Add(Source);
  }

  MergeRootfile( Target, FileList );

 //must delete Target to avoid a problem with dictionaries in~ TROOT
  delete Target;

  return 0;
}

void MergeRootfile( TDirectory *target, TList *sourcelist ) {

  //  cout << "Target path: " << target->GetPath() << endl;
  TString path( (char*)strstr( target->GetPath(), ":" ) );
  path.Remove( 0, 2 );

  TFile *first_source = (TFile*)sourcelist->First();
  first_source->cd( path );
  TDirectory *current_sourcedir = gDirectory;

  // loop over all keys in this directory
  TChain *globChain = 0;
  TIter nextkey( current_sourcedir->GetListOfKeys() );
  TKey *key, *oldkey=0;
  //gain time, do not add the objects in the list in memory
  TH1::AddDirectory(kFALSE);

  while ( (key = (TKey*)nextkey())) {

    //keep only the highest cycle number for each key
    if (oldkey && !strcmp(oldkey->GetName(),key->GetName())) continue;

    // read object from first source file
    first_source->cd( path );
    TObject *obj = key->ReadObj();

    if ( obj->IsA()->InheritsFrom( TH1::Class() ) ) {
      // descendant of TH1 -> merge it

      //      cout << "Merging histogram " << obj->GetName() << endl;
      TH1 *h1 = (TH1*)obj;
      TList listH;

      // loop over all source files and add the content of the
      // correspondant histogram to the one pointed to by "h1"
      TFile *nextsource = (TFile*)sourcelist->After( first_source );
      while ( nextsource ) {

        // make sure we are at the correct directory level by cd'ing to path
        nextsource->cd( path );
        TKey *key2 = (TKey*)gDirectory->GetListOfKeys()->FindObject(h1->GetName());
        if (key2) {
           listH.Add(key2->ReadObj());
           h1->Merge(&listH);
           listH.Delete();
        }

        nextsource = (TFile*)sourcelist->After( nextsource );
      }
    }
    else if ( obj->IsA()->InheritsFrom( "TTree" ) ) {

      // loop over all source files create a chain of Trees "globChain"
      if (!noTrees) {
         TString obj_name;
         if (path.Length()) {
            obj_name = path + "/" + obj->GetName();
         } else {
            obj_name = obj->GetName();
         }

         globChain = new TChain(obj_name);
         globChain->Add(first_source->GetName());
         TFile *nextsource = (TFile*)sourcelist->After( first_source );
         //      const char* file_name = nextsource->GetName();
         // cout << "file name  " << file_name << endl;
         while ( nextsource ) {
            globChain->Add(nextsource->GetName());
            nextsource = (TFile*)sourcelist->After( nextsource );
         }
      }

    } else if ( obj->IsA()->InheritsFrom( "TDirectory" ) ) {
      // it's a subdirectory

      cout << "Found subdirectory " << obj->GetName() << endl;

      // create a new subdir of same name and title in the target file
      target->cd();
      TDirectory *newdir = target->mkdir( obj->GetName(), obj->GetTitle() );

      // newdir is now the starting point of another round of merging
      // newdir still knows its depth within the target file via
      // GetPath(), so we can still figure out where we are in the recursion
      MergeRootfile( newdir, sourcelist );

    } else {

      // object is of no type that we know or can handle
      cout << "Unknown object type, name: "
           << obj->GetName() << " title: " << obj->GetTitle() << endl;
    }

    // now write the merged histogram (which is "in" obj) to the target file
    // note that this will just store obj in the current directory level,
    // which is not persistent until the complete directory itself is stored
    // by "target->Write()" below
    if ( obj ) {
       target->cd();

       //!!if the object is a tree, it is stored in globChain...
       if(obj->IsA()->InheritsFrom( "TTree" )) {
          if (!noTrees) {
             globChain->Merge(target->GetFile(),0,"keep");
             delete globChain;
          }
       } else {
          obj->Write( key->GetName() );
       }
    }
    oldkey = key;

  } // while ( ( TKey *key = (TKey*)nextkey() ) )

  // save modifications to target file
  target->SaveSelf(kTRUE);

}
<|endoftext|>


<commit_before>/*

  This program will add histograms from a list of root files and write them
  to a target root file. The target file is newly created and must not be
  identical to one of the source files.

  Syntax:

  hadd targetfile source1 source2 ...

  Author: Sven A. Schmidt, sven.schmidt@cern.ch
  Date:   13.2.2001

  This code is based on the hadd.C example by Rene Brun and Dirk Geppert,
  which had a problem with directories more than one level deep.
  (see macro hadd_old.C for this previous implementation).

  I have tested this macro on rootfiles with one and two dimensional
  histograms, and two levels of subdirectories. Feel free to send comments
  or bug reports to me.

 */

#include "TFile.h"
#include "TH1.h"
#include "TTree.h"
#include "TKey.h"
#include <string.h>
#include <iostream.h>

TList *FileList;
TFile *Target;

void MergeRootfile( TDirectory *target, TList *sourcelist );

int main( int argc, char **argv ) {

  if (argc < 4) {
     printf("******Error in invoking hadd\n");
     printf("===>  hadd targetfile source1 source2 ...\n");
     printf("      This program will add histograms from a list of root files and write them\n");
     printf("      to a target root file. The target file is newly created and must not be\n");
     printf("      identical to one of the source files.\n");
     printf("      supply at least two source files for this to make sense... ;-)\n");
     return 1;
  }
   FileList = new TList();

  cout << "Target file: " << argv[1] << endl;
  Target = TFile::Open( argv[1], "RECREATE" );

  for ( int i = 2; i < argc; i++ ) {
    cout << "Source file " << i-1 << ": " << argv[i] << endl;
    FileList->Add( TFile::Open( argv[i] ) );
  }

  MergeRootfile( Target, FileList );

}

void MergeRootfile( TDirectory *target, TList *sourcelist ) {
// Merge all files from sourcelist into the target directory.
// The directory level (depth) is determined by the target directory's
// current level

  //  cout << "Target path: " << target->GetPath() << endl;
  TString path( (char*)strstr( target->GetPath(), ":" ) );
  path.Remove( 0, 2 );

  TFile *first_source = (TFile*)sourcelist->First();
  first_source->cd( path );
  TDirectory *current_sourcedir = gDirectory;

  // loop over all keys in this directory
  TIter nextkey( current_sourcedir->GetListOfKeys() );
  TKey *key;
  while ( (key = (TKey*)nextkey() )) {

    // read object from first source file
    first_source->cd( path );
    TObject *obj = key->ReadObj();

    if ( obj->IsA()->InheritsFrom( "TH1" ) ) {
      // descendant of TH1 -> merge it

      //      cout << "Merging histogram " << obj->GetName() << endl;
      TH1 *h1 = (TH1*)obj;

      // loop over all source files and add the content of the
      // correspondant histogram to the one pointed to by "h1"
      TFile *nextsource = (TFile*)sourcelist->After( first_source );
      while ( nextsource ) {

        // make sure we are at the correct directory level by cd'ing to path
        nextsource->cd( path );
        TH1 *h2 = (TH1*)gDirectory->Get( h1->GetName() );
        if ( h2 ) {
          h1->Add( h2 );
          delete h2; // don't know if this is necessary, i.e. if
                     // h2 is created by the call to gDirectory above.
        }

        nextsource = (TFile*)sourcelist->After( nextsource );
      }

    } else if ( obj->IsA()->InheritsFrom( "TDirectory" ) ) {
      // it's a subdirectory

      cout << "Found subdirectory " << obj->GetName() << endl;

      // create a new subdir of same name and title in the target file
      target->cd();
      TDirectory *newdir = target->mkdir( obj->GetName(), obj->GetTitle() );

      // newdir is now the starting point of another round of merging
      // newdir still knows its depth within the target file via
      // GetPath(), so we can still figure out where we are in the recursion
      MergeRootfile( newdir, sourcelist );

    } else {
      // object is of no type that we know or can handle
      cout << "Unknown object type, name: "
           << obj->GetName() << " title: " << obj->GetTitle() << endl;
    }

    // now write the merged histogram (which is "in" obj) to the target file
    // note that this will just store obj in the current directory level,
    // which is not persistent until the complete directory itself is stored
    // by "target->Write()" below
    if ( obj ) {
      target->cd();
      obj->Write( key->GetName() );
    }

  } // while ( ( TKey *key = (TKey*)nextkey() ) )

  // save modifications to target file
  target->Write();

}
<commit_msg>Add a return statement in main program (required by some compilers).<commit_after>/*

  This program will add histograms from a list of root files and write them
  to a target root file. The target file is newly created and must not be
  identical to one of the source files.

  Syntax:

  hadd targetfile source1 source2 ...

  Author: Sven A. Schmidt, sven.schmidt@cern.ch
  Date:   13.2.2001

  This code is based on the hadd.C example by Rene Brun and Dirk Geppert,
  which had a problem with directories more than one level deep.
  (see macro hadd_old.C for this previous implementation).

  I have tested this macro on rootfiles with one and two dimensional
  histograms, and two levels of subdirectories. Feel free to send comments
  or bug reports to me.

 */

#include "TFile.h"
#include "TH1.h"
#include "TTree.h"
#include "TKey.h"
#include <string.h>
#include <iostream.h>

TList *FileList;
TFile *Target;

void MergeRootfile( TDirectory *target, TList *sourcelist );

int main( int argc, char **argv ) {

  if (argc < 4) {
     printf("******Error in invoking hadd\n");
     printf("===>  hadd targetfile source1 source2 ...\n");
     printf("      This program will add histograms from a list of root files and write them\n");
     printf("      to a target root file. The target file is newly created and must not be\n");
     printf("      identical to one of the source files.\n");
     printf("      supply at least two source files for this to make sense... ;-)\n");
     return 1;
  }
   FileList = new TList();

  cout << "Target file: " << argv[1] << endl;
  Target = TFile::Open( argv[1], "RECREATE" );

  for ( int i = 2; i < argc; i++ ) {
    cout << "Source file " << i-1 << ": " << argv[i] << endl;
    FileList->Add( TFile::Open( argv[i] ) );
  }

  MergeRootfile( Target, FileList );

  return 0;
}

void MergeRootfile( TDirectory *target, TList *sourcelist ) {
// Merge all files from sourcelist into the target directory.
// The directory level (depth) is determined by the target directory's
// current level

  //  cout << "Target path: " << target->GetPath() << endl;
  TString path( (char*)strstr( target->GetPath(), ":" ) );
  path.Remove( 0, 2 );

  TFile *first_source = (TFile*)sourcelist->First();
  first_source->cd( path );
  TDirectory *current_sourcedir = gDirectory;

  // loop over all keys in this directory
  TIter nextkey( current_sourcedir->GetListOfKeys() );
  TKey *key;
  while ( (key = (TKey*)nextkey() )) {

    // read object from first source file
    first_source->cd( path );
    TObject *obj = key->ReadObj();

    if ( obj->IsA()->InheritsFrom( "TH1" ) ) {
      // descendant of TH1 -> merge it

      //      cout << "Merging histogram " << obj->GetName() << endl;
      TH1 *h1 = (TH1*)obj;

      // loop over all source files and add the content of the
      // correspondant histogram to the one pointed to by "h1"
      TFile *nextsource = (TFile*)sourcelist->After( first_source );
      while ( nextsource ) {

        // make sure we are at the correct directory level by cd'ing to path
        nextsource->cd( path );
        TH1 *h2 = (TH1*)gDirectory->Get( h1->GetName() );
        if ( h2 ) {
          h1->Add( h2 );
          delete h2; // don't know if this is necessary, i.e. if
                     // h2 is created by the call to gDirectory above.
        }

        nextsource = (TFile*)sourcelist->After( nextsource );
      }

    } else if ( obj->IsA()->InheritsFrom( "TDirectory" ) ) {
      // it's a subdirectory

      cout << "Found subdirectory " << obj->GetName() << endl;

      // create a new subdir of same name and title in the target file
      target->cd();
      TDirectory *newdir = target->mkdir( obj->GetName(), obj->GetTitle() );

      // newdir is now the starting point of another round of merging
      // newdir still knows its depth within the target file via
      // GetPath(), so we can still figure out where we are in the recursion
      MergeRootfile( newdir, sourcelist );

    } else {
      // object is of no type that we know or can handle
      cout << "Unknown object type, name: "
           << obj->GetName() << " title: " << obj->GetTitle() << endl;
    }

    // now write the merged histogram (which is "in" obj) to the target file
    // note that this will just store obj in the current directory level,
    // which is not persistent until the complete directory itself is stored
    // by "target->Write()" below
    if ( obj ) {
      target->cd();
      obj->Write( key->GetName() );
    }

  } // while ( ( TKey *key = (TKey*)nextkey() ) )

  // save modifications to target file
  target->Write();

}
<|endoftext|>


<commit_before>/* -------------------------------------------------------------------------- */
/* Copyright 2002-2009, Distributed Systems Architecture Group, Universidad   */
/* Complutense de Madrid (dsa-research.org)                                   */
/*                                                                            */
/* Licensed under the Apache License, Version 2.0 (the "License"); you may    */
/* not use this file except in compliance with the License. You may obtain    */
/* a copy of the License at                                                   */
/*                                                                            */
/* http://www.apache.org/licenses/LICENSE-2.0                                 */
/*                                                                            */
/* Unless required by applicable law or agreed to in writing, software        */
/* distributed under the License is distributed on an "AS IS" BASIS,          */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
/* See the License for the specific language governing permissions and        */
/* limitations under the License.                                             */
/* -------------------------------------------------------------------------- */

#include <iostream>
#include <sstream>

#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <string.h>

#include "Mad.h"
#include "Nebula.h"

#include <cerrno>

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

Mad::~Mad()
{
    char    buf[]="FINALIZE\n";
    int     status;
    pid_t   rp;

    if ( pid==-1)
    {
        return;
    }

    // Finish the driver
    ::write(nebula_mad_pipe, buf, strlen(buf));

    close(mad_nebula_pipe);
    close(nebula_mad_pipe);

    rp = waitpid(pid, &status, WNOHANG);

    if ( rp == 0 )
    {
        sleep(1);
        waitpid(pid, &status, WNOHANG);
    }
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int Mad::start()
{
    int                            ne_mad_pipe[2];
    int                            mad_ne_pipe[2];

    map<string,string>::iterator   it;

    const char *                   owner = 0;
    const char *                   executable = 0;
    const char *                   arguments = 0;
    string                         exec_path;

    char                           buf[]="INIT\n";
    char                           c;

    stringbuf                      sbuf;
    iostream                       mstream(&sbuf);
    int                            rc;

    string                         action;
    string                         result;
    string                         info;

    ostringstream                  oss;

    // Open communication pipes

    if (pipe(ne_mad_pipe) == -1 ||
        pipe(mad_ne_pipe) == -1)
    {
        goto error_pipes;
    }

    // Get the attributes for this driver

    it = attributes.find("OWNER");

    if ( it != attributes.end() )
    {
        owner = it->second.c_str();
    }

    it = attributes.find("EXECUTABLE");

    if ( it != attributes.end() )
    {
        if ( it->second.empty() == false )
        {
            if (it->second[0] != '/') //Look in ONE_LOCATION/lib/mads or in "/usr/lib/one/mads"
            {
                Nebula& nd = Nebula::instance();

                exec_path = nd.get_mad_location() + it->second;
                executable= exec_path.c_str();
            }
            else //Absolute Path
            {
                executable = it->second.c_str();
            }
        }
    }

    it = attributes.find("ARGUMENTS");

    if ( it != attributes.end() )
    {
        arguments = it->second.c_str();
    }

    if ( (sudo_execution == true && owner == 0) || executable == 0 )
    {
        goto error_attributes;
    }

    //Create a new process for the driver

    pid = fork();

    switch (pid)
    {
    case -1: // Error
        goto error_fork;

    case 0:  // Child process (MAD)

        close(ne_mad_pipe[1]);
        close(mad_ne_pipe[0]);

        if (dup2(ne_mad_pipe[0], 0) != 0 || // stdin and stdout redirection
            dup2(mad_ne_pipe[1], 1) != 1)
        {
            goto error_dup2;
        }

        close(ne_mad_pipe[0]);
        close(mad_ne_pipe[1]);

        close(2);

        if ( sudo_execution == true )
        {
            rc = execlp("sudo","sudo","-H","-u",owner,executable,arguments,NULL);
        }
        else
        {
            rc = execlp(executable,executable,arguments,NULL);
        }

        goto error_exec;

    default: // Parent process (SE)
        fd_set          rfds;
        struct timeval  tv;

        close(ne_mad_pipe[0]);
        close(mad_ne_pipe[1]);

        nebula_mad_pipe = ne_mad_pipe[1];
        mad_nebula_pipe = mad_ne_pipe[0];

        // Close pipes in other MADs

        fcntl(nebula_mad_pipe, F_SETFD, FD_CLOEXEC);
        fcntl(mad_nebula_pipe, F_SETFD, FD_CLOEXEC);

        ::write(nebula_mad_pipe, buf, strlen(buf));

        do
        {
            FD_ZERO(&rfds);
            FD_SET(mad_nebula_pipe, &rfds);

            // Wait up to one 0.5 seconds
            tv.tv_sec  = 0;
            tv.tv_usec = 500000;

            rc = select(mad_nebula_pipe+1,&rfds,0,0, &tv);

            if ( rc <= 0 ) // MAD did not answered
            {
                goto error_mad_init;
            }

            rc = read(mad_nebula_pipe, (void *) &c, sizeof(char));
            mstream.put(c);
        }
        while ( rc > 0 && c != '\n');

        if ( rc <= 0 )
        {
            goto error_mad_init;
        }

        mstream >> action >> result >> ws;
        getline(mstream,info);

        if (action == "INIT")
        {
            if (result == "FAILURE")
            {
                goto error_mad_result;
            }
        }
        else
        {
            goto error_mad_action;
        }

        break;
    }

    return 0;

error_exec:
    oss.str("");
    oss << "Can not load driver " << executable << ", " << strerror(errno);
    Nebula::log("MAD", Log::ERROR, oss);
    exit(-1);

error_dup2:
    oss.str("");
    oss << "Can not duplicate descriptors, " << strerror(errno);
    Nebula::log("MAD", Log::ERROR, oss);
    exit(-1);

error_mad_result:
    oss.str("");
    oss << "MAD initialization failed, " << info;
    Nebula::log("MAD", Log::ERROR, oss);
    return -1;

error_mad_action:
    Nebula::log("MAD", Log::ERROR,"Wrong action in MAD response");
    return -1;

error_mad_init:
    Nebula::log("MAD", Log::ERROR, "MAD did not answer INIT command");
    return -1;

error_fork:
    oss.str("");
    oss << "Error forking to start MAD, " << strerror(errno);
    Nebula::log("MAD", Log::ERROR, oss);
    return -1;

error_attributes:
    Nebula::log("MAD", Log::ERROR, "Wrong attributes for the driver");
    return -1;

error_pipes:
    oss.str("");
    oss << "Can not create driver pipes, " << strerror(errno);
    Nebula::log("MAD", Log::ERROR, oss);
    return -1;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int Mad::reload()
{
    char    buf[]="FINALIZE\n";
    int     status;
    int     rc;
    pid_t   rp;

    // Finish the driver

    ::write(nebula_mad_pipe, buf, strlen(buf));

    close(nebula_mad_pipe);
    close(mad_nebula_pipe);

    rp = waitpid(pid, &status, WNOHANG);

    if ( rp == 0 )
    {
        sleep(1);
        waitpid(pid, &status, WNOHANG);
    }

    // Start the MAD again

    rc = start();

    return rc;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

<commit_msg>Mad loading waits 5 seconds<commit_after>/* -------------------------------------------------------------------------- */
/* Copyright 2002-2009, Distributed Systems Architecture Group, Universidad   */
/* Complutense de Madrid (dsa-research.org)                                   */
/*                                                                            */
/* Licensed under the Apache License, Version 2.0 (the "License"); you may    */
/* not use this file except in compliance with the License. You may obtain    */
/* a copy of the License at                                                   */
/*                                                                            */
/* http://www.apache.org/licenses/LICENSE-2.0                                 */
/*                                                                            */
/* Unless required by applicable law or agreed to in writing, software        */
/* distributed under the License is distributed on an "AS IS" BASIS,          */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
/* See the License for the specific language governing permissions and        */
/* limitations under the License.                                             */
/* -------------------------------------------------------------------------- */

#include <iostream>
#include <sstream>

#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <string.h>

#include "Mad.h"
#include "Nebula.h"

#include <cerrno>

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

Mad::~Mad()
{
    char    buf[]="FINALIZE\n";
    int     status;
    pid_t   rp;

    if ( pid==-1)
    {
        return;
    }

    // Finish the driver
    ::write(nebula_mad_pipe, buf, strlen(buf));

    close(mad_nebula_pipe);
    close(nebula_mad_pipe);

    rp = waitpid(pid, &status, WNOHANG);

    if ( rp == 0 )
    {
        sleep(1);
        waitpid(pid, &status, WNOHANG);
    }
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int Mad::start()
{
    int                            ne_mad_pipe[2];
    int                            mad_ne_pipe[2];

    map<string,string>::iterator   it;

    const char *                   owner = 0;
    const char *                   executable = 0;
    const char *                   arguments = 0;
    string                         exec_path;

    char                           buf[]="INIT\n";
    char                           c;

    stringbuf                      sbuf;
    iostream                       mstream(&sbuf);
    int                            rc;

    string                         action;
    string                         result;
    string                         info;

    ostringstream                  oss;

    // Open communication pipes

    if (pipe(ne_mad_pipe) == -1 ||
        pipe(mad_ne_pipe) == -1)
    {
        goto error_pipes;
    }

    // Get the attributes for this driver

    it = attributes.find("OWNER");

    if ( it != attributes.end() )
    {
        owner = it->second.c_str();
    }

    it = attributes.find("EXECUTABLE");

    if ( it != attributes.end() )
    {
        if ( it->second.empty() == false )
        {
            if (it->second[0] != '/') //Look in ONE_LOCATION/lib/mads or in "/usr/lib/one/mads"
            {
                Nebula& nd = Nebula::instance();

                exec_path = nd.get_mad_location() + it->second;
                executable= exec_path.c_str();
            }
            else //Absolute Path
            {
                executable = it->second.c_str();
            }
        }
    }

    it = attributes.find("ARGUMENTS");

    if ( it != attributes.end() )
    {
        arguments = it->second.c_str();
    }

    if ( (sudo_execution == true && owner == 0) || executable == 0 )
    {
        goto error_attributes;
    }

    //Create a new process for the driver

    pid = fork();

    switch (pid)
    {
    case -1: // Error
        goto error_fork;

    case 0:  // Child process (MAD)

        close(ne_mad_pipe[1]);
        close(mad_ne_pipe[0]);

        if (dup2(ne_mad_pipe[0], 0) != 0 || // stdin and stdout redirection
            dup2(mad_ne_pipe[1], 1) != 1)
        {
            goto error_dup2;
        }

        close(ne_mad_pipe[0]);
        close(mad_ne_pipe[1]);

        close(2);

        if ( sudo_execution == true )
        {
            rc = execlp("sudo","sudo","-H","-u",owner,executable,arguments,NULL);
        }
        else
        {
            rc = execlp(executable,executable,arguments,NULL);
        }

        goto error_exec;

    default: // Parent process (SE)
        fd_set          rfds;
        struct timeval  tv;

        close(ne_mad_pipe[0]);
        close(mad_ne_pipe[1]);

        nebula_mad_pipe = ne_mad_pipe[1];
        mad_nebula_pipe = mad_ne_pipe[0];

        // Close pipes in other MADs

        fcntl(nebula_mad_pipe, F_SETFD, FD_CLOEXEC);
        fcntl(mad_nebula_pipe, F_SETFD, FD_CLOEXEC);

        ::write(nebula_mad_pipe, buf, strlen(buf));

        do
        {
            FD_ZERO(&rfds);
            FD_SET(mad_nebula_pipe, &rfds);

            // Wait up to 5 seconds
            tv.tv_sec  = 5;
            tv.tv_usec = 0;

            rc = select(mad_nebula_pipe+1,&rfds,0,0, &tv);

            if ( rc <= 0 ) // MAD did not answered
            {
                goto error_mad_init;
            }

            rc = read(mad_nebula_pipe, (void *) &c, sizeof(char));
            mstream.put(c);
        }
        while ( rc > 0 && c != '\n');

        if ( rc <= 0 )
        {
            goto error_mad_init;
        }

        mstream >> action >> result >> ws;
        getline(mstream,info);

        if (action == "INIT")
        {
            if (result == "FAILURE")
            {
                goto error_mad_result;
            }
        }
        else
        {
            goto error_mad_action;
        }

        break;
    }

    return 0;

error_exec:
    oss.str("");
    oss << "Can not load driver " << executable << ", " << strerror(errno);
    Nebula::log("MAD", Log::ERROR, oss);
    exit(-1);

error_dup2:
    oss.str("");
    oss << "Can not duplicate descriptors, " << strerror(errno);
    Nebula::log("MAD", Log::ERROR, oss);
    exit(-1);

error_mad_result:
    oss.str("");
    oss << "MAD initialization failed, " << info;
    Nebula::log("MAD", Log::ERROR, oss);
    return -1;

error_mad_action:
    Nebula::log("MAD", Log::ERROR,"Wrong action in MAD response");
    return -1;

error_mad_init:
    Nebula::log("MAD", Log::ERROR, "MAD did not answer INIT command");
    return -1;

error_fork:
    oss.str("");
    oss << "Error forking to start MAD, " << strerror(errno);
    Nebula::log("MAD", Log::ERROR, oss);
    return -1;

error_attributes:
    Nebula::log("MAD", Log::ERROR, "Wrong attributes for the driver");
    return -1;

error_pipes:
    oss.str("");
    oss << "Can not create driver pipes, " << strerror(errno);
    Nebula::log("MAD", Log::ERROR, oss);
    return -1;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

int Mad::reload()
{
    char    buf[]="FINALIZE\n";
    int     status;
    int     rc;
    pid_t   rp;

    // Finish the driver

    ::write(nebula_mad_pipe, buf, strlen(buf));

    close(nebula_mad_pipe);
    close(mad_nebula_pipe);

    rp = waitpid(pid, &status, WNOHANG);

    if ( rp == 0 )
    {
        sleep(1);
        waitpid(pid, &status, WNOHANG);
    }

    // Start the MAD again

    rc = start();

    return rc;
}

/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2000, 2010 Oracle and/or its affiliates.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

#ifndef _TOOLKIT_CONTROLS_UNOCONTROLCONTAINERMODEL_HXX_
#define _TOOLKIT_CONTROLS_UNOCONTROLCONTAINERMODEL_HXX_

#include <toolkit/controls/unocontrolmodel.hxx>
#include <toolkit/helper/macros.hxx>
#include <toolkit/helper/servicenames.hxx>

//  ----------------------------------------------------
//  class ::com::sun::star::awt::UnoControlContainerModel
//  ----------------------------------------------------
class UnoControlContainerModel : public UnoControlModel
{
protected:
    ::com::sun::star::uno::Any                                                      ImplGetDefaultValue( sal_uInt16 nPropId ) const;
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertySetInfo >   getPropertySetInfo() const;
    ::cppu::IPropertyArrayHelper&                                                   SAL_CALL getInfoHelper();

public:
                        UnoControlContainerModel( const ::com::sun::star::uno::Reference< ::com::sun::star::lang::XMultiServiceFactory >& i_factory );
                        UnoControlContainerModel( const UnoControlContainerModel& rModel ) : UnoControlModel( rModel ) {;}

    UnoControlModel*    Clone() const { return new UnoControlContainerModel( *this ); }

    // ::com::sun::star::beans::XMultiPropertySet
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertySetInfo > SAL_CALL getPropertySetInfo(  ) throw(::com::sun::star::uno::RuntimeException);

    // ::com::sun::star::io::XPersistObject
    ::rtl::OUString SAL_CALL getServiceName() throw(::com::sun::star::uno::RuntimeException);

    // ::com::sun::star::lang::XServiceInfo
    DECLIMPL_SERVICEINFO_DERIVED( UnoControlContainerModel, UnoControlModel, szServiceName2_UnoControlContainerModel )
};

#endif // _TOOLKIT_CONTROLS_UNOCONTROLCONTAINERMODEL_HXX_

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<commit_msg>remove stray semicolon<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2000, 2010 Oracle and/or its affiliates.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

#ifndef _TOOLKIT_CONTROLS_UNOCONTROLCONTAINERMODEL_HXX_
#define _TOOLKIT_CONTROLS_UNOCONTROLCONTAINERMODEL_HXX_

#include <toolkit/controls/unocontrolmodel.hxx>
#include <toolkit/helper/macros.hxx>
#include <toolkit/helper/servicenames.hxx>

//  ----------------------------------------------------
//  class ::com::sun::star::awt::UnoControlContainerModel
//  ----------------------------------------------------
class UnoControlContainerModel : public UnoControlModel
{
protected:
    ::com::sun::star::uno::Any                                                      ImplGetDefaultValue( sal_uInt16 nPropId ) const;
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertySetInfo >   getPropertySetInfo() const;
    ::cppu::IPropertyArrayHelper&                                                   SAL_CALL getInfoHelper();

public:
                        UnoControlContainerModel( const ::com::sun::star::uno::Reference< ::com::sun::star::lang::XMultiServiceFactory >& i_factory );
                        UnoControlContainerModel( const UnoControlContainerModel& rModel ) : UnoControlModel( rModel ) {}

    UnoControlModel*    Clone() const { return new UnoControlContainerModel( *this ); }

    // ::com::sun::star::beans::XMultiPropertySet
    ::com::sun::star::uno::Reference< ::com::sun::star::beans::XPropertySetInfo > SAL_CALL getPropertySetInfo(  ) throw(::com::sun::star::uno::RuntimeException);

    // ::com::sun::star::io::XPersistObject
    ::rtl::OUString SAL_CALL getServiceName() throw(::com::sun::star::uno::RuntimeException);

    // ::com::sun::star::lang::XServiceInfo
    DECLIMPL_SERVICEINFO_DERIVED( UnoControlContainerModel, UnoControlModel, szServiceName2_UnoControlContainerModel )
};

#endif // _TOOLKIT_CONTROLS_UNOCONTROLCONTAINERMODEL_HXX_

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<|endoftext|>


<commit_before>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "lid_space_compaction_job_take2.h"
#include "i_document_scan_iterator.h"
#include "i_lid_space_compaction_handler.h"
#include "i_operation_storer.h"
#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
#include <vespa/searchcorespi/index/i_thread_service.h>
#include <vespa/persistence/spi/bucket_tasks.h>
#include <vespa/document/fieldvalue/document.h>
#include <vespa/vespalib/util/destructor_callbacks.h>
#include <vespa/vespalib/util/lambdatask.h>
#include <cassert>
#include <thread>

using search::DocumentMetaData;
using search::LidUsageStats;
using storage::spi::makeBucketTask;
using storage::spi::Bucket;
using vespalib::makeLambdaTask;

namespace proton::lidspace {

namespace {

class IncOnDestruct {
public:
    IncOnDestruct(std::atomic<size_t> & count) : _count(count) {}
    ~IncOnDestruct() {
        _count.fetch_add(1, std::memory_order_relaxed);
    }
private:
    std::atomic<size_t> & _count;
};

bool
isSameDocument(const search::DocumentMetaData & a, const search::DocumentMetaData & b) {
    return (a.lid == b.lid) &&
           (a.bucketId == b.bucketId) &&
           (a.gid == b.gid);
}

}

void
CompactionJob::failOperation() {
    _executedCount.fetch_add(1, std::memory_order_relaxed);
    _scanItr.reset();
}

bool
CompactionJob::scanDocuments(const LidUsageStats &stats)
{
    if (_scanItr->valid()) {
        DocumentMetaData document = getNextDocument(stats, false);
        if (document.valid()) {
            Bucket metaBucket(document::Bucket(_bucketSpace, document.bucketId));
            IDestructorCallback::SP context = getLimiter().beginOperation();
            auto bucketTask = makeBucketTask([this, meta=document, opsTracker=std::move(context)]
            (const Bucket & bucket, std::shared_ptr<IDestructorCallback> onDone) {
                assert(bucket.getBucketId() == meta.bucketId);
                using DoneContext = vespalib::KeepAlive<std::pair<IDestructorCallback::SP, IDestructorCallback::SP>>;
                moveDocument(meta, std::make_shared<DoneContext>(std::make_pair(std::move(opsTracker), std::move(onDone))));
            }, [this](const Bucket &) { _master.execute(makeLambdaTask([this] { failOperation(); } )); });

            _startedCount.fetch_add(1, std::memory_order_relaxed);
            _bucketExecutor.execute(metaBucket, std::move(bucketTask));
            if (isBlocked(BlockedReason::OUTSTANDING_OPS)) {
                return true;
            }
        }
    }
    return false;
}

void
CompactionJob::moveDocument(const search::DocumentMetaData & metaThen, std::shared_ptr<IDestructorCallback> context) {
    IncOnDestruct countGuard(_executedCount);
    if (_stopped.load(std::memory_order_relaxed)) return;
    // The real lid must be sampled in the master thread.
    //TODO remove target lid from createMoveOperation interface
    // Reread meta data as document might have been altered after move was initiated
    // If so it will fail the timestamp sanity check later on.
    search::DocumentMetaData metaNow = _handler->getMetaData(metaThen.lid);
    if ( ! isSameDocument(metaThen, metaNow)) return;
    auto op = _handler->createMoveOperation(metaNow, 0);
    if (!op || !op->getDocument()) return;
    // Early detection and force md5 calculation outside of master thread
    if (metaThen.gid != op->getDocument()->getId().getGlobalId()) return;

    _master.execute(makeLambdaTask([this, metaNow, moveOp=std::move(op), onDone=std::move(context)]() mutable {
        if (_stopped.load(std::memory_order_relaxed)) return;
        completeMove(metaNow, std::move(moveOp), std::move(onDone));
    }));
}

void
CompactionJob::completeMove(const search::DocumentMetaData & metaThen, std::unique_ptr<MoveOperation> moveOp,
                            std::shared_ptr<IDestructorCallback> onDone)
{
    search::DocumentMetaData metaNow = _handler->getMetaData(metaThen.lid);
    // This should be impossible and should probably be an assert
    if ( ! isSameDocument(metaThen, metaNow)) return;
    if (metaNow.gid != moveOp->getDocument()->getId().getGlobalId()) return;

    uint32_t lowestLid = _handler->getLidStatus().getLowestFreeLid();
    if (lowestLid >= metaNow.lid) return;
    moveOp->setTargetLid(lowestLid);
    _opStorer.appendOperation(*moveOp, onDone);
    _handler->handleMove(*moveOp, std::move(onDone));
}

CompactionJob::CompactionJob(const DocumentDBLidSpaceCompactionConfig &config,
                             std::shared_ptr<ILidSpaceCompactionHandler> handler,
                             IOperationStorer &opStorer,
                             IThreadService & master,
                             BucketExecutor & bucketExecutor,
                             IDiskMemUsageNotifier &diskMemUsageNotifier,
                             const BlockableMaintenanceJobConfig &blockableConfig,
                             IClusterStateChangedNotifier &clusterStateChangedNotifier,
                             bool nodeRetired,
                             document::BucketSpace bucketSpace)
    : LidSpaceCompactionJobBase(config, std::move(handler), opStorer, diskMemUsageNotifier,
                                blockableConfig, clusterStateChangedNotifier, nodeRetired),
      _master(master),
      _bucketExecutor(bucketExecutor),
      _bucketSpace(bucketSpace),
      _stopped(false),
      _startedCount(0),
      _executedCount(0)
{ }

CompactionJob::~CompactionJob() = default;

bool
CompactionJob::inSync() const {
    return _executedCount == _startedCount;
}

void
CompactionJob::onStop() {
    _stopped = true;
    while ( ! inSync() ) {
        std::this_thread::sleep_for(1ms);
    }
}

} // namespace proton
<commit_msg>Also verify timestamp<commit_after>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "lid_space_compaction_job_take2.h"
#include "i_document_scan_iterator.h"
#include "i_lid_space_compaction_handler.h"
#include "i_operation_storer.h"
#include <vespa/searchcore/proton/feedoperation/moveoperation.h>
#include <vespa/searchcorespi/index/i_thread_service.h>
#include <vespa/persistence/spi/bucket_tasks.h>
#include <vespa/document/fieldvalue/document.h>
#include <vespa/vespalib/util/destructor_callbacks.h>
#include <vespa/vespalib/util/lambdatask.h>
#include <cassert>
#include <thread>

using search::DocumentMetaData;
using search::LidUsageStats;
using storage::spi::makeBucketTask;
using storage::spi::Bucket;
using vespalib::makeLambdaTask;

namespace proton::lidspace {

namespace {

class IncOnDestruct {
public:
    IncOnDestruct(std::atomic<size_t> & count) : _count(count) {}
    ~IncOnDestruct() {
        _count.fetch_add(1, std::memory_order_relaxed);
    }
private:
    std::atomic<size_t> & _count;
};

bool
isSameDocument(const search::DocumentMetaData & a, const search::DocumentMetaData & b) {
    return (a.lid == b.lid) &&
           (a.bucketId == b.bucketId) &&
           (a.gid == b.gid) &&
           (a.timestamp == b.timestamp); // Timestamp check can be removed once logic has proved itself in large scale.
}

}

void
CompactionJob::failOperation() {
    _executedCount.fetch_add(1, std::memory_order_relaxed);
    _scanItr.reset();
}

bool
CompactionJob::scanDocuments(const LidUsageStats &stats)
{
    if (_scanItr->valid()) {
        DocumentMetaData document = getNextDocument(stats, false);
        if (document.valid()) {
            Bucket metaBucket(document::Bucket(_bucketSpace, document.bucketId));
            IDestructorCallback::SP context = getLimiter().beginOperation();
            auto bucketTask = makeBucketTask([this, meta=document, opsTracker=std::move(context)]
            (const Bucket & bucket, std::shared_ptr<IDestructorCallback> onDone) {
                assert(bucket.getBucketId() == meta.bucketId);
                using DoneContext = vespalib::KeepAlive<std::pair<IDestructorCallback::SP, IDestructorCallback::SP>>;
                moveDocument(meta, std::make_shared<DoneContext>(std::make_pair(std::move(opsTracker), std::move(onDone))));
            }, [this](const Bucket &) { _master.execute(makeLambdaTask([this] { failOperation(); } )); });

            _startedCount.fetch_add(1, std::memory_order_relaxed);
            _bucketExecutor.execute(metaBucket, std::move(bucketTask));
            if (isBlocked(BlockedReason::OUTSTANDING_OPS)) {
                return true;
            }
        }
    }
    return false;
}

void
CompactionJob::moveDocument(const search::DocumentMetaData & metaThen, std::shared_ptr<IDestructorCallback> context) {
    IncOnDestruct countGuard(_executedCount);
    if (_stopped.load(std::memory_order_relaxed)) return;
    // The real lid must be sampled in the master thread.
    //TODO remove target lid from createMoveOperation interface
    // Reread meta data as document might have been altered after move was initiated
    // If so it will fail the timestamp sanity check later on.
    search::DocumentMetaData metaNow = _handler->getMetaData(metaThen.lid);
    if ( ! isSameDocument(metaThen, metaNow)) return;
    auto op = _handler->createMoveOperation(metaNow, 0);
    if (!op || !op->getDocument()) return;
    // Early detection and force md5 calculation outside of master thread
    if (metaThen.gid != op->getDocument()->getId().getGlobalId()) return;

    _master.execute(makeLambdaTask([this, metaNow, moveOp=std::move(op), onDone=std::move(context)]() mutable {
        if (_stopped.load(std::memory_order_relaxed)) return;
        completeMove(metaNow, std::move(moveOp), std::move(onDone));
    }));
}

void
CompactionJob::completeMove(const search::DocumentMetaData & metaThen, std::unique_ptr<MoveOperation> moveOp,
                            std::shared_ptr<IDestructorCallback> onDone)
{
    search::DocumentMetaData metaNow = _handler->getMetaData(metaThen.lid);
    // This should be impossible and should probably be an assert
    if ( ! isSameDocument(metaThen, metaNow)) return;
    if (metaNow.gid != moveOp->getDocument()->getId().getGlobalId()) return;

    uint32_t lowestLid = _handler->getLidStatus().getLowestFreeLid();
    if (lowestLid >= metaNow.lid) return;
    moveOp->setTargetLid(lowestLid);
    _opStorer.appendOperation(*moveOp, onDone);
    _handler->handleMove(*moveOp, std::move(onDone));
}

CompactionJob::CompactionJob(const DocumentDBLidSpaceCompactionConfig &config,
                             std::shared_ptr<ILidSpaceCompactionHandler> handler,
                             IOperationStorer &opStorer,
                             IThreadService & master,
                             BucketExecutor & bucketExecutor,
                             IDiskMemUsageNotifier &diskMemUsageNotifier,
                             const BlockableMaintenanceJobConfig &blockableConfig,
                             IClusterStateChangedNotifier &clusterStateChangedNotifier,
                             bool nodeRetired,
                             document::BucketSpace bucketSpace)
    : LidSpaceCompactionJobBase(config, std::move(handler), opStorer, diskMemUsageNotifier,
                                blockableConfig, clusterStateChangedNotifier, nodeRetired),
      _master(master),
      _bucketExecutor(bucketExecutor),
      _bucketSpace(bucketSpace),
      _stopped(false),
      _startedCount(0),
      _executedCount(0)
{ }

CompactionJob::~CompactionJob() = default;

bool
CompactionJob::inSync() const {
    return _executedCount == _startedCount;
}

void
CompactionJob::onStop() {
    _stopped = true;
    while ( ! inSync() ) {
        std::this_thread::sleep_for(1ms);
    }
}

} // namespace proton
<|endoftext|>


<commit_before>/*********************************************************************************
 *
 * Copyright (c) 2017 Josef Adamsson, Denise Härnström
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2017 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#include <modules/crystalvisualization/processors/structuremesh.h>
#include <inviwo/core/datastructures/geometry/basicmesh.h>

#include <inviwo/core/interaction/events/pickingevent.h>
#include <inviwo/core/interaction/events/mouseevent.h>
#include <inviwo/core/datastructures/buffer/bufferramprecision.h>
#include <inviwo/core/datastructures/buffer/buffer.h>

namespace inviwo {

// The Class Identifier has to be globally unique. Use a reverse DNS naming scheme
const ProcessorInfo StructureMesh::processorInfo_{
    "envision.StructureMesh",      // Class identifier
    "Structure Mesh",                // Display name
    "Crystal",              // Category
    CodeState::Experimental,  // Code state
    Tags::None,               // Tags
};
const ProcessorInfo StructureMesh::getProcessorInfo() const {
    return processorInfo_;
}

StructureMesh::StructureMesh()
    : Processor()
    , structure_("coordinates")
    , mesh_("mesh")
    , scalingFactor_("scalingFactor", "Scaling factor", 1.f)
    , basis_("basis", "Basis", glm::mat3x3())
    , fullMesh_("fullMesh", "Full mesh", false)
    , animation_("animation", "Animation", false)
    , timestep_("timestep", "Time step", false)
    , enablePicking_("enablePicking", "Enable Picking", false)
    , spherePicking_(this, 0, [&](PickingEvent* p) { handlePicking(p); })
    , inds_("inds", "Picked atoms") {
    addPort(structure_);
    addPort(mesh_);
    addProperty(scalingFactor_);
    addProperty(basis_);
    addProperty(fullMesh_);
    addProperty(animation_);
    addProperty(timestep_);
    addProperty(enablePicking_);
    addProperty(inds_);

    structure_.onChange([&](){
        const auto data = structure_.getVectorData();
        for(int i = colors_.size(); i < data.size(); ++i) {
            colors_.push_back(std::make_unique<FloatVec4Property>("color" + toString(i), "Color " + toString(i),
                                                vec4(1.0f, 1.0f, 1.0f, 1.0f), vec4(0.0f), vec4(1.0f), vec4(0.01f),
                                                InvalidationLevel::InvalidOutput, PropertySemantics::Color));
            addProperty(colors_.back().get(), false);

            radii_.push_back(std::make_unique<FloatProperty>("radius"+ toString(i), "Atom Radius"+ toString(i), 1.0f));
            addProperty(radii_.back().get(), false);
            num_.push_back(std::make_unique<IntProperty>("atoms" + toString(i), "Atoms" + toString(i), 0));
            addProperty(num_.back().get(), false);
        }
        int numSpheres = 0;
        for (const auto &strucs : structure_)
            numSpheres += strucs->size();
        spherePicking_.resize(numSpheres);

    });

}

void StructureMesh::process() {
    if (fullMesh_.get()) {
	auto mesh = std::make_shared<BasicMesh>();
	mesh_.setData(mesh);
	size_t ind = 0;
	for (const auto &strucs : structure_) {
	    for (long long j = 0; j < static_cast<long long>(strucs->size()); ++j) {
		auto center = scalingFactor_.get() * basis_.get() * strucs->at(j) - 0.5f * (basis_.get()[0] + basis_.get()[1] + basis_.get()[2]);
		BasicMesh::sphere(center, radii_[ind]->get(), colors_[ind]->get(), mesh);
	    }
	    ++ind;
	}
    } else {
        if (structure_.isChanged() || animation_.isModified() || std::any_of(num_.begin(), num_.end(),
            [](const std::unique_ptr<IntProperty>& property) {
            return property->isModified();
            }) || enablePicking_.isModified()) {
            int numSpheres = 0;
            size_t pInd = 0;
            for (const auto &strucs : structure_) {
                if (animation_.get()) {
                    numSpheres += num_[pInd]->get();
                    ++pInd;
                }
                else {
                    numSpheres += strucs->size();
                }
            }
            vertexRAM_ = std::make_shared<BufferRAMPrecision<vec3>>(numSpheres);
            colorRAM_ = std::make_shared<BufferRAMPrecision<vec4>>(numSpheres);
            radiiRAM_ = std::make_shared<BufferRAMPrecision<float>>(numSpheres);
            auto mesh = std::make_shared<Mesh>(DrawType::Points, ConnectivityType::None);
            mesh->addBuffer(Mesh::BufferInfo(BufferType::PositionAttrib),
                std::make_shared<Buffer<vec3>>(vertexRAM_));
            mesh->addBuffer(Mesh::BufferInfo(BufferType::ColorAttrib),
                std::make_shared<Buffer<vec4>>(colorRAM_));
            mesh->addBuffer(Mesh::BufferInfo(BufferType::NumberOfBufferTypes, 5),
                std::make_shared<Buffer<float>>(radiiRAM_));

            mesh_.setData(mesh);

            if (enablePicking_.get()) {
                auto pickingRAM = std::make_shared<BufferRAMPrecision<uint32_t>>(numSpheres);
                auto& data = pickingRAM->getDataContainer();
                // fill in picking IDs
                std::iota(data.begin(), data.end(), static_cast<uint32_t>(spherePicking_.getPickingId(0)));

                mesh->addBuffer(Mesh::BufferInfo(BufferType::NumberOfBufferTypes, 4),
                    std::make_shared<Buffer<uint32_t>>(pickingRAM));
            }
        }

        auto& vertices = vertexRAM_->getDataContainer();
        auto& colors = colorRAM_->getDataContainer();
        auto& radii = radiiRAM_->getDataContainer();

        size_t portInd = 0;
        size_t sphereInd = 0;
        for (const auto &strucs : structure_) {
            long long j_start, j_stop;
            j_start = 0;
            j_stop = static_cast<long long>(strucs->size());
            if (animation_.get()) {
                j_start = num_[portInd]->get() * timestep_.get();
                j_stop = num_[portInd]->get() * (timestep_.get() + 1);
            }
            for (long long j = j_start; j < j_stop ; ++j) {
                auto center = scalingFactor_.get() * basis_.get() * strucs->at(j) - 0.5f * (basis_.get()[0] + basis_.get()[1] + basis_.get()[2]);
                vertices[sphereInd] = center;
                colors[sphereInd] = colors_[portInd]->get();
                radii[sphereInd] = radii_[portInd]->get();
                ++sphereInd;
            }
            ++portInd;
        }
        if (enablePicking_.get()) {
            //Set alpha-layer of picked atoms
            if (!inds_.get().empty()) {
                for (const auto &ind : inds_.get()) {
                    if (ind < sphereInd) {
                        colors[ind].w = 0.5;
                    }
                }
            }
        }
        colorRAM_->getOwner()->invalidateAllOther(colorRAM_.get());
        vertexRAM_->getOwner()->invalidateAllOther(vertexRAM_.get());
        radiiRAM_->getOwner()->invalidateAllOther(radiiRAM_.get());
        invalidate(InvalidationLevel::InvalidOutput);
    }

}

void StructureMesh::handlePicking(PickingEvent* p) {
    if (enablePicking_.get()) {
        if (p->getState() == PickingState::Updated &&
            p->getEvent()->hash() == MouseEvent::chash()) {
            auto me = p->getEventAs<MouseEvent>();
            if ((me->buttonState() & MouseButton::Left) && me->state() != MouseState::Move) {
                auto& color = colorRAM_->getDataContainer();
                std::vector<int> temp = inds_.get();
                auto picked = p->getPickedId();

                if( std::none_of(temp.begin(), temp.end(), [&](unsigned int i){return i == picked;}) ) {
                    temp.push_back(picked);
                    color[picked].w = 0.5;
                } else {
                    temp.erase(std::remove(temp.begin(), temp.end(), picked), temp.end());
                    color[picked].w = 1;
                }
                inds_.set(temp);

                colorRAM_->getOwner()->invalidateAllOther(colorRAM_.get());
                invalidate(InvalidationLevel::InvalidOutput);
                p->markAsUsed();
            }
        }
    }

}

} // namespace

<commit_msg>Removed translation of Structure Mesh basis.<commit_after>/*********************************************************************************
 *
 * Copyright (c) 2017 Josef Adamsson, Denise Härnström
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2017 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#include <modules/crystalvisualization/processors/structuremesh.h>
#include <inviwo/core/datastructures/geometry/basicmesh.h>

#include <inviwo/core/interaction/events/pickingevent.h>
#include <inviwo/core/interaction/events/mouseevent.h>
#include <inviwo/core/datastructures/buffer/bufferramprecision.h>
#include <inviwo/core/datastructures/buffer/buffer.h>

namespace inviwo {

// The Class Identifier has to be globally unique. Use a reverse DNS naming scheme
const ProcessorInfo StructureMesh::processorInfo_{
    "envision.StructureMesh",      // Class identifier
    "Structure Mesh",                // Display name
    "Crystal",              // Category
    CodeState::Experimental,  // Code state
    Tags::None,               // Tags
};
const ProcessorInfo StructureMesh::getProcessorInfo() const {
    return processorInfo_;
}

StructureMesh::StructureMesh()
    : Processor()
    , structure_("coordinates")
    , mesh_("mesh")
    , scalingFactor_("scalingFactor", "Scaling factor", 1.f)
    , basis_("basis", "Basis", glm::mat3x3())
    , fullMesh_("fullMesh", "Full mesh", false)
    , animation_("animation", "Animation", false)
    , timestep_("timestep", "Time step", false)
    , enablePicking_("enablePicking", "Enable Picking", false)
    , spherePicking_(this, 0, [&](PickingEvent* p) { handlePicking(p); })
    , inds_("inds", "Picked atoms") {
    addPort(structure_);
    addPort(mesh_);
    addProperty(scalingFactor_);
    addProperty(basis_);
    addProperty(fullMesh_);
    addProperty(animation_);
    addProperty(timestep_);
    addProperty(enablePicking_);
    addProperty(inds_);

    structure_.onChange([&](){
        const auto data = structure_.getVectorData();
        for(int i = colors_.size(); i < data.size(); ++i) {
            colors_.push_back(std::make_unique<FloatVec4Property>("color" + toString(i), "Color " + toString(i),
                                                vec4(1.0f, 1.0f, 1.0f, 1.0f), vec4(0.0f), vec4(1.0f), vec4(0.01f),
                                                InvalidationLevel::InvalidOutput, PropertySemantics::Color));
            addProperty(colors_.back().get(), false);

            radii_.push_back(std::make_unique<FloatProperty>("radius"+ toString(i), "Atom Radius"+ toString(i), 1.0f));
            addProperty(radii_.back().get(), false);
            num_.push_back(std::make_unique<IntProperty>("atoms" + toString(i), "Atoms" + toString(i), 0));
            addProperty(num_.back().get(), false);
        }
        int numSpheres = 0;
        for (const auto &strucs : structure_)
            numSpheres += strucs->size();
        spherePicking_.resize(numSpheres);

    });

}

void StructureMesh::process() {
    if (fullMesh_.get()) {
	auto mesh = std::make_shared<BasicMesh>();
	mesh_.setData(mesh);
	size_t ind = 0;
	for (const auto &strucs : structure_) {
	    for (long long j = 0; j < static_cast<long long>(strucs->size()); ++j) {
        auto center = scalingFactor_.get() * basis_.get() * strucs->at(j) - 0.5f * (basis_.get()[0] + basis_.get()[1] + basis_.get()[2]);
		BasicMesh::sphere(center, radii_[ind]->get(), colors_[ind]->get(), mesh);
	    }
	    ++ind;
	}
    } else {
        if (structure_.isChanged() || animation_.isModified() || std::any_of(num_.begin(), num_.end(),
            [](const std::unique_ptr<IntProperty>& property) {
            return property->isModified();
            }) || enablePicking_.isModified()) {
            int numSpheres = 0;
            size_t pInd = 0;
            for (const auto &strucs : structure_) {
                if (animation_.get()) {
                    numSpheres += num_[pInd]->get();
                    ++pInd;
                }
                else {
                    numSpheres += strucs->size();
                }
            }
            vertexRAM_ = std::make_shared<BufferRAMPrecision<vec3>>(numSpheres);
            colorRAM_ = std::make_shared<BufferRAMPrecision<vec4>>(numSpheres);
            radiiRAM_ = std::make_shared<BufferRAMPrecision<float>>(numSpheres);
            auto mesh = std::make_shared<Mesh>(DrawType::Points, ConnectivityType::None);
            mesh->addBuffer(Mesh::BufferInfo(BufferType::PositionAttrib),
                std::make_shared<Buffer<vec3>>(vertexRAM_));
            mesh->addBuffer(Mesh::BufferInfo(BufferType::ColorAttrib),
                std::make_shared<Buffer<vec4>>(colorRAM_));
            mesh->addBuffer(Mesh::BufferInfo(BufferType::NumberOfBufferTypes, 5),
                std::make_shared<Buffer<float>>(radiiRAM_));

            mesh_.setData(mesh);

            if (enablePicking_.get()) {
                auto pickingRAM = std::make_shared<BufferRAMPrecision<uint32_t>>(numSpheres);
                auto& data = pickingRAM->getDataContainer();
                // fill in picking IDs
                std::iota(data.begin(), data.end(), static_cast<uint32_t>(spherePicking_.getPickingId(0)));

                mesh->addBuffer(Mesh::BufferInfo(BufferType::NumberOfBufferTypes, 4),
                    std::make_shared<Buffer<uint32_t>>(pickingRAM));
            }
        }

        auto& vertices = vertexRAM_->getDataContainer();
        auto& colors = colorRAM_->getDataContainer();
        auto& radii = radiiRAM_->getDataContainer();

        LogInfo(scalingFactor_.get());
        size_t portInd = 0;
        size_t sphereInd = 0;
        for (const auto &strucs : structure_) {
            long long j_start, j_stop;
            j_start = 0;
            j_stop = static_cast<long long>(strucs->size());
            if (animation_.get()) {
                j_start = num_[portInd]->get() * timestep_.get();
                j_stop = num_[portInd]->get() * (timestep_.get() + 1);
            }
            for (long long j = j_start; j < j_stop ; ++j) {
                auto center = scalingFactor_.get() * basis_.get() * strucs->at(j); //- 0.5f * (basis_.get()[0] + basis_.get()[1] + basis_.get()[2]);
                vertices[sphereInd] = center;
                colors[sphereInd] = colors_[portInd]->get();
                radii[sphereInd] = radii_[portInd]->get();
                ++sphereInd;
            }
            ++portInd;
        }
        if (enablePicking_.get()) {
            //Set alpha-layer of picked atoms
            if (!inds_.get().empty()) {
                for (const auto &ind : inds_.get()) {
                    if (ind < sphereInd) {
                        colors[ind].w = 0.5;
                    }
                }
            }
        }
        colorRAM_->getOwner()->invalidateAllOther(colorRAM_.get());
        vertexRAM_->getOwner()->invalidateAllOther(vertexRAM_.get());
        radiiRAM_->getOwner()->invalidateAllOther(radiiRAM_.get());
        invalidate(InvalidationLevel::InvalidOutput);
    }

}

void StructureMesh::handlePicking(PickingEvent* p) {
    if (enablePicking_.get()) {
        if (p->getState() == PickingState::Updated &&
            p->getEvent()->hash() == MouseEvent::chash()) {
            auto me = p->getEventAs<MouseEvent>();
            if ((me->buttonState() & MouseButton::Left) && me->state() != MouseState::Move) {
                auto& color = colorRAM_->getDataContainer();
                std::vector<int> temp = inds_.get();
                auto picked = p->getPickedId();

                if( std::none_of(temp.begin(), temp.end(), [&](unsigned int i){return i == picked;}) ) {
                    temp.push_back(picked);
                    color[picked].w = 0.5;
                } else {
                    temp.erase(std::remove(temp.begin(), temp.end(), picked), temp.end());
                    color[picked].w = 1;
                }
                inds_.set(temp);

                colorRAM_->getOwner()->invalidateAllOther(colorRAM_.get());
                invalidate(InvalidationLevel::InvalidOutput);
                p->markAsUsed();
            }
        }
    }

}

} // namespace

<|endoftext|>