<commit_before>// Copyright 2018 Netherlands Institute for Radio Astronomy (ASTRON)
// Copyright 2018 Netherlands eScience Center
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <Memory.hpp>

void loadInput(AstroData::Observation & observation, const DeviceOptions & deviceOptions, const DataOptions & dataOptions, HostMemory & hostMemory, Timers & timers) {
  hostMemory.zappedChannels.resize(observation.getNrChannels(deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
  try {
    AstroData::readZappedChannels(observation, dataOptions.channelsFile, hostMemory.zappedChannels);
    AstroData::readIntegrationSteps(observation, dataOptions.integrationFile, hostMemory.integrationSteps);
  } catch ( AstroData::FileError & err ) {
    std::cerr << err.what() << std::endl;
    throw;
  }
  hostMemory.input.resize(observation.getNrBeams());
  if ( dataOptions.dataLOFAR ) {
#ifdef HAVE_HDF5
    hostMemory.input.at(0) = new std::vector<std::vector<inputDataType> *>(observation.getNrBatches());
    timers.inputLoad.start();
    if ( dataOptions.limit ) {
      AstroData::readLOFAR(dataOptions.headerFile, dataOptions.dataFile, observation, deviceOptions.padding.at(deviceOptions.deviceName), *(hostMemory.input.at(0)), observation.getNrBatches());
    } else {
      AstroData::readLOFAR(dataOptions.headerFile, dataOptions.dataFile, observation, deviceOptions.padding.at(deviceOptions.deviceName), *(hostMemory.input.at(0)));
    }
    timers.inputLoad.stop();
#endif // HAVE_HDF5
  } else if ( dataOptions.dataSIGPROC ) {
    hostMemory.input.at(0) = new std::vector<std::vector<inputDataType> *>(observation.getNrBatches());
    timers.inputLoad.start();
    AstroData::readSIGPROC(observation, deviceOptions.padding.at(deviceOptions.deviceName), inputBits, dataOptions.headerSizeSIGPROC, dataOptions.dataFile, *(hostMemory.input.at(0)));
    timers.inputLoad.stop();
  } else if ( dataOptions.dataPSRDADA ) {
#ifdef HAVE_PSRDADA
    hostMemory.ringBuffer = dada_hdu_create(0);
    dada_hdu_set_key(hostMemory.ringBuffer, dataOptions.dadaKey);
    if ( dada_hdu_connect(hostMemory.ringBuffer) != 0 ) {
      throw AstroData::RingBufferError("ERROR: impossible to connect to PSRDADA ringbuffer \"" + std::to_string(dataOptions.dadaKey) + "\".");
    }
    if ( dada_hdu_lock_read(hostMemory.ringBuffer) != 0 ) {
      throw AstroData::RingBufferError("ERROR: impossible to lock the PSRDADA ringbuffer for reading the header.");
    }
    timers.inputLoad.start();
    AstroData::readPSRDADAHeader(observation, *hostMemory.ringBuffer);
    timers.inputLoad.stop();
#endif // HAVE_PSRDADA
  }
}

void allocateHostMemory(AstroData::Observation & observation, const Options & options, const DeviceOptions & deviceOptions, const KernelConfigurations & kernelConfigurations, HostMemory & hostMemory) {
  if ( !options.subbandDedispersion ) {
    hostMemory.shiftsSingleStep = Dedispersion::getShifts(observation, deviceOptions.padding.at(deviceOptions.deviceName));
    if ( options.debug ) {
      std::cerr << "shiftsSingleStep" << std::endl;
      for ( unsigned int channel = 0; channel < observation.getNrChannels(); channel++ ) {
        std::cerr << hostMemory.shiftsSingleStep->at(channel) << " ";
      }
      std::cerr << std::endl << std::endl;
    }
    observation.setNrSamplesPerDispersedBatch(observation.getNrSamplesPerBatch() + static_cast<unsigned int>(hostMemory.shiftsSingleStep->at(0) * (observation.getFirstDM() + ((observation.getNrDMs() - 1) * observation.getDMStep()))));
    observation.setNrDelayBatches(static_cast<unsigned int>(std::ceil(static_cast<double>(observation.getNrSamplesPerDispersedBatch()) / observation.getNrSamplesPerBatch())));
#ifdef HAVE_PSRDADA
    if ( dataOptions.dataPSRDADA ) {
      hostMemory.inputDADA.resize(observation.getNrDelayBatches());
      for ( unsigned int batch = 0; batch < observation.getNrDelayBatches(); batch++ ) {
        if ( inputBits >= 8 ) {
          hostMemory.inputDADA.at(batch) = new std::vector<inputDataType>(observation.getNrBeams() * observation.getNrChannels() * observation.getNrSamplesPerBatch());
        } else {
          hostMemory.inputDADA.at(batch) = new std::vector<inputDataType>(observation.getNrBeams() * observation.getNrChannels() * (observation.getNrSamplesPerBatch() / (8 / inputBits)));
        }
      }
    }
#endif // HAVE_PSRDADA
    if ( kernelConfigurations.dedispersionSingleStepParameters.at(deviceOptions.deviceName)->at(observation.getNrDMs())->getSplitBatches() ) {
      // TODO: add support for splitBatches
    } else {
      if ( inputBits >= 8 ) {
        hostMemory.dispersedData.resize(observation.getNrBeams() * observation.getNrChannels() * observation.getNrSamplesPerDispersedBatch(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(inputDataType)));
      } else {
        hostMemory.dispersedData.resize(observation.getNrBeams() * observation.getNrChannels() * isa::utils::pad(observation.getNrSamplesPerDispersedBatch() / (8 / inputBits), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(inputDataType)));
      }
    }
    hostMemory.beamMapping.resize(observation.getNrSynthesizedBeams() * observation.getNrChannels(deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
    hostMemory.dedispersedData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
    hostMemory.integratedData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs() * isa::utils::pad(observation.getNrSamplesPerBatch() / *(integrationSteps.begin()), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
    hostMemory.snrData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(float)));
    hostMemory.snrSamples.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
  } else {
    hostMemory.shiftsStepOne = Dedispersion::getShifts(observation, deviceOptions.padding.at(deviceOptions.deviceName));
    hostMemory.shiftsStepTwo = Dedispersion::getShiftsStepTwo(observation, deviceOptions.padding.at(deviceOptions.deviceName));
    if ( options.debug ) {
      std::cerr << "shiftsStepOne" << std::endl;
      for ( unsigned int channel = 0; channel < observation.getNrChannels(); channel++ ) {
        std::cerr << hostMemory.shiftsStepOne->at(channel) << " ";
      }
      std::cerr << std::endl << "shiftsStepTwo" << std::endl;
      for ( unsigned int subband = 0; subband < observation.getNrSubbands(); subband++ ) {
        std::cerr << hostMemory.shiftsStepTwo->at(subband) << " ";
      }
      std::cerr << std::endl << std::endl;
    }
    observation.setNrSamplesPerBatch(observation.getNrSamplesPerBatch() + static_cast<unsigned int>(hostMemory.shiftsStepTwo->at(0) * (observation.getFirstDM() + ((observation.getNrDMs() - 1) * observation.getDMStep()))), true);
    observation.setNrSamplesPerDispersedBatch(observation.getNrSamplesPerBatch(true) + static_cast<unsigned int>(hostMemory.shiftsStepOne->at(0) * (observation.getFirstDM(true) + ((observation.getNrDMs(true) - 1) * observation.getDMStep(true)))), true);
    observation.setNrDelayBatches(static_cast<unsigned int>(std::ceil(static_cast<double>(observation.getNrSamplesPerDispersedBatch(true)) / observation.getNrSamplesPerBatch())), true);
#ifdef HAVE_PSRDADA
    if ( dataOptions.dataPSRDADA ) {
      hostMemory.inputDADA.resize(observation.getNrDelayBatches(true));
      for ( unsigned int batch = 0; batch < observation.getNrDelayBatches(true); batch++ ) {
        if ( inputBits >= 8 ) {
          hostMemory.inputDADA.at(batch) = new std::vector<inputDataType>(observation.getNrBeams() * observation.getNrChannels() * observation.getNrSamplesPerBatch());
        } else {
          hostMemory.inputDADA.at(batch) = new std::vector<inputDataType>(observation.getNrBeams() * observation.getNrChannels() * (observation.getNrSamplesPerBatch() / (8 / inputBits)));
        }
      }
    }
#endif // HAVE_PSRDADA
    if ( kernelConfigurations.dedispersionStepOneParameters.at(deviceOptions.deviceName)->at(observation.getNrDMs(true))->getSplitBatches() ) {
      // TODO: add support for splitBatches
    } else {
      if ( inputBits >= 8 ) {
        hostMemory.dispersedData.resize(observation.getNrBeams() * observation.getNrChannels() * observation.getNrSamplesPerDispersedBatch(true, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(inputDataType)));
      } else {
        hostMemory.dispersedData.resize(observation.getNrBeams() * observation.getNrChannels() * isa::utils::pad(observation.getNrSamplesPerDispersedBatch(true) / (8 / inputBits), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(inputDataType)));
      }
    }
    hostMemory.beamMapping.resize(observation.getNrSynthesizedBeams() * observation.getNrSubbands(deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
    hostMemory.subbandedData.resize(observation.getNrBeams() * observation.getNrDMs(true) * observation.getNrSubbands() * observation.getNrSamplesPerBatch(true, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
    hostMemory.dedispersedData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(true) * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
    hostMemory.integratedData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(true) * observation.getNrDMs() * isa::utils::pad(observation.getNrSamplesPerBatch() / *(integrationSteps.begin()), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
   hostMemory.snrData.resize(observation.getNrSynthesizedBeams() * isa::utils::pad(observation.getNrDMs(true) * observation.getNrDMs(), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(float)));
    hostMemory.snrSamples.resize(observation.getNrSynthesizedBeams() * isa::utils::pad(observation.getNrDMs(true) * observation.getNrDMs(), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
  }
  AstroData::generateBeamMapping(observation, hostMemory.beamMapping, deviceOptions.padding.at(deviceOptions.deviceName), options.subbandDedispersion);
}

void allocateDeviceMemory(const OpenCLRunTime & openclRunTime, const Options & options, const DeviceOptions & deviceOptions, const HostMemory & hostMemory, DeviceMemory & deviceMemory) {
  if ( !options.subbandDedispersion ) {
    deviceMemory.shiftsSingleStep = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.shiftsSingleStep->size() * sizeof(float), 0, 0);
    openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.shiftsSingleStep, CL_FALSE, 0, hostMemory.shiftsSingleStep->size() * sizeof(float), reinterpret_cast<void *>(hostMemory.shiftsSingleStep->data()));
  } else {
    deviceMemory.shiftsStepOne = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.shiftsStepOne->size() * sizeof(float), 0, 0);
    openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.shiftsStepOne, CL_FALSE, 0, hostMemory.shiftsStepOne->size() * sizeof(float), reinterpret_cast<void *>(hostMemory.shiftsStepOne->data()));
    deviceMemory.shiftsStepTwo = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.shiftsStepTwo->size() * sizeof(float), 0, 0);
    openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.shiftsStepTwo, CL_FALSE, 0, hostMemory.shiftsStepTwo->size() * sizeof(float), reinterpret_cast<void *>(hostMemory.shiftsStepTwo->data()));
    deviceMemory.subbandedData = cl::Buffer(*openclRunTime.context, CL_MEM_READ_WRITE, hostMemory.subbandedData.size() * sizeof(outputDataType), 0, 0);
  }
  deviceMemory.zappedChannels = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.zappedChannels.size() * sizeof(unsigned int), 0, 0);
  openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.zappedChannels, CL_FALSE, 0, hostMemory.zappedChannels.size() * sizeof(unsigned int), reinterpret_cast< void * >(hostMemory.zappedChannels.data()));
  deviceMemory.beamMapping = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.beamMapping.size() * sizeof(unsigned int), 0, 0);
  openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.beamMapping, CL_FALSE, 0, hostMemory.beamMapping.size() * sizeof(unsigned int), reinterpret_cast<void *>(hostMemory.beamMapping.data()));
  deviceMemory.dispersedData = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.dispersedData.size() * sizeof(inputDataType), 0, 0);
  deviceMemory.dedispersedData = cl::Buffer(*openclRunTime.context, CL_MEM_READ_WRITE, hostMemory.dedispersedData.size() * sizeof(outputDataType), 0, 0);
  deviceMemory.integratedData = cl::Buffer(*openclRunTime.context, CL_MEM_READ_WRITE, hostMemory.integratedData.size() * sizeof(outputDataType), 0, 0);
  deviceMemory.snrData = cl::Buffer(*openclRunTime.context, CL_MEM_WRITE_ONLY, hostMemory.snrData.size() * sizeof(float), 0, 0);
  deviceMemory.snrSamples = cl::Buffer(*openclRunTime.context, CL_MEM_WRITE_ONLY, hostMemory.snrSamples.size() * sizeof(unsigned int), 0, 0);
  openclRunTime.queues->at(deviceOptions.deviceID).at(0).finish();
}
<commit_msg>Use the right integrationSteps.<commit_after>// Copyright 2018 Netherlands Institute for Radio Astronomy (ASTRON)
// Copyright 2018 Netherlands eScience Center
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <Memory.hpp>

void loadInput(AstroData::Observation & observation, const DeviceOptions & deviceOptions, const DataOptions & dataOptions, HostMemory & hostMemory, Timers & timers) {
  hostMemory.zappedChannels.resize(observation.getNrChannels(deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
  try {
    AstroData::readZappedChannels(observation, dataOptions.channelsFile, hostMemory.zappedChannels);
    AstroData::readIntegrationSteps(observation, dataOptions.integrationFile, hostMemory.integrationSteps);
  } catch ( AstroData::FileError & err ) {
    std::cerr << err.what() << std::endl;
    throw;
  }
  hostMemory.input.resize(observation.getNrBeams());
  if ( dataOptions.dataLOFAR ) {
#ifdef HAVE_HDF5
    hostMemory.input.at(0) = new std::vector<std::vector<inputDataType> *>(observation.getNrBatches());
    timers.inputLoad.start();
    if ( dataOptions.limit ) {
      AstroData::readLOFAR(dataOptions.headerFile, dataOptions.dataFile, observation, deviceOptions.padding.at(deviceOptions.deviceName), *(hostMemory.input.at(0)), observation.getNrBatches());
    } else {
      AstroData::readLOFAR(dataOptions.headerFile, dataOptions.dataFile, observation, deviceOptions.padding.at(deviceOptions.deviceName), *(hostMemory.input.at(0)));
    }
    timers.inputLoad.stop();
#endif // HAVE_HDF5
  } else if ( dataOptions.dataSIGPROC ) {
    hostMemory.input.at(0) = new std::vector<std::vector<inputDataType> *>(observation.getNrBatches());
    timers.inputLoad.start();
    AstroData::readSIGPROC(observation, deviceOptions.padding.at(deviceOptions.deviceName), inputBits, dataOptions.headerSizeSIGPROC, dataOptions.dataFile, *(hostMemory.input.at(0)));
    timers.inputLoad.stop();
  } else if ( dataOptions.dataPSRDADA ) {
#ifdef HAVE_PSRDADA
    hostMemory.ringBuffer = dada_hdu_create(0);
    dada_hdu_set_key(hostMemory.ringBuffer, dataOptions.dadaKey);
    if ( dada_hdu_connect(hostMemory.ringBuffer) != 0 ) {
      throw AstroData::RingBufferError("ERROR: impossible to connect to PSRDADA ringbuffer \"" + std::to_string(dataOptions.dadaKey) + "\".");
    }
    if ( dada_hdu_lock_read(hostMemory.ringBuffer) != 0 ) {
      throw AstroData::RingBufferError("ERROR: impossible to lock the PSRDADA ringbuffer for reading the header.");
    }
    timers.inputLoad.start();
    AstroData::readPSRDADAHeader(observation, *hostMemory.ringBuffer);
    timers.inputLoad.stop();
#endif // HAVE_PSRDADA
  }
}

void allocateHostMemory(AstroData::Observation & observation, const Options & options, const DeviceOptions & deviceOptions, const KernelConfigurations & kernelConfigurations, HostMemory & hostMemory) {
  if ( !options.subbandDedispersion ) {
    hostMemory.shiftsSingleStep = Dedispersion::getShifts(observation, deviceOptions.padding.at(deviceOptions.deviceName));
    if ( options.debug ) {
      std::cerr << "shiftsSingleStep" << std::endl;
      for ( unsigned int channel = 0; channel < observation.getNrChannels(); channel++ ) {
        std::cerr << hostMemory.shiftsSingleStep->at(channel) << " ";
      }
      std::cerr << std::endl << std::endl;
    }
    observation.setNrSamplesPerDispersedBatch(observation.getNrSamplesPerBatch() + static_cast<unsigned int>(hostMemory.shiftsSingleStep->at(0) * (observation.getFirstDM() + ((observation.getNrDMs() - 1) * observation.getDMStep()))));
    observation.setNrDelayBatches(static_cast<unsigned int>(std::ceil(static_cast<double>(observation.getNrSamplesPerDispersedBatch()) / observation.getNrSamplesPerBatch())));
#ifdef HAVE_PSRDADA
    if ( dataOptions.dataPSRDADA ) {
      hostMemory.inputDADA.resize(observation.getNrDelayBatches());
      for ( unsigned int batch = 0; batch < observation.getNrDelayBatches(); batch++ ) {
        if ( inputBits >= 8 ) {
          hostMemory.inputDADA.at(batch) = new std::vector<inputDataType>(observation.getNrBeams() * observation.getNrChannels() * observation.getNrSamplesPerBatch());
        } else {
          hostMemory.inputDADA.at(batch) = new std::vector<inputDataType>(observation.getNrBeams() * observation.getNrChannels() * (observation.getNrSamplesPerBatch() / (8 / inputBits)));
        }
      }
    }
#endif // HAVE_PSRDADA
    if ( kernelConfigurations.dedispersionSingleStepParameters.at(deviceOptions.deviceName)->at(observation.getNrDMs())->getSplitBatches() ) {
      // TODO: add support for splitBatches
    } else {
      if ( inputBits >= 8 ) {
        hostMemory.dispersedData.resize(observation.getNrBeams() * observation.getNrChannels() * observation.getNrSamplesPerDispersedBatch(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(inputDataType)));
      } else {
        hostMemory.dispersedData.resize(observation.getNrBeams() * observation.getNrChannels() * isa::utils::pad(observation.getNrSamplesPerDispersedBatch() / (8 / inputBits), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(inputDataType)));
      }
    }
    hostMemory.beamMapping.resize(observation.getNrSynthesizedBeams() * observation.getNrChannels(deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
    hostMemory.dedispersedData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
    hostMemory.integratedData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs() * isa::utils::pad(observation.getNrSamplesPerBatch() / *(hostMemory.integrationSteps.begin()), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
    hostMemory.snrData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(float)));
    hostMemory.snrSamples.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
  } else {
    hostMemory.shiftsStepOne = Dedispersion::getShifts(observation, deviceOptions.padding.at(deviceOptions.deviceName));
    hostMemory.shiftsStepTwo = Dedispersion::getShiftsStepTwo(observation, deviceOptions.padding.at(deviceOptions.deviceName));
    if ( options.debug ) {
      std::cerr << "shiftsStepOne" << std::endl;
      for ( unsigned int channel = 0; channel < observation.getNrChannels(); channel++ ) {
        std::cerr << hostMemory.shiftsStepOne->at(channel) << " ";
      }
      std::cerr << std::endl << "shiftsStepTwo" << std::endl;
      for ( unsigned int subband = 0; subband < observation.getNrSubbands(); subband++ ) {
        std::cerr << hostMemory.shiftsStepTwo->at(subband) << " ";
      }
      std::cerr << std::endl << std::endl;
    }
    observation.setNrSamplesPerBatch(observation.getNrSamplesPerBatch() + static_cast<unsigned int>(hostMemory.shiftsStepTwo->at(0) * (observation.getFirstDM() + ((observation.getNrDMs() - 1) * observation.getDMStep()))), true);
    observation.setNrSamplesPerDispersedBatch(observation.getNrSamplesPerBatch(true) + static_cast<unsigned int>(hostMemory.shiftsStepOne->at(0) * (observation.getFirstDM(true) + ((observation.getNrDMs(true) - 1) * observation.getDMStep(true)))), true);
    observation.setNrDelayBatches(static_cast<unsigned int>(std::ceil(static_cast<double>(observation.getNrSamplesPerDispersedBatch(true)) / observation.getNrSamplesPerBatch())), true);
#ifdef HAVE_PSRDADA
    if ( dataOptions.dataPSRDADA ) {
      hostMemory.inputDADA.resize(observation.getNrDelayBatches(true));
      for ( unsigned int batch = 0; batch < observation.getNrDelayBatches(true); batch++ ) {
        if ( inputBits >= 8 ) {
          hostMemory.inputDADA.at(batch) = new std::vector<inputDataType>(observation.getNrBeams() * observation.getNrChannels() * observation.getNrSamplesPerBatch());
        } else {
          hostMemory.inputDADA.at(batch) = new std::vector<inputDataType>(observation.getNrBeams() * observation.getNrChannels() * (observation.getNrSamplesPerBatch() / (8 / inputBits)));
        }
      }
    }
#endif // HAVE_PSRDADA
    if ( kernelConfigurations.dedispersionStepOneParameters.at(deviceOptions.deviceName)->at(observation.getNrDMs(true))->getSplitBatches() ) {
      // TODO: add support for splitBatches
    } else {
      if ( inputBits >= 8 ) {
        hostMemory.dispersedData.resize(observation.getNrBeams() * observation.getNrChannels() * observation.getNrSamplesPerDispersedBatch(true, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(inputDataType)));
      } else {
        hostMemory.dispersedData.resize(observation.getNrBeams() * observation.getNrChannels() * isa::utils::pad(observation.getNrSamplesPerDispersedBatch(true) / (8 / inputBits), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(inputDataType)));
      }
    }
    hostMemory.beamMapping.resize(observation.getNrSynthesizedBeams() * observation.getNrSubbands(deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
    hostMemory.subbandedData.resize(observation.getNrBeams() * observation.getNrDMs(true) * observation.getNrSubbands() * observation.getNrSamplesPerBatch(true, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
    hostMemory.dedispersedData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(true) * observation.getNrDMs() * observation.getNrSamplesPerBatch(false, deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
    hostMemory.integratedData.resize(observation.getNrSynthesizedBeams() * observation.getNrDMs(true) * observation.getNrDMs() * isa::utils::pad(observation.getNrSamplesPerBatch() / *(hostMemory.integrationSteps.begin()), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(outputDataType)));
   hostMemory.snrData.resize(observation.getNrSynthesizedBeams() * isa::utils::pad(observation.getNrDMs(true) * observation.getNrDMs(), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(float)));
    hostMemory.snrSamples.resize(observation.getNrSynthesizedBeams() * isa::utils::pad(observation.getNrDMs(true) * observation.getNrDMs(), deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
  }
  AstroData::generateBeamMapping(observation, hostMemory.beamMapping, deviceOptions.padding.at(deviceOptions.deviceName), options.subbandDedispersion);
}

void allocateDeviceMemory(const OpenCLRunTime & openclRunTime, const Options & options, const DeviceOptions & deviceOptions, const HostMemory & hostMemory, DeviceMemory & deviceMemory) {
  if ( !options.subbandDedispersion ) {
    deviceMemory.shiftsSingleStep = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.shiftsSingleStep->size() * sizeof(float), 0, 0);
    openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.shiftsSingleStep, CL_FALSE, 0, hostMemory.shiftsSingleStep->size() * sizeof(float), reinterpret_cast<void *>(hostMemory.shiftsSingleStep->data()));
  } else {
    deviceMemory.shiftsStepOne = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.shiftsStepOne->size() * sizeof(float), 0, 0);
    openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.shiftsStepOne, CL_FALSE, 0, hostMemory.shiftsStepOne->size() * sizeof(float), reinterpret_cast<void *>(hostMemory.shiftsStepOne->data()));
    deviceMemory.shiftsStepTwo = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.shiftsStepTwo->size() * sizeof(float), 0, 0);
    openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.shiftsStepTwo, CL_FALSE, 0, hostMemory.shiftsStepTwo->size() * sizeof(float), reinterpret_cast<void *>(hostMemory.shiftsStepTwo->data()));
    deviceMemory.subbandedData = cl::Buffer(*openclRunTime.context, CL_MEM_READ_WRITE, hostMemory.subbandedData.size() * sizeof(outputDataType), 0, 0);
  }
  deviceMemory.zappedChannels = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.zappedChannels.size() * sizeof(unsigned int), 0, 0);
  openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.zappedChannels, CL_FALSE, 0, hostMemory.zappedChannels.size() * sizeof(unsigned int), reinterpret_cast< void * >(hostMemory.zappedChannels.data()));
  deviceMemory.beamMapping = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.beamMapping.size() * sizeof(unsigned int), 0, 0);
  openclRunTime.queues->at(deviceOptions.deviceID).at(0).enqueueWriteBuffer(deviceMemory.beamMapping, CL_FALSE, 0, hostMemory.beamMapping.size() * sizeof(unsigned int), reinterpret_cast<void *>(hostMemory.beamMapping.data()));
  deviceMemory.dispersedData = cl::Buffer(*openclRunTime.context, CL_MEM_READ_ONLY, hostMemory.dispersedData.size() * sizeof(inputDataType), 0, 0);
  deviceMemory.dedispersedData = cl::Buffer(*openclRunTime.context, CL_MEM_READ_WRITE, hostMemory.dedispersedData.size() * sizeof(outputDataType), 0, 0);
  deviceMemory.integratedData = cl::Buffer(*openclRunTime.context, CL_MEM_READ_WRITE, hostMemory.integratedData.size() * sizeof(outputDataType), 0, 0);
  deviceMemory.snrData = cl::Buffer(*openclRunTime.context, CL_MEM_WRITE_ONLY, hostMemory.snrData.size() * sizeof(float), 0, 0);
  deviceMemory.snrSamples = cl::Buffer(*openclRunTime.context, CL_MEM_WRITE_ONLY, hostMemory.snrSamples.size() * sizeof(unsigned int), 0, 0);
  openclRunTime.queues->at(deviceOptions.deviceID).at(0).finish();
}
<|endoftext|>


<commit_before>#include "nanoweb/nanoweb.hpp"
#include "server/find_cmake.hpp"
#include "server/find_executable.hpp"
#include "server/find_git.hpp"
#include "server/cmake.hpp"
#include <silicium/asio/tcp_acceptor.hpp>
#include <silicium/asio/writing_observable.hpp>
#include <silicium/asio/posting_observable.hpp>
#include <silicium/observable/spawn_observable.hpp>
#include <silicium/observable/erased_observer.hpp>
#include <silicium/observable/total_consumer.hpp>
#include <silicium/observable/while.hpp>
#include <silicium/observable/thread.hpp>
#include <silicium/sink/ostream_sink.hpp>
#include <silicium/open.hpp>
#include <silicium/fast_variant.hpp>
#include <silicium/std_threading.hpp>
#include <silicium/run_process.hpp>
#include <silicium/html.hpp>
#include <silicium/async_process.hpp>
#include <silicium/absolute_path.hpp>
#include <silicium/path_segment.hpp>
#include <boost/program_options.hpp>
#include <boost/optional.hpp>
#include <boost/range/algorithm/equal.hpp>
#include <boost/thread.hpp>
#include <unordered_map>
#include <functional>
#include <iostream>

namespace
{
	struct notification
	{
	};

	template <class Observer>
	struct saturating_notifier
	{
		typedef notification element_type;

		template <class ActualObserver>
		void async_get_one(ActualObserver &&observer)
		{
			return Si::visit<void>(
				m_observer_or_notification,
				[this, &observer](Observer &my_observer) mutable
				{
					assert(!my_observer.get());
					my_observer = Observer(observer);
				},
				[this, &observer](notification)
				{
					m_observer_or_notification = Observer();
					std::move(observer).got_element(notification());
				}
			);
		}

		void notify()
		{
			Si::visit<void>(
				m_observer_or_notification,
				[this](Observer &observer)
				{
					if (observer.get())
					{
						Si::exchange(observer, Observer()).got_element(notification());
					}
					else
					{
						m_observer_or_notification = notification();
					}
				},
				[](notification const &)
				{
				}
			);
		}

	private:

		Si::fast_variant<Observer, notification> m_observer_or_notification;
	};

	template <class YieldContext, class NotifierObserver>
	nanoweb::request_handler_result notify(
		boost::asio::ip::tcp::socket &client,
		YieldContext &&yield,
		Si::noexcept_string const &path,
		Si::noexcept_string const &secret,
		saturating_notifier<NotifierObserver> &notifier)
	{
		if (std::string::npos == path.find(secret))
		{
			nanoweb::quick_final_response(client, yield, "403", "Forbidden", Si::make_c_str_range("the path does not contain the correct secret"));
			return nanoweb::request_handler_result::handled;
		}

		notifier.notify();

		nanoweb::quick_final_response(client, yield, "200", "OK", Si::make_c_str_range("the server has been successfully notified"));
		return nanoweb::request_handler_result::handled;
	}

	enum class build_result
	{
		success,
		failure
	};

	struct overview_state
	{
		bool is_building = false;
		boost::optional<build_result> last_result;
	};

	template <class CharSink>
	void render_overview_page(CharSink &&rendered, overview_state const &overview)
	{
		auto doc = Si::html::make_generator(std::forward<CharSink>(rendered));
		doc("html", [&]()
		{
			doc("head", [&]()
			{
				doc("title", [&]()
				{
					doc.write("buildserver overview");
				});
			});
			doc("body", [&]()
			{
				doc("h1", [&]()
				{
					doc.write("Overview");
				});
				doc("p", [&]()
				{
					doc.write(overview.is_building ? "building.." : "idle");
				});
				doc("hr");
				if (overview.last_result)
				{
					doc("p", [&]()
					{
						doc.write("last build ");
						switch (*overview.last_result)
						{
						case build_result::success:
							doc.write("succeeded");
							break;
						case build_result::failure:
							doc.write("failed");
							break;
						}
					});
				}
			});
		});
	}

	template <class NotifierObserver>
	nanoweb::request_handler make_root_request_handler(Si::noexcept_string const &secret, saturating_notifier<NotifierObserver> &notifier, overview_state const &overview)
	{
		auto handle_request = nanoweb::make_directory({
			{
				Si::make_c_str_range(""),
				nanoweb::request_handler([&overview](boost::asio::ip::tcp::socket &client, Si::http::request const &, Si::iterator_range<Si::memory_range const *>, Si::spawn_context yield)
				{
					std::vector<char> content;
					render_overview_page(Si::make_container_sink(content), overview);
					nanoweb::quick_final_response(client, yield, "200", "OK", Si::make_memory_range(content));
					return nanoweb::request_handler_result::handled;
				})
			},
			{
				Si::make_c_str_range("notify"),
				nanoweb::request_handler([&secret, &notifier](boost::asio::ip::tcp::socket &client, Si::http::request const &request, Si::iterator_range<Si::memory_range const *>, Si::spawn_context yield)
				{
					return notify(client, yield, request.path, secret, notifier);
				})
			}
		});
		return handle_request;
	}

	struct options
	{
		Si::os_string repository;
		boost::uint16_t port;
		Si::noexcept_string secret;
		Si::absolute_path workspace;
	};

	boost::optional<options> parse_options(int argc, char **argv)
	{
		options result;
		result.port = 8080;

		boost::program_options::options_description desc("Allowed options");
		desc.add_options()
			("help", "produce help message")
			("repository,r", boost::program_options::
#ifdef _WIN32
				wvalue
#else
				value
#endif
				(&result.repository), "the URI for git cloning the code")
			("port,p", boost::program_options::value(&result.port), "port to listen on for POSTed push notifications")
			("secret,s", boost::program_options::value(&result.secret), "a string that needs to be in the query for the notification to be accepted")
			("workspace,w", boost::program_options::value(&result.workspace), "")
		;

		boost::program_options::positional_options_description positional;
		positional.add("repository", 1);
		positional.add("port", 1);
		positional.add("secret", 1);
		positional.add("workspace", 1);
		boost::program_options::variables_map vm;
		try
		{
			boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(desc).positional(positional).run(), vm);
		}
		catch (boost::program_options::error const &ex)
		{
			std::cerr
				<< ex.what() << '\n'
				<< desc << "\n";
			return boost::none;
		}

		boost::program_options::notify(vm);

		if (vm.count("help"))
		{
		    std::cerr << desc << "\n";
			return boost::none;
		}

		if (result.repository.empty())
		{
			std::cerr << "Missing option value --repository\n";
			std::cerr << desc << "\n";
			return boost::none;
		}

		if (result.workspace.empty())
		{
			std::cerr << "Missing option value --workspace\n";
			std::cerr << desc << "\n";
			return boost::none;
		}

		return std::move(result);
	}

	int run_process(Si::async_process_parameters const &parameters, Si::sink<char, Si::success> &output)
	{
		Si::pipe standard_output_and_error = SILICIUM_MOVE_IF_COMPILER_LACKS_RVALUE_QUALIFIERS(Si::make_pipe().get());
		Si::file_handle standard_input = SILICIUM_MOVE_IF_COMPILER_LACKS_RVALUE_QUALIFIERS(Si::open_reading("/dev/null").get());
		Si::async_process process = SILICIUM_MOVE_IF_COMPILER_LACKS_RVALUE_QUALIFIERS(Si::launch_process(
			parameters,
			standard_input.handle,
			standard_output_and_error.write.handle,
			standard_output_and_error.write.handle
		).get());
		boost::asio::io_service io;
		Si::experimental::read_from_anonymous_pipe(io, Si::ref_sink(output), std::move(standard_output_and_error.read));
		standard_output_and_error.write.close();
		io.run();
		int exit_code = process.wait_for_exit().get();
		return exit_code;
	}

	void git_clone(Si::os_string const &repository, Si::absolute_path const &destination, Si::path_segment const &clone_name, Si::absolute_path const &git_exe, Si::sink<char, Si::success> &output)
	{
		Si::async_process_parameters parameters;
		parameters.executable = git_exe;
		parameters.current_path = destination;
		parameters.arguments.emplace_back(Si::to_os_string("clone"));
		parameters.arguments.emplace_back(repository);
		parameters.arguments.emplace_back((destination / clone_name).c_str());
		int exit_code = run_process(parameters, output);
		if (exit_code != 0)
		{
			throw std::runtime_error("git-clone failed");
		}
	}

	build_result run_test(Si::absolute_path const &build_dir, Si::sink<char, Si::success> &output)
	{
		Si::absolute_path const test_dir = build_dir / "test";
		Si::absolute_path const test_exe = test_dir / "unit_test";
		Si::async_process_parameters parameters;
		parameters.executable = test_exe;
		parameters.current_path = test_dir;
		int exit_code = run_process(parameters, output);
		if (exit_code == 0)
		{
			return build_result::success;
		}
		else
		{
			return build_result::failure;
		}
	}

	build_result build(
		Si::os_string const &repository,
		Si::absolute_path const &workspace,
		Si::absolute_path const &git,
		Si::absolute_path const &cmake,
		Si::sink<char, Si::success> &output)
	{
		Si::path_segment const clone_name = *Si::path_segment::create("source.git");
		git_clone(repository, workspace, clone_name, git, output);
		Si::absolute_path const source = workspace / clone_name;

		Si::absolute_path const build = workspace / "build";
		boost::filesystem::create_directories(build.to_boost_path());

		buildserver::cmake_exe cmake_builder(cmake);
		boost::system::error_code error = cmake_builder.generate(source, build, boost::unordered_map<std::string, std::string>{}, output);
		if (error)
		{
			boost::throw_exception(boost::system::system_error(error));
		}

		error = cmake_builder.build(build, boost::thread::hardware_concurrency(), output);
		if (error)
		{
			boost::throw_exception(boost::system::system_error(error));
		}

		return run_test(build, output);
	}
}

int main(int argc, char **argv)
{
	auto parsed_options = parse_options(argc, argv);
	if (!parsed_options)
	{
		return 1;
	}

	Si::optional<Si::absolute_path> maybe_git = buildserver::find_git().get();
	if (!maybe_git)
	{
		std::cerr << "Could not find Git\n";
		return 1;
	}

	Si::optional<Si::absolute_path> maybe_cmake = buildserver::find_cmake().get();
	if (!maybe_cmake)
	{
		std::cerr << "Could not find CMake\n";
		return 1;
	}

	boost::asio::io_service io;

	Si::spawn_coroutine([&](Si::spawn_context yield)
	{
		saturating_notifier<Si::erased_observer<notification>> notifier;
		overview_state overview;

		nanoweb::request_handler root_request_handler = make_root_request_handler(parsed_options->secret, notifier, overview);
		Si::spawn_observable(Si::transform(
			Si::asio::make_tcp_acceptor(boost::asio::ip::tcp::acceptor(io, boost::asio::ip::tcp::endpoint(boost::asio::ip::address_v4::any(), parsed_options->port))),
			[&root_request_handler](Si::asio::tcp_acceptor_result maybe_client) -> Si::nothing
			{
				auto client = maybe_client.get();
				Si::spawn_coroutine([client, &root_request_handler](Si::spawn_context yield)
				{
					auto error = nanoweb::serve_client(*client, yield, root_request_handler);
					if (!!error)
					{
						std::cerr << client->remote_endpoint().address() << ": " << error << '\n';
					}
				});
				return{};
			}
		));

		for (;;)
		{
			Si::optional<notification> notification_ = yield.get_one(Si::ref(notifier));
			assert(notification_);
			std::cerr << "Received a notification\n";
			try
			{
				overview.is_building = true;
				Si::optional<std::future<build_result>> maybe_result = yield.get_one(
					Si::asio::make_posting_observable(
						io,
						Si::make_thread_observable<Si::std_threading>([&]()
						{
							boost::filesystem::remove_all(parsed_options->workspace.to_boost_path());
							boost::filesystem::create_directories(parsed_options->workspace.to_boost_path());
							auto output = Si::virtualize_sink(Si::ostream_ref_sink(std::cerr));
							return build(parsed_options->repository, parsed_options->workspace, *maybe_git, *maybe_cmake, output);
						})
					)
				);
				assert(maybe_result);
				auto const result = maybe_result->get();
				switch (result)
				{
				case build_result::success:
					std::cerr << "Build success\n";
					break;

				case build_result::failure:
					std::cerr << "Build failure\n";
					break;
				}
				overview.last_result = result;
			}
			catch (std::exception const &ex)
			{
				std::cerr << "Exception: " <<  ex.what() << '\n';
				overview.last_result = build_result::failure;
			}
			overview.is_building = false;
		}
	});

	io.run();
}
<commit_msg>replace boost::optional with Si::optional<commit_after>#include "nanoweb/nanoweb.hpp"
#include "server/find_cmake.hpp"
#include "server/find_executable.hpp"
#include "server/find_git.hpp"
#include "server/cmake.hpp"
#include <silicium/asio/tcp_acceptor.hpp>
#include <silicium/asio/writing_observable.hpp>
#include <silicium/asio/posting_observable.hpp>
#include <silicium/observable/spawn_observable.hpp>
#include <silicium/observable/erased_observer.hpp>
#include <silicium/observable/total_consumer.hpp>
#include <silicium/observable/while.hpp>
#include <silicium/observable/thread.hpp>
#include <silicium/sink/ostream_sink.hpp>
#include <silicium/open.hpp>
#include <silicium/fast_variant.hpp>
#include <silicium/std_threading.hpp>
#include <silicium/run_process.hpp>
#include <silicium/html.hpp>
#include <silicium/async_process.hpp>
#include <silicium/absolute_path.hpp>
#include <silicium/path_segment.hpp>
#include <boost/program_options.hpp>
#include <boost/optional.hpp>
#include <boost/range/algorithm/equal.hpp>
#include <boost/thread.hpp>
#include <unordered_map>
#include <functional>
#include <iostream>

namespace
{
	struct notification
	{
	};

	template <class Observer>
	struct saturating_notifier
	{
		typedef notification element_type;

		template <class ActualObserver>
		void async_get_one(ActualObserver &&observer)
		{
			return Si::visit<void>(
				m_observer_or_notification,
				[this, &observer](Observer &my_observer) mutable
				{
					assert(!my_observer.get());
					my_observer = Observer(observer);
				},
				[this, &observer](notification)
				{
					m_observer_or_notification = Observer();
					std::move(observer).got_element(notification());
				}
			);
		}

		void notify()
		{
			Si::visit<void>(
				m_observer_or_notification,
				[this](Observer &observer)
				{
					if (observer.get())
					{
						Si::exchange(observer, Observer()).got_element(notification());
					}
					else
					{
						m_observer_or_notification = notification();
					}
				},
				[](notification const &)
				{
				}
			);
		}

	private:

		Si::fast_variant<Observer, notification> m_observer_or_notification;
	};

	template <class YieldContext, class NotifierObserver>
	nanoweb::request_handler_result notify(
		boost::asio::ip::tcp::socket &client,
		YieldContext &&yield,
		Si::noexcept_string const &path,
		Si::noexcept_string const &secret,
		saturating_notifier<NotifierObserver> &notifier)
	{
		if (std::string::npos == path.find(secret))
		{
			nanoweb::quick_final_response(client, yield, "403", "Forbidden", Si::make_c_str_range("the path does not contain the correct secret"));
			return nanoweb::request_handler_result::handled;
		}

		notifier.notify();

		nanoweb::quick_final_response(client, yield, "200", "OK", Si::make_c_str_range("the server has been successfully notified"));
		return nanoweb::request_handler_result::handled;
	}

	enum class build_result
	{
		success,
		failure
	};

	struct overview_state
	{
		bool is_building = false;
		Si::optional<build_result> last_result;
	};

	template <class CharSink>
	void render_overview_page(CharSink &&rendered, overview_state const &overview)
	{
		auto doc = Si::html::make_generator(std::forward<CharSink>(rendered));
		doc("html", [&]()
		{
			doc("head", [&]()
			{
				doc("title", [&]()
				{
					doc.write("buildserver overview");
				});
			});
			doc("body", [&]()
			{
				doc("h1", [&]()
				{
					doc.write("Overview");
				});
				doc("p", [&]()
				{
					doc.write(overview.is_building ? "building.." : "idle");
				});
				doc("hr");
				if (overview.last_result)
				{
					doc("p", [&]()
					{
						doc.write("last build ");
						switch (*overview.last_result)
						{
						case build_result::success:
							doc.write("succeeded");
							break;
						case build_result::failure:
							doc.write("failed");
							break;
						}
					});
				}
			});
		});
	}

	template <class NotifierObserver>
	nanoweb::request_handler make_root_request_handler(Si::noexcept_string const &secret, saturating_notifier<NotifierObserver> &notifier, overview_state const &overview)
	{
		auto handle_request = nanoweb::make_directory({
			{
				Si::make_c_str_range(""),
				nanoweb::request_handler([&overview](boost::asio::ip::tcp::socket &client, Si::http::request const &, Si::iterator_range<Si::memory_range const *>, Si::spawn_context yield)
				{
					std::vector<char> content;
					render_overview_page(Si::make_container_sink(content), overview);
					nanoweb::quick_final_response(client, yield, "200", "OK", Si::make_memory_range(content));
					return nanoweb::request_handler_result::handled;
				})
			},
			{
				Si::make_c_str_range("notify"),
				nanoweb::request_handler([&secret, &notifier](boost::asio::ip::tcp::socket &client, Si::http::request const &request, Si::iterator_range<Si::memory_range const *>, Si::spawn_context yield)
				{
					return notify(client, yield, request.path, secret, notifier);
				})
			}
		});
		return handle_request;
	}

	struct options
	{
		Si::os_string repository;
		boost::uint16_t port;
		Si::noexcept_string secret;
		Si::absolute_path workspace;
	};

	boost::optional<options> parse_options(int argc, char **argv)
	{
		options result;
		result.port = 8080;

		boost::program_options::options_description desc("Allowed options");
		desc.add_options()
			("help", "produce help message")
			("repository,r", boost::program_options::
#ifdef _WIN32
				wvalue
#else
				value
#endif
				(&result.repository), "the URI for git cloning the code")
			("port,p", boost::program_options::value(&result.port), "port to listen on for POSTed push notifications")
			("secret,s", boost::program_options::value(&result.secret), "a string that needs to be in the query for the notification to be accepted")
			("workspace,w", boost::program_options::value(&result.workspace), "")
		;

		boost::program_options::positional_options_description positional;
		positional.add("repository", 1);
		positional.add("port", 1);
		positional.add("secret", 1);
		positional.add("workspace", 1);
		boost::program_options::variables_map vm;
		try
		{
			boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(desc).positional(positional).run(), vm);
		}
		catch (boost::program_options::error const &ex)
		{
			std::cerr
				<< ex.what() << '\n'
				<< desc << "\n";
			return boost::none;
		}

		boost::program_options::notify(vm);

		if (vm.count("help"))
		{
		    std::cerr << desc << "\n";
			return boost::none;
		}

		if (result.repository.empty())
		{
			std::cerr << "Missing option value --repository\n";
			std::cerr << desc << "\n";
			return boost::none;
		}

		if (result.workspace.empty())
		{
			std::cerr << "Missing option value --workspace\n";
			std::cerr << desc << "\n";
			return boost::none;
		}

		return std::move(result);
	}

	int run_process(Si::async_process_parameters const &parameters, Si::sink<char, Si::success> &output)
	{
		Si::pipe standard_output_and_error = SILICIUM_MOVE_IF_COMPILER_LACKS_RVALUE_QUALIFIERS(Si::make_pipe().get());
		Si::file_handle standard_input = SILICIUM_MOVE_IF_COMPILER_LACKS_RVALUE_QUALIFIERS(Si::open_reading("/dev/null").get());
		Si::async_process process = SILICIUM_MOVE_IF_COMPILER_LACKS_RVALUE_QUALIFIERS(Si::launch_process(
			parameters,
			standard_input.handle,
			standard_output_and_error.write.handle,
			standard_output_and_error.write.handle
		).get());
		boost::asio::io_service io;
		Si::experimental::read_from_anonymous_pipe(io, Si::ref_sink(output), std::move(standard_output_and_error.read));
		standard_output_and_error.write.close();
		io.run();
		int exit_code = process.wait_for_exit().get();
		return exit_code;
	}

	void git_clone(Si::os_string const &repository, Si::absolute_path const &destination, Si::path_segment const &clone_name, Si::absolute_path const &git_exe, Si::sink<char, Si::success> &output)
	{
		Si::async_process_parameters parameters;
		parameters.executable = git_exe;
		parameters.current_path = destination;
		parameters.arguments.emplace_back(Si::to_os_string("clone"));
		parameters.arguments.emplace_back(repository);
		parameters.arguments.emplace_back((destination / clone_name).c_str());
		int exit_code = run_process(parameters, output);
		if (exit_code != 0)
		{
			throw std::runtime_error("git-clone failed");
		}
	}

	build_result run_test(Si::absolute_path const &build_dir, Si::sink<char, Si::success> &output)
	{
		Si::absolute_path const test_dir = build_dir / "test";
		Si::absolute_path const test_exe = test_dir / "unit_test";
		Si::async_process_parameters parameters;
		parameters.executable = test_exe;
		parameters.current_path = test_dir;
		int exit_code = run_process(parameters, output);
		if (exit_code == 0)
		{
			return build_result::success;
		}
		else
		{
			return build_result::failure;
		}
	}

	build_result build(
		Si::os_string const &repository,
		Si::absolute_path const &workspace,
		Si::absolute_path const &git,
		Si::absolute_path const &cmake,
		Si::sink<char, Si::success> &output)
	{
		Si::path_segment const clone_name = *Si::path_segment::create("source.git");
		git_clone(repository, workspace, clone_name, git, output);
		Si::absolute_path const source = workspace / clone_name;

		Si::absolute_path const build = workspace / "build";
		boost::filesystem::create_directories(build.to_boost_path());

		buildserver::cmake_exe cmake_builder(cmake);
		boost::system::error_code error = cmake_builder.generate(source, build, boost::unordered_map<std::string, std::string>{}, output);
		if (error)
		{
			boost::throw_exception(boost::system::system_error(error));
		}

		error = cmake_builder.build(build, boost::thread::hardware_concurrency(), output);
		if (error)
		{
			boost::throw_exception(boost::system::system_error(error));
		}

		return run_test(build, output);
	}
}

int main(int argc, char **argv)
{
	auto parsed_options = parse_options(argc, argv);
	if (!parsed_options)
	{
		return 1;
	}

	Si::optional<Si::absolute_path> maybe_git = buildserver::find_git().get();
	if (!maybe_git)
	{
		std::cerr << "Could not find Git\n";
		return 1;
	}

	Si::optional<Si::absolute_path> maybe_cmake = buildserver::find_cmake().get();
	if (!maybe_cmake)
	{
		std::cerr << "Could not find CMake\n";
		return 1;
	}

	boost::asio::io_service io;

	Si::spawn_coroutine([&](Si::spawn_context yield)
	{
		saturating_notifier<Si::erased_observer<notification>> notifier;
		overview_state overview;

		nanoweb::request_handler root_request_handler = make_root_request_handler(parsed_options->secret, notifier, overview);
		Si::spawn_observable(Si::transform(
			Si::asio::make_tcp_acceptor(boost::asio::ip::tcp::acceptor(io, boost::asio::ip::tcp::endpoint(boost::asio::ip::address_v4::any(), parsed_options->port))),
			[&root_request_handler](Si::asio::tcp_acceptor_result maybe_client) -> Si::nothing
			{
				auto client = maybe_client.get();
				Si::spawn_coroutine([client, &root_request_handler](Si::spawn_context yield)
				{
					auto error = nanoweb::serve_client(*client, yield, root_request_handler);
					if (!!error)
					{
						std::cerr << client->remote_endpoint().address() << ": " << error << '\n';
					}
				});
				return{};
			}
		));

		for (;;)
		{
			Si::optional<notification> notification_ = yield.get_one(Si::ref(notifier));
			assert(notification_);
			std::cerr << "Received a notification\n";
			try
			{
				overview.is_building = true;
				Si::optional<std::future<build_result>> maybe_result = yield.get_one(
					Si::asio::make_posting_observable(
						io,
						Si::make_thread_observable<Si::std_threading>([&]()
						{
							boost::filesystem::remove_all(parsed_options->workspace.to_boost_path());
							boost::filesystem::create_directories(parsed_options->workspace.to_boost_path());
							auto output = Si::virtualize_sink(Si::ostream_ref_sink(std::cerr));
							return build(parsed_options->repository, parsed_options->workspace, *maybe_git, *maybe_cmake, output);
						})
					)
				);
				assert(maybe_result);
				auto const result = maybe_result->get();
				switch (result)
				{
				case build_result::success:
					std::cerr << "Build success\n";
					break;

				case build_result::failure:
					std::cerr << "Build failure\n";
					break;
				}
				overview.last_result = result;
			}
			catch (std::exception const &ex)
			{
				std::cerr << "Exception: " <<  ex.what() << '\n';
				overview.last_result = build_result::failure;
			}
			overview.is_building = false;
		}
	});

	io.run();
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/extensions/api/file_system/file_system_api.h"

#include "base/bind.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "chrome/browser/extensions/shell_window_registry.h"
#include "chrome/browser/platform_util.h"
#include "chrome/browser/ui/extensions/shell_window.h"
#include "chrome/browser/ui/select_file_dialog.h"
#include "chrome/common/extensions/api/file_system.h"
#include "content/public/browser/child_process_security_policy.h"
#include "content/public/browser/render_view_host.h"
#include "content/public/browser/render_process_host.h"
#include "content/public/browser/web_contents.h"
#include "webkit/fileapi/file_system_util.h"
#include "webkit/fileapi/isolated_context.h"

const char kInvalidParameters[] = "Invalid parameters";
const char kSecurityError[] = "Security error";
const char kInvalidCallingPage[] = "Invalid calling page";
const char kUserCancelled[] = "User cancelled";
const char kWritableFileError[] = "Invalid file for writing";

const char kSaveFileOption[] = "saveFile";

namespace file_system = extensions::api::file_system;
namespace ChooseFile = file_system::ChooseFile;

namespace {

bool GetFilePathOfFileEntry(const std::string& filesystem_name,
                            const std::string& filesystem_path,
                            const content::RenderViewHost* render_view_host,
                            FilePath* file_path,
                            std::string* error) {
  std::string filesystem_id;
  if (!fileapi::CrackIsolatedFileSystemName(filesystem_name, &filesystem_id)) {
    *error = kInvalidParameters;
    return false;
  }

  // Only return the display path if the process has read access to the
  // filesystem.
  content::ChildProcessSecurityPolicy* policy =
      content::ChildProcessSecurityPolicy::GetInstance();
  if (!policy->CanReadFileSystem(render_view_host->GetProcess()->GetID(),
                                 filesystem_id)) {
    *error = kSecurityError;
    return false;
  }

  fileapi::IsolatedContext* context = fileapi::IsolatedContext::GetInstance();
  FilePath relative_path = FilePath::FromUTF8Unsafe(filesystem_path);
  FilePath virtual_path = context->CreateVirtualPath(filesystem_id,
                                                     relative_path);
  if (!context->CrackIsolatedPath(virtual_path,
                                  &filesystem_id,
                                  NULL,
                                  file_path)) {
    *error = kInvalidParameters;
    return false;
  }

  return true;
}

bool DoCheckWritableFile(const FilePath& path) {
  // Don't allow links.
  if (file_util::PathExists(path) && file_util::IsLink(path))
    return false;

  // Create the file if it doesn't already exist.
  base::PlatformFileError error = base::PLATFORM_FILE_OK;
  int creation_flags = base::PLATFORM_FILE_CREATE |
                       base::PLATFORM_FILE_READ |
                       base::PLATFORM_FILE_WRITE;
  base::CreatePlatformFile(path, creation_flags, NULL, &error);
  return error == base::PLATFORM_FILE_OK;
}

}  // namespace

namespace extensions {

bool FileSystemGetDisplayPathFunction::RunImpl() {
  std::string filesystem_name;
  std::string filesystem_path;
  EXTENSION_FUNCTION_VALIDATE(args_->GetString(0, &filesystem_name));
  EXTENSION_FUNCTION_VALIDATE(args_->GetString(1, &filesystem_path));

  FilePath file_path;
  if (!GetFilePathOfFileEntry(filesystem_name, filesystem_path,
                              render_view_host_, &file_path, &error_)) {
    return false;
  }

  result_.reset(base::Value::CreateStringValue(file_path.value()));
  return true;
}

// Handles showing a dialog to the user to ask for the filename for a file to
// save.
class FileSystemPickerFunction::FilePicker : public SelectFileDialog::Listener {
 public:
  FilePicker(FileSystemPickerFunction* function,
             content::WebContents* web_contents,
             const FilePath& suggested_path,
             bool for_save)
      : suggested_path_(suggested_path),
        for_save_(for_save),
        function_(function) {
    select_file_dialog_ = SelectFileDialog::Create(this);
    SelectFileDialog::FileTypeInfo file_type_info;
    FilePath::StringType extension = suggested_path.Extension();
    if (!extension.empty()) {
      extension.erase(extension.begin());  // drop the .
      file_type_info.extensions.resize(1);
      file_type_info.extensions[0].push_back(extension);
    }
    file_type_info.include_all_files = true;
    gfx::NativeWindow owning_window = web_contents ?
        platform_util::GetTopLevel(web_contents->GetNativeView()) : NULL;

    select_file_dialog_->SelectFile(for_save ?
                                        SelectFileDialog::SELECT_SAVEAS_FILE :
                                        SelectFileDialog::SELECT_OPEN_FILE,
                                    string16(),
                                    suggested_path,
                                    &file_type_info, 0, FILE_PATH_LITERAL(""),
                                    web_contents, owning_window, NULL);
  }

  virtual ~FilePicker() {}

 private:
  // SelectFileDialog::Listener implementation.
  virtual void FileSelected(const FilePath& path,
                            int index,
                            void* params) OVERRIDE {
    function_->FileSelected(path, for_save_);
    delete this;
  }

  virtual void FileSelectionCanceled(void* params) OVERRIDE {
    function_->FileSelectionCanceled();
    delete this;
  }

  FilePath suggested_path_;

  // for_save_ is false when using the picker to open a file, and true
  // when saving. It affects the style of the dialog and also what happens
  // after a file is selected by the user.
  bool for_save_;

  scoped_refptr<SelectFileDialog> select_file_dialog_;
  scoped_refptr<FileSystemPickerFunction> function_;

  DISALLOW_COPY_AND_ASSIGN(FilePicker);
};

bool FileSystemPickerFunction::ShowPicker(const FilePath& suggested_path,
                                          bool for_save) {
  ShellWindowRegistry* registry = ShellWindowRegistry::Get(profile());
  DCHECK(registry);
  ShellWindow* shell_window = registry->GetShellWindowForRenderViewHost(
      render_view_host());
  if (!shell_window) {
    error_ = kInvalidCallingPage;
    return false;
  }

  // The file picker will hold a reference to this function instance, preventing
  // its destruction (and subsequent sending of the function response) until the
  // user has selected a file or cancelled the picker. At that point, the picker
  // will delete itself, which will also free the function instance.
  new FilePicker(this, shell_window->web_contents(), suggested_path, for_save);
  return true;
}

void FileSystemPickerFunction::FileSelected(const FilePath& path,
                                            bool for_save) {
  if (for_save) {
    content::BrowserThread::PostTask(content::BrowserThread::FILE, FROM_HERE,
        base::Bind(&FileSystemPickerFunction::CheckWritableFile, this, path));
    return;
  }

  // Don't need to check the file, it's for reading.
  RegisterFileSystemAndSendResponse(path, false);
}

void FileSystemPickerFunction::FileSelectionCanceled() {
  error_ = kUserCancelled;
  SendResponse(false);
}

void FileSystemPickerFunction::CheckWritableFile(const FilePath& path) {
  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::FILE));
  if (DoCheckWritableFile(path)) {
    content::BrowserThread::PostTask(content::BrowserThread::UI, FROM_HERE,
        base::Bind(&FileSystemPickerFunction::RegisterFileSystemAndSendResponse,
            this, path, true));
    return;
  }

  content::BrowserThread::PostTask(content::BrowserThread::UI, FROM_HERE,
      base::Bind(&FileSystemPickerFunction::HandleWritableFileError, this));
}

void FileSystemPickerFunction::RegisterFileSystemAndSendResponse(
    const FilePath& path, bool for_save) {
  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
  std::set<FilePath> filesets;
  filesets.insert(path);

  fileapi::IsolatedContext* isolated_context =
      fileapi::IsolatedContext::GetInstance();
  DCHECK(isolated_context);
  std::string filesystem_id = isolated_context->RegisterIsolatedFileSystem(
      filesets);

  content::ChildProcessSecurityPolicy* policy =
      content::ChildProcessSecurityPolicy::GetInstance();
  int renderer_id = render_view_host_->GetProcess()->GetID();
  if (for_save)
    policy->GrantReadWriteFileSystem(renderer_id, filesystem_id);
  else
    policy->GrantReadFileSystem(renderer_id, filesystem_id);

  // We only need file level access for reading FileEntries. Saving FileEntries
  // just needs the file system to have read/write access, which is granted
  // above if required.
  if (!policy->CanReadFile(renderer_id, path))
    policy->GrantReadFile(renderer_id, path);

  DictionaryValue* dict = new DictionaryValue();
  result_.reset(dict);
  dict->SetString("fileSystemId", filesystem_id);
  dict->SetString("baseName", path.BaseName().AsUTF8Unsafe());
  SendResponse(true);
}

void FileSystemPickerFunction::HandleWritableFileError() {
  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
  error_ = kWritableFileError;
  SendResponse(false);
}

bool FileSystemGetWritableFileEntryFunction::RunImpl() {
  std::string filesystem_name;
  std::string filesystem_path;
  EXTENSION_FUNCTION_VALIDATE(args_->GetString(0, &filesystem_name));
  EXTENSION_FUNCTION_VALIDATE(args_->GetString(1, &filesystem_path));

  FilePath suggested_path;
  if (!GetFilePathOfFileEntry(filesystem_name, filesystem_path,
                              render_view_host_, &suggested_path, &error_)) {
    return false;
  }

  return ShowPicker(suggested_path, true);
}

bool FileSystemChooseFileFunction::RunImpl() {
  scoped_ptr<ChooseFile::Params> params(ChooseFile::Params::Create(*args_));
  EXTENSION_FUNCTION_VALIDATE(params.get());

  bool for_save = false;
  file_system::ChooseFileOptions* options = params->options.get();
  if (options) {
    if (options->type.get() &&  *options->type == kSaveFileOption)
      for_save = true;
  }

  return ShowPicker(FilePath(), for_save);
}

}  // namespace extensions
<commit_msg>Fix fileSystem extension API to work when user chooses existing file for writing.<commit_after>// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/extensions/api/file_system/file_system_api.h"

#include "base/bind.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "chrome/browser/extensions/shell_window_registry.h"
#include "chrome/browser/platform_util.h"
#include "chrome/browser/ui/extensions/shell_window.h"
#include "chrome/browser/ui/select_file_dialog.h"
#include "chrome/common/extensions/api/file_system.h"
#include "content/public/browser/child_process_security_policy.h"
#include "content/public/browser/render_view_host.h"
#include "content/public/browser/render_process_host.h"
#include "content/public/browser/web_contents.h"
#include "webkit/fileapi/file_system_util.h"
#include "webkit/fileapi/isolated_context.h"

const char kInvalidParameters[] = "Invalid parameters";
const char kSecurityError[] = "Security error";
const char kInvalidCallingPage[] = "Invalid calling page";
const char kUserCancelled[] = "User cancelled";
const char kWritableFileError[] = "Invalid file for writing";

const char kSaveFileOption[] = "saveFile";

namespace file_system = extensions::api::file_system;
namespace ChooseFile = file_system::ChooseFile;

namespace {

bool GetFilePathOfFileEntry(const std::string& filesystem_name,
                            const std::string& filesystem_path,
                            const content::RenderViewHost* render_view_host,
                            FilePath* file_path,
                            std::string* error) {
  std::string filesystem_id;
  if (!fileapi::CrackIsolatedFileSystemName(filesystem_name, &filesystem_id)) {
    *error = kInvalidParameters;
    return false;
  }

  // Only return the display path if the process has read access to the
  // filesystem.
  content::ChildProcessSecurityPolicy* policy =
      content::ChildProcessSecurityPolicy::GetInstance();
  if (!policy->CanReadFileSystem(render_view_host->GetProcess()->GetID(),
                                 filesystem_id)) {
    *error = kSecurityError;
    return false;
  }

  fileapi::IsolatedContext* context = fileapi::IsolatedContext::GetInstance();
  FilePath relative_path = FilePath::FromUTF8Unsafe(filesystem_path);
  FilePath virtual_path = context->CreateVirtualPath(filesystem_id,
                                                     relative_path);
  if (!context->CrackIsolatedPath(virtual_path,
                                  &filesystem_id,
                                  NULL,
                                  file_path)) {
    *error = kInvalidParameters;
    return false;
  }

  return true;
}

bool DoCheckWritableFile(const FilePath& path) {
  // Don't allow links.
  if (file_util::PathExists(path) && file_util::IsLink(path))
    return false;

  // Create the file if it doesn't already exist.
  base::PlatformFileError error = base::PLATFORM_FILE_OK;
  int creation_flags = base::PLATFORM_FILE_CREATE |
                       base::PLATFORM_FILE_READ |
                       base::PLATFORM_FILE_WRITE;
  base::CreatePlatformFile(path, creation_flags, NULL, &error);
  return error == base::PLATFORM_FILE_OK ||
         error == base::PLATFORM_FILE_ERROR_EXISTS;
}

}  // namespace

namespace extensions {

bool FileSystemGetDisplayPathFunction::RunImpl() {
  std::string filesystem_name;
  std::string filesystem_path;
  EXTENSION_FUNCTION_VALIDATE(args_->GetString(0, &filesystem_name));
  EXTENSION_FUNCTION_VALIDATE(args_->GetString(1, &filesystem_path));

  FilePath file_path;
  if (!GetFilePathOfFileEntry(filesystem_name, filesystem_path,
                              render_view_host_, &file_path, &error_)) {
    return false;
  }

  result_.reset(base::Value::CreateStringValue(file_path.value()));
  return true;
}

// Handles showing a dialog to the user to ask for the filename for a file to
// save.
class FileSystemPickerFunction::FilePicker : public SelectFileDialog::Listener {
 public:
  FilePicker(FileSystemPickerFunction* function,
             content::WebContents* web_contents,
             const FilePath& suggested_path,
             bool for_save)
      : suggested_path_(suggested_path),
        for_save_(for_save),
        function_(function) {
    select_file_dialog_ = SelectFileDialog::Create(this);
    SelectFileDialog::FileTypeInfo file_type_info;
    FilePath::StringType extension = suggested_path.Extension();
    if (!extension.empty()) {
      extension.erase(extension.begin());  // drop the .
      file_type_info.extensions.resize(1);
      file_type_info.extensions[0].push_back(extension);
    }
    file_type_info.include_all_files = true;
    gfx::NativeWindow owning_window = web_contents ?
        platform_util::GetTopLevel(web_contents->GetNativeView()) : NULL;

    select_file_dialog_->SelectFile(for_save ?
                                        SelectFileDialog::SELECT_SAVEAS_FILE :
                                        SelectFileDialog::SELECT_OPEN_FILE,
                                    string16(),
                                    suggested_path,
                                    &file_type_info, 0, FILE_PATH_LITERAL(""),
                                    web_contents, owning_window, NULL);
  }

  virtual ~FilePicker() {}

 private:
  // SelectFileDialog::Listener implementation.
  virtual void FileSelected(const FilePath& path,
                            int index,
                            void* params) OVERRIDE {
    function_->FileSelected(path, for_save_);
    delete this;
  }

  virtual void FileSelectionCanceled(void* params) OVERRIDE {
    function_->FileSelectionCanceled();
    delete this;
  }

  FilePath suggested_path_;

  // for_save_ is false when using the picker to open a file, and true
  // when saving. It affects the style of the dialog and also what happens
  // after a file is selected by the user.
  bool for_save_;

  scoped_refptr<SelectFileDialog> select_file_dialog_;
  scoped_refptr<FileSystemPickerFunction> function_;

  DISALLOW_COPY_AND_ASSIGN(FilePicker);
};

bool FileSystemPickerFunction::ShowPicker(const FilePath& suggested_path,
                                          bool for_save) {
  ShellWindowRegistry* registry = ShellWindowRegistry::Get(profile());
  DCHECK(registry);
  ShellWindow* shell_window = registry->GetShellWindowForRenderViewHost(
      render_view_host());
  if (!shell_window) {
    error_ = kInvalidCallingPage;
    return false;
  }

  // The file picker will hold a reference to this function instance, preventing
  // its destruction (and subsequent sending of the function response) until the
  // user has selected a file or cancelled the picker. At that point, the picker
  // will delete itself, which will also free the function instance.
  new FilePicker(this, shell_window->web_contents(), suggested_path, for_save);
  return true;
}

void FileSystemPickerFunction::FileSelected(const FilePath& path,
                                            bool for_save) {
  if (for_save) {
    content::BrowserThread::PostTask(content::BrowserThread::FILE, FROM_HERE,
        base::Bind(&FileSystemPickerFunction::CheckWritableFile, this, path));
    return;
  }

  // Don't need to check the file, it's for reading.
  RegisterFileSystemAndSendResponse(path, false);
}

void FileSystemPickerFunction::FileSelectionCanceled() {
  error_ = kUserCancelled;
  SendResponse(false);
}

void FileSystemPickerFunction::CheckWritableFile(const FilePath& path) {
  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::FILE));
  if (DoCheckWritableFile(path)) {
    content::BrowserThread::PostTask(content::BrowserThread::UI, FROM_HERE,
        base::Bind(&FileSystemPickerFunction::RegisterFileSystemAndSendResponse,
            this, path, true));
    return;
  }

  content::BrowserThread::PostTask(content::BrowserThread::UI, FROM_HERE,
      base::Bind(&FileSystemPickerFunction::HandleWritableFileError, this));
}

void FileSystemPickerFunction::RegisterFileSystemAndSendResponse(
    const FilePath& path, bool for_save) {
  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
  std::set<FilePath> filesets;
  filesets.insert(path);

  fileapi::IsolatedContext* isolated_context =
      fileapi::IsolatedContext::GetInstance();
  DCHECK(isolated_context);
  std::string filesystem_id = isolated_context->RegisterIsolatedFileSystem(
      filesets);

  content::ChildProcessSecurityPolicy* policy =
      content::ChildProcessSecurityPolicy::GetInstance();
  int renderer_id = render_view_host_->GetProcess()->GetID();
  if (for_save)
    policy->GrantReadWriteFileSystem(renderer_id, filesystem_id);
  else
    policy->GrantReadFileSystem(renderer_id, filesystem_id);

  // We only need file level access for reading FileEntries. Saving FileEntries
  // just needs the file system to have read/write access, which is granted
  // above if required.
  if (!policy->CanReadFile(renderer_id, path))
    policy->GrantReadFile(renderer_id, path);

  DictionaryValue* dict = new DictionaryValue();
  result_.reset(dict);
  dict->SetString("fileSystemId", filesystem_id);
  dict->SetString("baseName", path.BaseName().AsUTF8Unsafe());
  SendResponse(true);
}

void FileSystemPickerFunction::HandleWritableFileError() {
  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
  error_ = kWritableFileError;
  SendResponse(false);
}

bool FileSystemGetWritableFileEntryFunction::RunImpl() {
  std::string filesystem_name;
  std::string filesystem_path;
  EXTENSION_FUNCTION_VALIDATE(args_->GetString(0, &filesystem_name));
  EXTENSION_FUNCTION_VALIDATE(args_->GetString(1, &filesystem_path));

  FilePath suggested_path;
  if (!GetFilePathOfFileEntry(filesystem_name, filesystem_path,
                              render_view_host_, &suggested_path, &error_)) {
    return false;
  }

  return ShowPicker(suggested_path, true);
}

bool FileSystemChooseFileFunction::RunImpl() {
  scoped_ptr<ChooseFile::Params> params(ChooseFile::Params::Create(*args_));
  EXTENSION_FUNCTION_VALIDATE(params.get());

  bool for_save = false;
  file_system::ChooseFileOptions* options = params->options.get();
  if (options) {
    if (options->type.get() &&  *options->type == kSaveFileOption)
      for_save = true;
  }

  return ShowPicker(FilePath(), for_save);
}

}  // namespace extensions
<|endoftext|>


<commit_before>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/command_line.h"
#include "chrome/browser/extensions/extension_apitest.h"
#include "chrome/common/chrome_switches.h"

IN_PROC_BROWSER_TEST_F(ExtensionApiTest, WebNavigation) {
  CommandLine::ForCurrentProcess()->AppendSwitch(
      switches::kEnableExperimentalExtensionApis);

  ASSERT_TRUE(RunExtensionTest("webnavigation/api")) << message_;
}

IN_PROC_BROWSER_TEST_F(ExtensionApiTest, WebNavigationEvents) {
  CommandLine::ForCurrentProcess()->AppendSwitch(
      switches::kEnableExperimentalExtensionApis);

  ASSERT_TRUE(RunExtensionTest("webnavigation/navigation")) << message_;
}
<commit_msg>Mark ExtensionApiTest.WebNavigationEvents flaky<commit_after>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/command_line.h"
#include "chrome/browser/extensions/extension_apitest.h"
#include "chrome/common/chrome_switches.h"

IN_PROC_BROWSER_TEST_F(ExtensionApiTest, WebNavigation) {
  CommandLine::ForCurrentProcess()->AppendSwitch(
      switches::kEnableExperimentalExtensionApis);

  ASSERT_TRUE(RunExtensionTest("webnavigation/api")) << message_;
}

// This test is flaky: http://crbug.com/60229
IN_PROC_BROWSER_TEST_F(ExtensionApiTest, FLAKY_WebNavigationEvents) {
  CommandLine::ForCurrentProcess()->AppendSwitch(
      switches::kEnableExperimentalExtensionApis);

  ASSERT_TRUE(RunExtensionTest("webnavigation/navigation")) << message_;
}
<|endoftext|>


<commit_before><commit_msg>Remove a reference to IDR_WARNING_ICON<commit_after><|endoftext|>


<commit_before>/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2012 Heiko Strathmann
 */

#include <shogun/base/init.h>
#include <shogun/statistics/QuadraticTimeMMD.h>
#include <shogun/kernel/GaussianKernel.h>
#include <shogun/kernel/CustomKernel.h>
#include <shogun/features/DenseFeatures.h>
#include <shogun/mathematics/Statistics.h>
#include <shogun/features/DataGenerator.h>

using namespace shogun;

/** tests the quadratic mmd statistic for a single data case and ensures
 * equality with matlab implementation */
void test_quadratic_mmd_fixed()
{
	index_t n=2;
	index_t d=3;
	float64_t sigma=2;
	float64_t sq_sigma_twice=sigma*sigma*2;
	SGMatrix<float64_t> data(d,2*n);
	for (index_t i=0; i<2*d*n; ++i)
		data.matrix[i]=i;

	CDenseFeatures<float64_t>* features=new CDenseFeatures<float64_t>(data);
	CGaussianKernel* kernel=new CGaussianKernel(10, sq_sigma_twice);
	kernel->init(features, features);

	CQuadraticTimeMMD* mmd=new CQuadraticTimeMMD(kernel, features, n);

	/* unbiased statistic, shogun return m*MMD */
	mmd->set_statistic_type(UNBIASED);
	float64_t difference=CMath::abs(mmd->compute_statistic()/n-0.051325806508381);
	ASSERT(difference<=10E-16);

	/* biased statistic */
	mmd->set_statistic_type(BIASED);
	difference=CMath::abs(mmd->compute_statistic()/n-1.017107688196714);
	ASSERT(difference<=10E-16);

	SG_UNREF(mmd);
}

/** tests the quadratic mmd statistic bootstrapping for a random data case and
 * ensures equality with matlab implementation (unbiased statistic) */
void test_quadratic_mmd_bootstrap()
{
	/* reproducable results */
	CMath::init_random(1);

	index_t dimension=3;
	index_t m=100;
	float64_t difference=0.5;
	float64_t sigma=2;
	index_t num_iterations=1000;
	num_iterations=10; //speed up

	SGMatrix<float64_t> data=CDataGenerator::generate_mean_data(m, dimension,
			difference);
	CDenseFeatures<float64_t>* features=new CDenseFeatures<float64_t>(data);

	/* shoguns kernel width is different */
	CGaussianKernel* kernel=new CGaussianKernel(100, sigma*sigma*2);
	CQuadraticTimeMMD* mmd=new CQuadraticTimeMMD(kernel, features, m);
	mmd->set_statistic_type(UNBIASED);
	mmd->set_bootstrap_iterations(num_iterations);

	/* use fixed seed */
	CMath::init_random(1);
	SGVector<float64_t> null_samples=mmd->bootstrap_null();

	float64_t mean=CStatistics::mean(null_samples);
	float64_t var=CStatistics::variance(null_samples);

	/* MATLAB mean 2-sigma confidence interval for 1000 repretitions is
	 * [-3.169406734013459e-04, 3.296399498466372e-04] */
	SG_SPRINT("mean %f\n", mean);
//	ASSERT(mean>-3.169406734013459e-04);
//	ASSERT(mean<3.296399498466372e-04);

	/* MATLAB variance 2-sigma confidence interval for 1000 repretitions is
	 * [2.194192869469228e-05,2.936672859339959e-05] */
	SG_SPRINT("var %f\n", var);
//	ASSERT(var>2.194192869469228e-05);
//	ASSERT(var<2.936672859339959e-05);

	/* now again but with a precomputed kernel, same features.
	 * This avoids re-computing the kernel matrix in every bootstrapping
	 * iteration and should be num_iterations times faster */
	SG_REF(features);

	/* re-init kernel before kernel matrix is computed: this is due to a design
	 * error in subsets and should be worked on! */
	kernel->init(features, features);
	CCustomKernel* precomputed_kernel=new CCustomKernel(kernel);
	SG_UNREF(mmd);
	mmd=new CQuadraticTimeMMD(precomputed_kernel, features, m);
	mmd->set_statistic_type(UNBIASED);
	mmd->set_bootstrap_iterations(num_iterations);
	CMath::init_random(1);
	null_samples=mmd->bootstrap_null();

	/* assert that results do not change */
	SG_SPRINT("mean %f, var %f\n", CStatistics::mean(null_samples),
			CStatistics::variance(null_samples));
	ASSERT(CMath::abs(mean-CStatistics::mean(null_samples))<10E-5);
	ASSERT(CMath::abs(var-CStatistics::variance(null_samples))<10E-5);

	SG_UNREF(mmd);
	SG_UNREF(features);
}

#ifdef HAVE_LAPACK
/** tests the quadratic mmd statistic threshold method spectrum for radnom data
 * case and ensures equality with matlab implementation */
void test_quadratic_mmd_spectrum()
{
	index_t dimension=3;
	index_t m=100;
	float64_t difference=0.5;
	float64_t sigma=2;

	SGMatrix<float64_t> data=CDataGenerator::generate_mean_data(m, dimension,
			difference);

	CDenseFeatures<float64_t>* features=new CDenseFeatures<float64_t>(data);

	/* shoguns kernel width is different */
	CGaussianKernel* kernel=new CGaussianKernel(100, sigma*sigma*2);
	CQuadraticTimeMMD* mmd=new CQuadraticTimeMMD(kernel, features, m);

	mmd->set_num_samples_sepctrum(1000);
	mmd->set_num_samples_sepctrum(10); //speed up
	mmd->set_num_eigenvalues_spectrum(m);
	mmd->set_null_approximation_method(MMD2_SPECTRUM);
	mmd->set_statistic_type(BIASED);

	/* compute p-value for a fixed statistic value */
	float64_t p=mmd->compute_p_value(2);

	/* MATLAB 1000 iterations 3 sigma confidence interval is
	 * [0.021240218376709, 0.060875781623291] */
	SG_SPRINT("p %f\n", p);
//	ASSERT(p>0.021240218376709);
//	ASSERT(p<0.060875781623291);

	SG_UNREF(mmd);
}
#endif // HAVE_LAPACK

/** tests the quadratic mmd statistic threshold method gamma for fixed data
 * case and ensures equality with matlab implementation */
void test_quadratic_mmd_gamma()
{
	index_t dimension=3;
	index_t m=100;
	float64_t sigma=4;

	/* note: fixed data this time */
	SGMatrix<float64_t> data(dimension, 2*m);
	for (index_t i=0; i<2*dimension*m; ++i)
		data.matrix[i]=i;

	CDenseFeatures<float64_t>* features=new CDenseFeatures<float64_t>(data);

	/* shoguns kernel width is different */
	CGaussianKernel* kernel=new CGaussianKernel(100, sigma*sigma*2);
	CQuadraticTimeMMD* mmd=new CQuadraticTimeMMD(kernel, features, m);

	mmd->set_null_approximation_method(MMD2_GAMMA);
	mmd->set_statistic_type(BIASED);

	/* compute p-value for a fixed statistic value */
	float64_t p=mmd->compute_p_value(2);
	SG_SPRINT("p: %f\n", p);

	/* MATLAB 1000 iterations mean: 0.511547577996229 with variance 10E-15,
	 * asserting with only 10-12 to avoid problems. Shold never fail.
	 */
	ASSERT(CMath::abs(p-0.511547577996229)<10E-12);

	SG_UNREF(mmd);
}

/** tests the quadratic mmd statistic for a random data case (fixed distribution
 * and ensures equality with matlab implementation (unbiased case) */
void test_quadratic_mmd_random()
{
	index_t dimension=3;
	index_t m=300;
	float64_t difference=0.5;
	float64_t sigma=2;

	index_t num_runs=100;
	num_runs=10; //speed up
	SGVector<float64_t> mmds(num_runs);

	/* pre-allocate data matrix and features, just change elements later */
	SGMatrix<float64_t> data(dimension, 2*m);
	CDenseFeatures<float64_t>* features=new CDenseFeatures<float64_t>(data);

	/* shoguns kernel width is different */
	CGaussianKernel* kernel=new CGaussianKernel(100, sigma*sigma*2);
	CQuadraticTimeMMD* mmd=new CQuadraticTimeMMD(kernel, features, m);
	mmd->set_statistic_type(UNBIASED);
	for (index_t i=0; i<num_runs; ++i)
	{
		/* use pre-allocated space for data generation */
		CDataGenerator::generate_mean_data(m, dimension, difference, data);
		kernel->init(features, features);
		mmds[i]=mmd->compute_statistic();
	}

	/* MATLAB 95% mean confidence interval 0.007495841715582 0.037960088792417 */
	float64_t mean=CStatistics::mean(mmds);
	SG_SPRINT("mean %f\n", mean);
//	ASSERT((mean>0.007495841715582) && (mean<0.037960088792417));

	/* MATLAB variance is 5.800439687240292e-05 quite stable */
	float64_t variance=CStatistics::variance(mmds);
	SG_SPRINT("variance: %f\n", variance);
//	ASSERT(CMath::abs(variance-5.800439687240292e-05)<10E-5);
	SG_UNREF(mmd);
}

int main(int argc, char** argv)
{
	init_shogun_with_defaults();

//	sg_io->set_loglevel(MSG_DEBUG);

	/* all tests have been "speed up" by reducing the number of runs/samples.
	 * If you have any doubts in the results, set all num_runs to original
	 * numbers and activate asserts. If they fail, something is wrong. */

	test_quadratic_mmd_fixed();
	test_quadratic_mmd_random();
	test_quadratic_mmd_bootstrap();
#ifdef HAVE_LAPACK
	test_quadratic_mmd_spectrum();
#endif
	test_quadratic_mmd_gamma();

	exit_shogun();
	return 0;
}

<commit_msg>added an illustrativ example for the quadratic time MMD<commit_after>/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2013 Heiko Strathmann
 */

#include <shogun/base/init.h>
#include <shogun/statistics/QuadraticTimeMMD.h>
#include <shogun/kernel/GaussianKernel.h>
#include <shogun/features/DenseFeatures.h>
#include <shogun/features/streaming/generators/MeanShiftDataGenerator.h>
#include <shogun/mathematics/Statistics.h>

using namespace shogun;

void quadratic_time_mmd()
{
	/* number of examples kept low in order to make things fast */
	index_t m=30;
	index_t dim=2;
	float64_t difference=0.5;

	/* streaming data generator for mean shift distributions */
	CMeanShiftDataGenerator* gen_p=new CMeanShiftDataGenerator(0, dim);
	CMeanShiftDataGenerator* gen_q=new CMeanShiftDataGenerator(difference, dim);

	/* stream some data from generator */
	CFeatures* feat_p=gen_p->get_streamed_features(m);
	CFeatures* feat_q=gen_q->get_streamed_features(m);

	/* set kernel a-priori. usually one would do some kernel selection. See
	 * other examples for this. */
	float64_t width=10;
	CGaussianKernel* kernel=new CGaussianKernel(10, width);

	/* create quadratic time mmd instance. Note that this constructor
	 * copies p and q and does not reference them */
	CQuadraticTimeMMD* mmd=new CQuadraticTimeMMD(kernel, feat_p, feat_q);

	/* perform test: compute p-value and test if null-hypothesis is rejected for
	 * a test level of 0.05 */
	float64_t alpha=0.05;

	/* using bootstrapping (slow, not the most reliable way. Consider pre-
	 * computing the kernel when using it, see below).
	 * Also, in practice, use at least 250 iterations */
	mmd->set_null_approximation_method(BOOTSTRAP);
	mmd->set_bootstrap_iterations(3);
	float64_t p_value_bootstrap=mmd->perform_test();
	/* reject if p-value is smaller than test level */
	SG_SPRINT("bootstrap: p!=q: %d\n", p_value_bootstrap<alpha);

	/* using spectrum method. Use at least 250 samples from null.
	 * This is consistent but sometimes breaks, always monitor type I error.
	 * See tutorial for number of eigenvalues to use .
	 * Only works with BIASED statistic */
	mmd->set_statistic_type(BIASED);
	mmd->set_null_approximation_method(MMD2_SPECTRUM);
	mmd->set_num_eigenvalues_spectrum(3);
	mmd->set_num_samples_sepctrum(250);
	p_value_bootstrap=mmd->perform_test();
	/* reject if p-value is smaller than test level */
	SG_SPRINT("spectrum: p!=q: %d\n", p_value_bootstrap<alpha);

	/* using gamma method. This is a quick hack, which works most of the time
	 * but is NOT guaranteed to. See tutorial for details.
	 * Only works with BIASED statistic */
	mmd->set_statistic_type(BIASED);
	mmd->set_null_approximation_method(MMD2_GAMMA);
	p_value_bootstrap=mmd->perform_test();
	/* reject if p-value is smaller than test level */
	SG_SPRINT("gamma: p!=q: %d\n", p_value_bootstrap<alpha);

	/* compute tpye I and II error (use many more trials in practice).
	 * Type I error is not necessary if one uses bootstrapping. We do it here
	 * anyway, but note that this is an efficient way of computing it.
	 * Also note that testing has to happen on
	 * difference data than kernel selection, but the linear time mmd does this
	 * implicitly and we used a fixed kernel here. */
	index_t num_trials=3;
	SGVector<float64_t> typeIerrors(num_trials);
	SGVector<float64_t> typeIIerrors(num_trials);
	SGVector<index_t> inds(2*m);
	inds.range_fill();
	CFeatures* p_and_q=mmd->get_p_and_q();

	/* use a precomputed kernel to be faster */
	for (index_t i=0; i<num_trials; ++i)
	{
		/* this effectively means that p=q - rejecting is tpye I error */
		inds.permute();
		p_and_q->add_subset(inds);
		typeIerrors[i]=mmd->perform_test()>alpha;
		p_and_q->remove_subset();

		typeIIerrors[i]=mmd->perform_test()>alpha;
	}
	SG_UNREF(p_and_q);

	SG_SPRINT("type I error: %f\n", CStatistics::mean(typeIerrors));
	SG_SPRINT("type II error: %f\n", CStatistics::mean(typeIIerrors));

	/* clean up */
	SG_UNREF(mmd);
	SG_UNREF(gen_p);
	SG_UNREF(gen_q);

	/* convienience constructor of MMD was used, these were not referenced */
	SG_UNREF(feat_p);
	SG_UNREF(feat_q);
}

int main(int argc, char** argv)
{
	init_shogun_with_defaults();
//	sg_io->set_loglevel(MSG_DEBUG);

	quadratic_time_mmd();

	exit_shogun();
	return 0;
}

<|endoftext|>