Tries to get the scan parameters needed for analysis from the raw data file
509728
def _deduce_settings_from_file(self, opened_raw_data_file):  # TODO: parse better
        '''
        '''
        try:  # take infos raw data files (not avalable in old files)
            flavor = opened_raw_data_file.root.configuration.miscellaneous[:][np.where(opened_raw_data_file.root.configuration.miscellaneous[:]['name'] == 'Flavor')]['value'][0]
            self._settings_from_file_set = True
            # adding this for special cases e.g., stop-mode scan
            if "trig_count" in opened_raw_data_file.root.configuration.run_conf[:]['name']:
                trig_count = opened_raw_data_file.root.configuration.run_conf[:][np.where(opened_raw_data_file.root.configuration.run_conf[:]['name'] == 'trig_count')]['value'][0]
            else:
                trig_count = opened_raw_data_file.root.configuration.global_register[:][np.where(opened_raw_data_file.root.configuration.global_register[:]['name'] == 'Trig_Count')]['value'][0]
            vcal_c0 = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'Vcal_Coeff_0')]['value'][0]
            vcal_c1 = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'Vcal_Coeff_1')]['value'][0]
            c_low = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'C_Inj_Low')]['value'][0]
            c_mid = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'C_Inj_Med')]['value'][0]
            c_high = opened_raw_data_file.root.configuration.calibration_parameters[:][np.where(opened_raw_data_file.root.configuration.calibration_parameters[:]['name'] == 'C_Inj_High')]['value'][0]
            self.c_low_mask = opened_raw_data_file.root.configuration.C_Low[:]
            self.c_high_mask = opened_raw_data_file.root.configuration.C_High[:]
            self.fei4b = False if str(flavor) == 'fei4a' else True
            self.trig_count = int(trig_count)
            self.vcal_c0 = float(vcal_c0)
            self.vcal_c1 = float(vcal_c1)
            self.c_low = float(c_low)
            self.c_mid = float(c_mid)
            self.c_high = float(c_high)
            self.n_injections = int(opened_raw_data_file.root.configuration.run_conf[:][np.where(opened_raw_data_file.root.configuration.run_conf[:]['name'] == 'n_injections')]['value'][0])
        except tb.exceptions.NoSuchNodeError:
            if not self._settings_from_file_set:
                logging.warning('No settings stored in raw data file %s, use standard settings', opened_raw_data_file.filename)
            else:
                logging.info('No settings provided in raw data file %s, use already set settings', opened_raw_data_file.filename)
        except IndexError:  # happens if setting is not available (e.g. repeat_command)
            pass
54355
def create_scans(urls_file):
    
    cli_logger.debug('Starting to process batch input file')
    created_scans = []

    for line in urls_file:
        line = line.strip()

        if line.startswith('#'):
            continue

        if not line:
            continue

        try:
            protocol, domain, port, path = parse_url(line)
        except ValueError, ve:
            cli_logger.debug(str(ve))
            continue

        for scan in created_scans:
            if scan.matches(protocol, domain, port):
                scan.add_path(path)
                args = (path, scan.get_root_url())
                cli_logger.debug('Added %s to %s' % args)
                break
        else:
            scan = BatchScan(protocol, domain, port, path)
            created_scans.append(scan)
            cli_logger.debug('Added a new scan to %s' % scan.get_root_url())

    cli_logger.debug('Created a total of %s scans' % len(created_scans))
    return created_scans
1022209
def parameter(self):
        '''
        
        '''
        method = self.request.method.lower()
        arguments = self.request.arguments
        files = self.request.files

        rpd = {}  # request parameter dict

        def filter_parameter(key, tp, default=None):
            if tp not in self._types:
                raise ValueError(
                    '%s parameter expected types %s' % (key, self._types))

            if not isinstance(tp, file_types):
                if key not in arguments:
                    rpd[key] = default
                    return

                if tp in [ObjectId, int, float, bool]:
                    rpd[key] = getattr(self, 'to_%s' % getattr(
                        tp, '__name__').lower())(self.get_argument(key))
                    return

                if tp == basestring_type or issubclass(tp, basestring_type):
                    rpd[key] = self.get_argument(key, strip=False)
                    return

                if tp == list:
                    rpd[key] = self.get_arguments(key)
                    return

            if tp == file:
                if key not in files:
                    rpd[key] = []
                    return

                rpd[key] = self.request.files[key]

        required_params = getattr(self, '_required_params', None)
        if isinstance(required_params, list):
            for key, tp, default in required_params:
                filter_parameter(key, tp, default)

        # extract method required params
        method_required_params = getattr(
            self, '_%s_required_params' % method, None)
        if isinstance(method_required_params, list):
            for key, tp, default in method_required_params:
                filter_parameter(key, tp, default)

        params = getattr(self, '_%s_params' % method, None)
        if params is None:
            return rpd

        # need arguments
        try:
            for key, tp in params.get('need', []):
                if tp == list:
                    filter_parameter(key, tp, [])
                else:
                    filter_parameter(key, tp)
        except ValueError as e:
            app_log.error(
                '%s request need arguments parse error: %s' % (method, e))
            raise ValueError(e)
        except Exception as e:
            app_log.error(
                '%s request need arguments parse error: %s' % (method, e))
            raise e

        # option arguments
        for key, tp, default in params.get('option', []):
            filter_parameter(key, tp, default)

        return rpd
337569
def scan_file(self, this_file):
        
        params = {'apikey': self.api_key}
        try:
            if type(this_file) == str and os.path.isfile(this_file):
                files = {'file': (this_file, open(this_file, 'rb'))}
            elif isinstance(this_file, StringIO.StringIO):
                files = {'file': this_file.read()}
            else:
                files = {'file': this_file}
        except TypeError as e:
            return dict(error=e.message)

        try:
            response = requests.post(self.base + 'file/scan', files=files, params=params, proxies=self.proxies)
        except requests.RequestException as e:
            return dict(error=e.message)

        return _return_response_and_status_code(response)
1102156
def print_raw_data_file(input_file, start_index=0, limit=200, flavor='fei4b', select=None, tdc_trig_dist=False, trigger_data_mode=0, meta_data_v2=True):
    
    with tb.open_file(input_file + '.h5', mode="r") as file_h5:
        if meta_data_v2:
            index_start = file_h5.root.meta_data.read(field='index_start')
            index_stop = file_h5.root.meta_data.read(field='index_stop')
        else:
            index_start = file_h5.root.meta_data.read(field='start_index')
            index_stop = file_h5.root.meta_data.read(field='stop_index')
        total_words = 0
        for read_out_index, (index_start, index_stop) in enumerate(np.column_stack((index_start, index_stop))):
            if start_index < index_stop:
                print "\nchunk %d with length %d (from index %d to %d)\n" % (read_out_index, (index_stop - index_start), index_start, index_stop)
                raw_data = file_h5.root.raw_data.read(index_start, index_stop)
                total_words += print_raw_data(raw_data=raw_data, start_index=max(start_index - index_start, 0), limit=limit - total_words, flavor=flavor, index_offset=index_start, select=select, tdc_trig_dist=tdc_trig_dist, trigger_data_mode=trigger_data_mode)
                if limit and total_words >= limit:
                    break
245815
def scan_handler(parser, args):
    """"""
    opts = namespace_to_dict(args)
    opts.update({"ftp_debug": args.verbose >= 6})
    target = make_target(args.target, opts)
    target.readonly = True
    root_depth = target.root_dir.count("/")
    start = time.time()
    dir_count = 1
    file_count = 0
    processed_files = set()

    opts = namespace_to_dict(args)
    process_options(opts)

    def _pred(entry):
        """Walker predicate that check match/exclude options."""
        if not match_path(entry, opts):
            return False

    try:
        target.open()
        for e in target.walk(recursive=args.recursive, pred=_pred):
            is_dir = isinstance(e, DirectoryEntry)
            indent = "    " * (target.cur_dir.count("/") - root_depth)

            if is_dir:
                dir_count += 1
            else:
                file_count += 1

            if args.list:
                if is_dir:
                    print(indent, "[{e.name}]".format(e=e))
                else:
                    delta = e.mtime_org - e.mtime
                    dt_modified = pretty_stamp(e.mtime)
                    if delta:
                        prefix = "+" if delta > 0 else ""
                        print(
                            indent,
                            "{e.name:<40} {dt_modified} (system: {prefix}{delta})".format(
                                e=e,
                                prefix=prefix,
                                delta=timedelta(seconds=delta),
                                dt_modified=dt_modified,
                            ),
                        )
                    else:
                        print(
                            indent,
                            "{e.name:<40} {dt_modified}".format(
                                e=e, dt_modified=dt_modified
                            ),
                        )

            if (
                args.remove_meta
                and target.cur_dir_meta
                and target.cur_dir_meta.was_read
            ):
                fspec = target.cur_dir_meta.get_full_path()
                if fspec not in processed_files:
                    processed_files.add(fspec)
                    print("DELETE {}".format(fspec))

            if (
                args.remove_locks
                and not is_dir
                and e.name == DirMetadata.LOCK_FILE_NAME
            ):
                fspec = e.get_rel_path()
                print("DELETE {}".format(fspec))
    finally:
        target.close()

    print(
        "Scanning {:,} files in {:,} directories took {:02.2f} seconds.".format(
            file_count, dir_count, time.time() - start
        )
    )
59891
def execute(self, input_data):
        '''  '''
        raw_bytes = input_data['sample']['raw_bytes']

        # Have the PE File module process the file
        try:
            pefile_handle = pefile.PE(data=raw_bytes, fast_load=False)
        except (AttributeError, pefile.PEFormatError), error:
            return {'error':  str(error), 'match_list': []}

        # Now get information from PEID module
        peid_match = self.peid_features(pefile_handle)
        return {'match_list': peid_match}
1165970
def process_file(self, path, dryrun):
        
        # if dryrun just return files
        if dryrun:
            return path

        # scan file and match lines
        ret = []
        with open(path, "r") as infile:
            for line in infile:
                if re.search(self.__exp, line):
                    ret.append(line)

        # if found matches return list of lines, else return None
        return ret if len(ret) > 0 else None
1094633
def getAnalogChannelData(self,ChNumber):
        

        if not self.DatFileContent:
            print "No data file content. Use the method ReadDataFile first"
            return 0
        
        if (ChNumber > self.A):
            print "Channel number greater than the total number of channels."
            return 0
            
        # Fomating string for struct module:
        str_struct = "ii%dh" %(self.A + int(numpy.ceil((float(self.D)/float(16)))))
        # Number of bytes per sample:
        NB = 4 + 4 + self.A*2 + int(numpy.ceil((float(self.D)/float(16))))*2        
        # Number of samples:
        N = self.getNumberOfSamples()
        
        # Empty column vector:
        values = numpy.empty((N,1))

        ch_index = self.An.index(ChNumber)

        # Reading the values from DatFileContent string:
        for i in range(N):
            data = struct.unpack(str_struct,self.DatFileContent[i*NB:(i*NB)+NB])
            values[i] = data[ChNumber+1] # The first two number ar the sample index and timestamp

        values = values * self.a[ch_index] # a factor
        values = values + self.b[ch_index] # b factor
        
        return values
406271
def gather_data(mypath, key="width"):
    
    filenames = [join(mypath, f) for f in listdir(mypath) if isfile(join(mypath, f))]
    filenames = [f for f in filenames if f.endswith(".p")] 
    W = []
    for filename in filenames:
        with open(filename, 'r') as f:
            data = pickle.load(f)
            W.append(np.expand_dims(data[key], axis=-1) )

    W = np.concatenate(W, axis=-1)
    return W
793809
def scan_file(pym, filename, sentinel, installed):
    '''
    '''
    if not utils.is_python_script(filename):
        return

    if not pym:
        # This is for finding a previously created instance, not finding an
        # installed module with the same name. Might need to base the name
        # on the actual paths to reduce ambiguity in the printed scan results.
        module = os.path.basename(filename)
        pym = utils.find_package(module, installed)
        if not pym:
            pym = PyModule(module, 'SCRIPT', os.path.abspath(filename))
            installed.insert(0, pym)
        else:
            pym.is_scan = True

    for imp_type, import_path, extra_file_scan in _scan_file(filename, sentinel):
        dep = utils.find_package(import_path, installed)
        if dep:
            dep.add_dependant(pym)
            pym.add_dependency(dep)

            if imp_type != 'import':
                pym.add_framework(imp_type)

        if extra_file_scan:
            # extra_file_scan should be a directory or file containing the
            # import name
            scan_filename = utils.file_containing_import(import_path, extra_file_scan)
            log.info('Related scan: %s - %s', import_path, scan_filename)
            if scan_filename.endswith('__init__.py'):
                scan_directory(pym, os.path.dirname(scan_filename), sentinel, installed)
            else:
                scan_file(pym, scan_filename, sentinel, installed)

    return pym
872860
def execute(self, input_data):

        '''  '''
        raw_bytes = input_data['sample']['raw_bytes']

        # Have the PE File module process the file
        pefile_handle, error_str = self.open_using_pefile('unknown', raw_bytes)
        if not pefile_handle:
            return {'error': error_str, 'dense_features': [], 'sparse_features': []}

        # Now extract the various features using pefile
        dense_features, sparse_features = self.extract_features_using_pefile(pefile_handle)

        # Okay set my response
        return {'dense_features': dense_features, 'sparse_features': sparse_features, 'tags': input_data['tags']['tags']}
961937
def scannerData(self, reqId, rank, contractDetails, distance, benchmark, projection, legsStr):
        """"""
        return _swigibpy.EWrapper_scannerData(self, reqId, rank, contractDetails, distance, benchmark, projection, legsStr)
85490
def cmdline_params(self, surface_sample_file_name, cell_file_name):
        
        parameters = []

        parameters += [surface_sample_file_name]
        parameters += [cell_file_name]
        parameters += [self._OUTPUT_FILE_NAME]

        return map(str, parameters)
465246
def _scanDataFiles(self, dataDir, patterns):
        
        numDataFiles = 0
        for pattern in patterns:
            scanPath = os.path.join(dataDir, pattern)
            for filename in glob.glob(scanPath):
                self._addDataFile(filename)
                numDataFiles += 1
        if numDataFiles == 0:
            raise exceptions.EmptyDirException(dataDir, patterns)
92689
def _get_preseq_params(data, preseq_cmd, read_count):
    
    defaults = {
        'seg_len': 100000,        # maximum segment length when merging paired end bam reads
        'steps': 300,             # number of points on the plot
        'extrap_fraction': 3,     # extrapolate up to X times read_count
        'extrap': None,           # extrapolate up to X reads
        'step': None,             # step size (number of reads between points on the plot)
        'options': '',
    }
    params = {}

    main_opts = [("-e", "-extrap"), ("-l", "-seg_len"), ("-s", "-step")]
    other_opts = config_utils.get_resources("preseq", data["config"]).get("options", [])
    if isinstance(other_opts, str):
        other_opts = [other_opts]
    for sht, lng in main_opts:
        if sht in other_opts:
            i = other_opts.index(sht)
        elif lng in other_opts:
            i = other_opts.index(lng)
        else:
            i = None
        if i is not None:
            params[lng[1:]] = other_opts[i + 1]
            other_opts = other_opts[:i] + other_opts[i + 2:]
    params['options'] = ' '.join(other_opts)
    for k, v in config_utils.get_resources("preseq", data["config"]).items():
        if k != 'options':
            params[k] = v

    params['steps'] = params.get('steps', defaults['steps'])

    if preseq_cmd == 'c_curve':
        params['extrap_fraction'] = 1

    else:
        if params.get('step') is None:
            if params.get('extrap') is None:
                unrounded__extrap = read_count * params.get('extrap_fraction', defaults['extrap_fraction'])
                unrounded__step = unrounded__extrap // params['steps']
                if params.get('extrap_fraction') is not None:  # extrap_fraction explicitly provided
                    params['extrap'] = unrounded__extrap
                    params['step'] = unrounded__step
                else:
                    power_of_10 = 10 ** math.floor(math.log(unrounded__step, 10))
                    rounded__step = int(math.floor(unrounded__step // power_of_10) * power_of_10)
                    rounded__extrap = int(rounded__step) * params['steps']
                    params['step'] = rounded__step
                    params['extrap'] = rounded__extrap
            else:
                params['step'] = params['extrap'] // params['steps']

        elif params.get('extrap') is None:
            params['extrap'] = params['step'] * params['steps']

    params['step'] = params.get('step', defaults['step'])
    params['extrap'] = params.get('extrap', defaults['extrap'])
    params['seg_len'] = params.get('seg_len', defaults['seg_len'])

    logger.info("Preseq: running {steps} steps of size {step}, extap limit {extrap}".format(**params))
    return params
671554
def _get_input_args(bam_file, data, out_base, background):
    
    if dd.get_genome_build(data) in ["hg19"]:
        return ["--PileupFile", _create_pileup(bam_file, data, out_base, background)]
    else:
        return ["--BamFile", bam_file]