Runs the extractor on a single file and return processed source.
512268
def extract_source_from_file(file_path, env = {})
      source = HtmlMockup::Template.open(file_path, :partials_path => self.project.partial_path, :layouts_path => self.project.layouts_path).render(env.dup)

      if @options[:url_relativize]
        source = relativize_urls(source, file_path)
      end
      
      source
    end
979710
def _run(self):
        # type: (Downloader) -> None
        
        # mark start
        self._start_time = blobxfer.util.datetime_now()
        logger.info('blobxfer start time: {0}'.format(self._start_time))
        # ensure destination path
        blobxfer.operations.download.Downloader.ensure_local_destination(
            self._creds, self._spec, self._general_options.dry_run)
        logger.info('downloading blobs/files to local path: {}'.format(
            self._spec.destination.path))
        self._catalog_local_files_for_deletion()
        # initialize resume db if specified
        if self._general_options.resume_file is not None:
            self._resume = blobxfer.operations.resume.DownloadResumeManager(
                self._general_options.resume_file)
        # initialize MD5 processes
        if (self._spec.options.check_file_md5 and
                self._general_options.concurrency.md5_processes > 0):
            self._md5_offload = blobxfer.operations.md5.LocalFileMd5Offload(
                num_workers=self._general_options.concurrency.md5_processes)
            self._md5_offload.initialize_check_thread(
                self._check_for_downloads_from_md5)
        # initialize crypto processes
        if self._general_options.concurrency.crypto_processes > 0:
            self._crypto_offload = blobxfer.operations.crypto.CryptoOffload(
                num_workers=self._general_options.concurrency.crypto_processes)
            self._crypto_offload.initialize_check_thread(
                self._check_for_crypto_done)
        # initialize download threads
        self._initialize_transfer_threads()
        self._initialize_disk_threads()
        # initialize local counters
        files_processed = 0
        skipped_files = 0
        skipped_size = 0
        # iterate through source paths to download
        for src in self._spec.sources:
            for rfile in src.files(
                    self._creds, self._spec.options,
                    self._general_options.dry_run):
                # form local path for remote file
                if (not self._spec.destination.is_dir and
                        self._spec.options.rename):
                    lpath = pathlib.Path(self._spec.destination.path)
                else:
                    lpath = None
                    if self._spec.options.strip_components > 0:
                        _lparts = pathlib.Path(rfile.name).parts
                        _strip = min(
                            (len(_lparts) - 1,
                             self._spec.options.strip_components)
                        )
                        if _strip > 0:
                            lpath = pathlib.Path(*_lparts[_strip:])
                    if lpath is None:
                        lpath = pathlib.Path(rfile.name)
                    lpath = pathlib.Path(self._spec.destination.path) / lpath
                files_processed += 1
                # check on download conditions
                action = self._check_download_conditions(lpath, rfile)
                # remove from delete after set
                try:
                    self._delete_after.remove(lpath)
                except KeyError:
                    pass
                if action == DownloadAction.Skip:
                    skipped_files += 1
                    skipped_size += rfile.size
                    if self._general_options.dry_run:
                        logger.info('[DRY RUN] skipping: {} -> {}'.format(
                            lpath, rfile.path))
                    continue
                # add potential download to set
                dlid = (
                    blobxfer.operations.download.Downloader.
                    create_unique_transfer_operation_id(rfile)
                )
                with self._transfer_lock:
                    self._transfer_set.add(dlid)
                    self._download_total += 1
                    self._download_bytes_total += rfile.size
                # either MD5 check or download now
                if action == DownloadAction.CheckMd5:
                    self._pre_md5_skip_on_check(lpath, rfile)
                elif action == DownloadAction.Download:
                    if self._general_options.dry_run:
                        logger.info(
                            '[DRY RUN] download: {} -> {}'.format(
                                rfile.path, lpath))
                        with self._transfer_lock:
                            self._transfer_set.remove(dlid)
                            self._download_total -= 1
                            self._download_bytes_total -= rfile.size
                    else:
                        self._add_to_download_queue(lpath, rfile)
        # set remote files processed
        with self._md5_meta_lock:
            self._all_remote_files_processed = True
        with self._transfer_lock:
            download_size_mib = (
                self._download_bytes_total / blobxfer.util.MEGABYTE
            )
            logger.debug(
                ('{0} files {1:.4f} MiB filesize and/or lmt_ge '
                 'skipped').format(
                    skipped_files, skipped_size / blobxfer.util.MEGABYTE))
            logger.debug(
                ('{0} remote files processed, waiting for download '
                 'completion of approx. {1:.4f} MiB').format(
                     files_processed, download_size_mib))
        del files_processed
        del skipped_files
        del skipped_size
        # wait for downloads to complete
        self._wait_for_transfer_threads(terminate=False)
        self._wait_for_disk_threads(terminate=False)
        end_time = blobxfer.util.datetime_now()
        # update progress bar
        self._update_progress_bar()
        # check for exceptions
        if len(self._exceptions) > 0:
            logger.error('exceptions encountered while downloading')
            # raise the first one
            raise self._exceptions[0]
        # check for mismatches
        if (self._download_sofar != self._download_total or
                self._download_bytes_sofar != self._download_bytes_total):
            raise RuntimeError(
                'download mismatch: [count={}/{} bytes={}/{}]'.format(
                    self._download_sofar, self._download_total,
                    self._download_bytes_sofar, self._download_bytes_total))
        # delete all remaining local files not accounted for if
        # delete extraneous enabled
        self._delete_extraneous_files()
        # delete resume file if we've gotten this far
        if self._resume is not None:
            self._resume.delete()
        # output throughput
        if self._download_start_time is not None:
            dltime = (end_time - self._download_start_time).total_seconds()
            if dltime == 0:  # noqa
                dltime = 1e-9
            download_size_mib = (
                self._download_bytes_total / blobxfer.util.MEGABYTE
            )
            dlmibspeed = download_size_mib / dltime
            logger.info(
                ('elapsed download + verify time and throughput of {0:.4f} '
                 'GiB: {1:.3f} sec, {2:.4f} Mbps ({3:.3f} MiB/sec)').format(
                     download_size_mib / 1024, dltime, dlmibspeed * 8,
                     dlmibspeed))
        end_time = blobxfer.util.datetime_now()
        logger.info('blobxfer end time: {0} (elapsed: {1:.3f} sec)'.format(
            end_time, (end_time - self._start_time).total_seconds()))
203182
def _run_scapy(self, scapy_all):
        """"""
        # if not self._flag_a:
        #     self._flag_a = True
        #     warnings.warn(f"'Extractor(engine=scapy)' object is not iterable; "
        #                     "so 'auto=False' will be ignored", AttributeWarning, stacklevel=stacklevel())

        if self._exlyr != 'None' or self._exptl != 'null':
            warnings.warn("'Extractor(engine=scapy)' does not support protocol and layer threshold; "
                          f"'layer={self._exlyr}' and 'protocol={self._exptl}' ignored",
                          AttributeWarning, stacklevel=stacklevel())

        # extract & analyse file
        self._expkg = scapy_all
        self._extmp = iter(scapy_all.sniff(offline=self._ifnm))

        # start iteration
        self.record_frames()
738126
function extract(sources, options) {
		return sources
			.map(function(src) {
				return [src, grunt.file.read(src)];
			})
			.filter(function(list) {
				return syntax.valid(grunt, list[0], list[1]);
			})
			.map(function(list) {
				return extras(list[0], list[1], options);
			});
	}
405633
public void extract()
    {
        ExecutorService executor = Executors.newFixedThreadPool(this.extractorConfig.getThreadPoolSize());
        try
        {
            Map<Pair<RelatedFinder, Object>, List<MithraDataObject>> extract = extractData(executor);
            writeToFiles(executor, extract);
        }
        finally
        {
            executor.shutdown();
            try
            {
                executor.awaitTermination(extractorConfig.getTimeoutSeconds(), TimeUnit.SECONDS);
            }
            catch (InterruptedException e)
            {
                //
            }
        }
    }
461163
def _run_server(self, multiprocessing):
        """"""
        if not self._flag_m:
            raise UnsupportedCall(f"Extractor(engine={self._exeng})' has no attribute '_run_server'")

        if not self._flag_q:
            self._flag_q = True
            warnings.warn("'Extractor(engine=pipeline)' does not support output; "
                          f"'fout={self._ofnm}' ignored", AttributeWarning, stacklevel=stacklevel())

        self._frnum = 1                                                 # frame number (revised)
        self._expkg = multiprocessing                                   # multiprocessing module
        self._mpsvc = NotImplemented                                    # multiprocessing server process
        self._mpprc = list()                                            # multiprocessing process list
        self._mpfdp = collections.defaultdict(multiprocessing.Queue)    # multiprocessing file pointer

        self._mpmng = multiprocessing.Manager()                         # multiprocessing manager
        self._mpbuf = self._mpmng.dict()                                # multiprocessing frame dict
        self._mpfrm = self._mpmng.list()                                # multiprocessing frame storage
        self._mprsm = self._mpmng.list()                                # multiprocessing reassembly buffer

        self._mpkit = self._mpmng.Namespace()                           # multiprocessing work kit
        self._mpkit.counter = 0                                         # work count (on duty)
        self._mpkit.pool = 1                                            # work pool (ready)
        self._mpkit.eof = False                                         # EOF flag
        self._mpkit.trace = None                                        # flow tracer

        # preparation
        self.record_header()
        self._mpfdp[0].put(self._gbhdr.length)
        self._mpsvc = multiprocessing.Process(
            target=self._server_analyse_frame,
            kwargs={'mpfrm': self._mpfrm, 'mprsm': self._mprsm, 'mpbuf': self._mpbuf, 'mpkit': self._mpkit}
        )
        self._mpsvc.start()

        # extraction
        while True:
            # check EOF
            if self._mpkit.eof:
                self._update_eof()
                break

            # check counter
            if self._mpkit.pool and self._mpkit.counter < CPU_CNT - 1:
                # update file offset
                self._ifile.seek(self._mpfdp.pop(self._frnum-1).get(), os.SEEK_SET)

                # create worker
                # print(self._frnum, 'start')
                proc = multiprocessing.Process(
                        target=self._server_extract_frame,
                        kwargs={'mpkit': self._mpkit, 'mpbuf': self._mpbuf, 'mpfdp': self._mpfdp[self._frnum]}
                )

                # update status
                self._mpkit.pool -= 1
                self._mpkit.counter += 1

                # start and record
                proc.start()
                self._frnum += 1
                self._mpprc.append(proc)

            # check buffer
            if len(self._mpprc) >= CPU_CNT - 1:
                [proc.join() for proc in self._mpprc[:-4]]
                del self._mpprc[:-4]
83226
def extract(self, target):
        

        if not foundations.common.path_exists(self.__archive):
            raise foundations.exceptions.FileExistsError("{0} | '{1}' file doesn't exists!".format(
                self.__class__.__name__, self.__archive))

        if not foundations.common.path_exists(target):
            raise foundations.exceptions.DirectoryExistsError("{0} | '{1}' directory doesn't exists!".format(
                self.__class__.__name__, target))

        archive = zipfile.ZipFile(self.__archive)
        content = archive.namelist()

        directories = [item for item in content if item.endswith("/")]
        files = [item for item in content if not item.endswith("/")]

        directories.sort()
        directories.reverse()

        for directory in directories:
            not os.path.isdir(os.path.join(target, directory)) and foundations.io.set_directory(
                os.path.join(target, directory))

        for file in files:
            LOGGER.info("{0} | Extracting '{1}' file!".format(self.__class__.__name__, file))
            with open(os.path.join(target, file), "w") as output:
                buffer = StringIO(archive.read(file))
                bufferSize = 2 ** 20
                data = buffer.read(bufferSize)
                while data:
                    output.write(data)
                    data = buffer.read(bufferSize)
        return True
904854
def parse_file(self, cu, analysis):
        """"""
        if hasattr(analysis, 'parser'):
            filename = cu.file_locator.relative_filename(cu.filename)
            source_lines = analysis.parser.lines
            with cu.source_file() as source_file:
                source = source_file.read()

            try:
                if sys.version_info < (3, 0):
                    encoding = source_encoding(source)
                    if encoding != 'utf-8':
                        source = source.decode(encoding).encode('utf-8')
            except UnicodeDecodeError:
                log.warning(
                    'Source file %s can not be properly decoded, skipping. '
                    'Please check if encoding declaration is ok',
                    os.path.basename(cu.filename))
                return
        else:
            if hasattr(cu, 'relative_filename'):
                filename = cu.relative_filename()
            else:
                filename = analysis.coverage.file_locator.relative_filename(
                    cu.filename)

            token_lines = analysis.file_reporter.source_token_lines()
            source_lines = list(enumerate(token_lines))
            source = analysis.file_reporter.source()

        coverage_lines = [self.get_hits(i, analysis)
                          for i in range(1, len(source_lines) + 1)]

        # ensure results are properly merged between platforms
        posix_filename = filename.replace(os.path.sep, '/')

        results = {
            'name': posix_filename,
            'source': source,
            'coverage': coverage_lines,
        }

        branches = self.get_arcs(analysis)
        if branches:
            results['branches'] = branches

        self.source_files.append(results)
1330684
def run(*args):
    """"""

    if not args:
        args = sys.argv[1:]

    if len(args) < 2:
        print('Usage: runenv <envfile> <command> <params>')
        sys.exit(0)
    os.environ.update(create_env(args[0]))
    os.environ['_RUNENV_WRAPPED'] = '1'
    runnable_path = args[1]

    if not runnable_path.startswith(('/', '.')):
        runnable_path = spawn.find_executable(runnable_path)

    try:
        if not(stat.S_IXUSR & os.stat(runnable_path)[stat.ST_MODE]):
            print('File `%s is not executable' % runnable_path)
            sys.exit(1)
        return subprocess.check_call(
            args[1:], env=os.environ
        )
    except subprocess.CalledProcessError as e:
        return e.returncode
817368
def run(args):
    
    html_content_extractor = HTMLContentExtractor()

    with warnings.catch_warnings():
        warnings.simplefilter('ignore')

        extractions = html_content_extractor.extract(html_text=args.input_file)
        for e in extractions:
            print(e.value)
141196
def run(args):
    
    regex_extractor = RegexExtractor(pattern=args.pattern)

    with warnings.catch_warnings():
        warnings.simplefilter('ignore')

        for line in args.input_file:
            extractions = regex_extractor.extract(line)
            for e in extractions:
                print(e.value)
653719
def _extract(archive, compression, cmd, format, verbosity, outdir):
    """"""
    targetname = util.get_single_outfile(outdir, archive)
    try:
        with lzma.LZMAFile(archive, **_get_lzma_options(format)) as lzmafile:
            with open(targetname, 'wb') as targetfile:
                data = lzmafile.read(READ_SIZE_BYTES)
                while data:
                    targetfile.write(data)
                    data = lzmafile.read(READ_SIZE_BYTES)
    except Exception as err:
        msg = "error extracting %s to %s: %s" % (archive, targetname, err)
        raise util.PatoolError(msg)
    return None
592108
public function extract()
	{
		$this->reader->read($this->file);

		$this->converter
			 ->setExtension($this->extension)
			 ->process($this->reader->getData());
	}
862095
function Extractor(ac , options) {
    EventEmitter.call(this);
    var self = this;
    self.matchers = [];
    self.vars = ac || {};
    self.options = options || {};
    self.watchers = [];
    self._listen = function (car, file) {
        car.once('end', function () {
            self.emit('end', self.vars);
        });
        car.on('line', function (line) {
            var i;
            var firstMatch = true;
            for (i = 0; i < self.matchers.length; i++) {
                var matcher = self.matchers[i];
                var m;
                while(matcher.handler && (m = matcher.re.exec(line)) !== null){
                    matcher.handler(m, self.vars, file , firstMatch);
                    firstMatch = false;
                    if(!self.options.successive){
                        i = self.matchers.length;
                        break;
                    }
                }
            }
        });
        return self;
    };
}
218877
def extract(args):
    
    p = OptionParser(extract.__doc__)
    p.add_option("--format", default=False, action="store_true",
            help="enable flag to reformat header into a symbol separated list of constituent reads "+ \
            "[default: %default]")
    p.add_option("--singlets", default=False, action="store_true",
            help="ask the program to look in the singlets file (should be in the same folder) for " +\
            "unused reads and put them in the resultant fasta file [default: %default]")
    p.set_sep(sep="|", help="Separator used to list the reads in the FASTA header")
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    acefile, = args
    ace = Ace.read(must_open(acefile))
    logging.debug('Loaded ace file {0}'.format(acefile))

    fastafile = acefile.rsplit(".", 1)[0] + ".fasta"
    fw = open(fastafile, "w")
    for c in ace.contigs:
        id = c.name
        if opts.format:
           id = opts.sep.join([read.name for read in c.af])

        seqrec = SeqRecord(Seq(c.sequence), id=id, description="")
        SeqIO.write([seqrec], fw, "fasta")

    if opts.singlets:
        singletsfile = acefile.rsplit(".", 1)[0] + ".singlets"
        if os.path.getsize(singletsfile) > 0:
            fp = SeqIO.parse(must_open(singletsfile), "fasta")
            for rec in fp:
                SeqIO.write(rec, fw, "fasta")

    fw.close()
    logging.debug('Wrote contigs to fasta file {0}'.format(fastafile))
334749
def process_source(self):
        """"""
        if isinstance(self.source, str):
            # Replace single and double quotes with escaped single-quote
            self.source = self.source.replace("'", "\'").replace('"', "\'")
            return "\"{source}\"".format(source=self.source)
        return self.source
107111
def run(self, interpreter):
        
        with utils.ChangeDir(self.dirname):
            command_list = ['PYTHONPATH=' + main_dir, interpreter,
                            self.filename] + list(self.args)
            try:
                proc = Popen(' '.join(command_list), stdout=PIPE, stderr=PIPE,
                             shell=True)
                stream_data = proc.communicate()
            except Exception as e:
                logger.error(
                    "Error {0} while executing extract_dist command.".format(e))
                raise ExtractionError
            stream_data = [utils.console_to_str(s) for s in stream_data]
            if proc.returncode:
                logger.error(
                    "Subprocess failed, stdout: {0[0]}, stderr: {0[1]}".format(
                        stream_data))
            self._result = json.loads(stream_data[0].split(
                "extracted json data:\n")[-1].split("\n")[0])