Process a collection of ingested sources to make source tables.
760897
def source_schema(self, sources=None, tables=None, clean=False):
        

        sources = self._resolve_sources(sources, tables, None,
                                        predicate=lambda s: s.is_processable)

        for source in sources:
            source.update_table()
            self.log("Creating source schema for '{}': Table {},  {} columns"
                     .format(source.name, source.source_table.name, len(source.source_table.columns)))

        self.commit()
622203
def _ProcessGrepSource(self, source):
    """"""
    attributes = source.base_source.attributes
    paths = artifact_utils.InterpolateListKbAttributes(
        attributes["paths"], self.knowledge_base,
        self.ignore_interpolation_errors)
    regex = utils.RegexListDisjunction(attributes["content_regex_list"])
    condition = rdf_file_finder.FileFinderCondition.ContentsRegexMatch(
        regex=regex, mode="ALL_HITS")
    file_finder_action = rdf_file_finder.FileFinderAction.Stat()
    request = rdf_file_finder.FileFinderArgs(
        paths=paths,
        action=file_finder_action,
        conditions=[condition],
        follow_links=True)
    action = file_finder.FileFinderOSFromClient

    yield action, request
598380
def to_table_data(self):
        

        self._validate_source_data()

        for table_key, json_records in six.iteritems(self._buffer):
            attr_name_set = set()
            for json_record in json_records:
                attr_name_set = attr_name_set.union(six.viewkeys(json_record))
            headers = sorted(attr_name_set)

            self._loader.inc_table_count()
            self._table_key = table_key

            yield TableData(
                self._make_table_name(),
                headers,
                json_records,
                dp_extractor=self._loader.dp_extractor,
                type_hints=self._extract_type_hints(headers),
            )
506966
public void processSource(String[] filenames)
    {
        for (int i = 0; i != filenames.length; ++i) {
            String filename = filenames[i];
            if (!filename.endsWith(".js")) {
                addError("msg.extension.not.js", filename);
                return;
            }
            File f = new File(filename);
            String source = readSource(f);
            if (source == null) return;

            String mainClassName = targetName;
            if (mainClassName == null) {
                String name = f.getName();
                String nojs = name.substring(0, name.length() - 3);
                mainClassName = getClassName(nojs);
            }
            if (targetPackage.length() != 0) {
                mainClassName = targetPackage+"."+mainClassName;
            }

            Object[] compiled
                = compiler.compileToClassFiles(source, filename, 1,
                                               mainClassName);
            if (compiled == null || compiled.length == 0) {
                return;
            }

            File targetTopDir = null;
            if (destinationDir != null) {
                targetTopDir = new File(destinationDir);
            } else {
                String parent = f.getParent();
                if (parent != null) {
                    targetTopDir = new File(parent);
                }
            }
            for (int j = 0; j != compiled.length; j += 2) {
                String className = (String)compiled[j];
                byte[] bytes = (byte[])compiled[j + 1];
                File outfile = getOutputFile(targetTopDir, className);
                try {
                    FileOutputStream os = new FileOutputStream(outfile);
                    try {
                        os.write(bytes);
                    } finally {
                        os.close();
                    }
                } catch (IOException ioe) {
                    addFormatedError(ioe.toString());
                }
            }
        }
    }
531115
function sources(k) {
      (src[k] || []).forEach(function(s) { deps[s] = k; sources(s); });
    }
1162838
public void process( FastQueue<AssociatedIndex> matches , int numSource , int numDestination ) {

		if( checkSource ) {
			if( checkDestination ) {
				processSource(matches, numSource, firstPass);
				processDestination(firstPass,numDestination,pruned);
			} else {
				processSource(matches, numSource, pruned);
			}
		} else if( checkDestination ) {
			processDestination(matches,numDestination,pruned);
		} else {
			// well this was pointless, just return the input set
			pruned = matches;
		}
	}
134270
protected void processData(Collection<RI> data) {
        // Check data against all rules
        for (Rule<Collection<RI>, RO> rule : rules) {
            processResult(rule.validate(data));
        }
    }
430552
function process(sources, options) {
		sources = explodeall(sources);
		if (!grunt.fail.errorcount) {
			var content = extract(sources, options);
			return enclose(content.join(SPACER));
		}
	}
262933
def ingest_mp(b, stage, source_name, clean_files):
    """"""

    source = b.source(source_name)

    with b.progress.start('ingest_mp',0,message="MP ingestion", source=source) as ps:
        r =  b._ingest_source(source, ps, clean_files)

    return r
1176730
private void addSources( final Collection<File> items, final Collection<String> sourceRoots )
    {
        for ( String path : sourceRoots )
        {
            items.add( new File( path ) );
        }
    }
1125497
def calculate_sources(self, targets):
    """"""
    sources = set()
    for target in targets:
      sources.update(
        source for source in target.sources_relative_to_buildroot()
        if source.endswith(self._PYTHON_SOURCE_EXTENSION)
      )
    return sources
860813
private void processResourceRates()
   {
      List<Row> rows = getRows("rsrcrate", null, null);
      m_reader.processResourceRates(rows);
   }
932513
private void processDatabaseMappingCollection(final Object databaseMappingCollection) {
        if (databaseMappingCollection == null || !isCastable("java.util.Collection", databaseMappingCollection.getClass())) {
            return;
        }

        final Collection<?> c = (Collection<?>) databaseMappingCollection;
        for (Object descriptor : c) {
            processDatabaseMappingObject(descriptor);
        }
    }
712645
private void processResources() throws SQLException
   {
      List<Row> rows = getRows("select * from " + m_schema + "rsrc where delete_date is null and rsrc_id in (select rsrc_id from " + m_schema + "taskrsrc t where proj_id=? and delete_date is null) order by rsrc_seq_num", m_projectID);
      m_reader.processResources(rows);
   }
711665
private function processingSourceFiles(): void
  {
    $this->logVerbose('Replace references to resource files with references to optimized/minimized resource files.');

    foreach ($this->sourceFilesInfo as $source_filename)
    {
      $this->logVerbose("Processing %s.", $source_filename);

      $content = file_get_contents($source_filename);
      if ($content===false) $this->logError("Unable to read file '%s'.", $source_filename);

      $new_content = $content;

      if (strncmp($content, '<?php', 5)===0)
      {
        // Source is a PHP file.
        $new_content = $this->processPhpSourceFile($source_filename, $new_content);
      }

      $new_content = strtr($new_content, $this->replacePairs);

      if ($content!=$new_content)
      {
        $time = null;

        // If required determine the latest modification time of the source file and its referenced resource files.
        if ($this->preserveModificationTime)
        {
          $time = $this->getMaxModificationTime($source_filename, $new_content);
        }

        // Write sources file with modified references to resource files.
        $status = file_put_contents($source_filename, $new_content);
        if ($status===false) $this->logError("Updating file '%s' failed.", $source_filename);
        $this->logInfo("Updated file '%s'.", $source_filename);

        // If required set the mtime to the latest modification time of the source file and its referenced resource
        // files.
        if ($this->preserveModificationTime)
        {
          $status = touch($source_filename, $time);
          if ($status===false) $this->logError("Unable to set mtime for file '%s'.", $source_filename);
        }
      }
    }
  }
508847
def build(self, sources=None, tables=None, stage=None, force=False):
        

        from operator import attrgetter
        from itertools import groupby
        from .concurrent import build_mp, unify_mp
        from ambry.bundle.events import TAG

        self.log('==== Building ====')
        self.state = self.STATES.BUILDING
        self.commit()

        class SourceSet(object):
            """Container for sources that can reload them after they get expired from the session"""
            def __init__(self, bundle, v):
                self.bundle = bundle
                self.sources = v
                self._s_vids = [s.vid for s in self.sources]

            def reload(self):
                self.sources = [self.bundle.source(vid) for vid in self._s_vids]

            def __iter__(self):
                for s in self.sources:
                    yield s

            def __len__(self):
                return len(self._s_vids)

        self._run_events(TAG.BEFORE_BUILD, 0)

        resolved_sources = SourceSet(self, self._resolve_sources(sources, tables, stage=stage,
                                                                 predicate=lambda s: s.is_processable))
        with self.progress.start('build', stage, item_total=len(resolved_sources)) as ps:

            if len(resolved_sources) == 0:
                ps.update(message='No sources', state='skipped')
                self.log('No processable sources, skipping build stage {}'.format(stage))
                return True

            if not self.pre_build(force):
                ps.update(message='Pre-build failed', state='skipped')
                return False

            if force:
                self._reset_build(resolved_sources)

            resolved_sources.reload()

            e = [
                (stage, SourceSet(self, list(stage_sources)))
                for stage, stage_sources in groupby(sorted(resolved_sources, key=attrgetter('stage')),
                                                    attrgetter('stage'))

                ]

            for stage, stage_sources in e:

                stage_sources.reload()

                for s in stage_sources:
                    s.state = self.STATES.WAITING
                self.commit()

                stage_sources.reload()

                self.log('Processing {} sources, stage {} ; first 10: {}'
                         .format(len(stage_sources), stage, [x.name for x in stage_sources.sources[:10]]))
                self._run_events(TAG.BEFORE_BUILD, stage)

                if self.multi:

                    try:
                        # The '1' for chunksize ensures that the subprocess only gets one
                        # source to build. Combined with maxchildspertask = 1 in the pool,
                        # each process will only handle one source before exiting.

                        args = [(self.identity.vid, stage, source.vid, force) for source in stage_sources]
                        pool = self.library.process_pool(limited_run=self.limited_run)
                        r = pool.map_async(build_mp, args, 1)
                        completed_sources = r.get()

                        ps.add('Finished MP building {} sources. Starting MP coalescing'
                               .format(len(completed_sources)))

                        partition_names = [(self.identity.vid, k) for k, v
                                           in self.collect_segment_partitions().items()]

                        r = pool.map_async(unify_mp, partition_names, 1)

                        completed_partitions = r.get()

                        ps.add('Finished MP coalescing {} partitions'.format(len(completed_partitions)))

                        pool.close()
                        pool.join()

                    except KeyboardInterrupt:
                        self.log('Got keyboard interrrupt; terminating workers')
                        pool.terminate()

                else:

                    for i, source in enumerate(stage_sources):
                        id_ = ps.add(message='Running source {}'.format(source.name),
                                     source=source, item_count=i, state='running')

                        self.build_source(stage, source, ps, force=force)

                        ps.update(message='Finished processing source', state='done')

                        # This bit seems to solve a problem where the records from the ps.add above
                        # never gets closed out.
                        ps.get(id_).state = 'done'
                        self.progress.commit()

                    self.unify_partitions()

                self._run_events(TAG.AFTER_BUILD, stage)

        self.state = self.STATES.BUILT
        self.commit()

        self._run_events(TAG.AFTER_BUILD, 0)

        self.close_session()

        self.log('==== Done Building ====')
        self.buildstate.commit()
        self.state = self.STATES.BUILT
        self.commit()
        return True
867949
def build_source(self, stage, source, ps, force=False):
        """"""
        from ambry.bundle.process import call_interval

        assert source.is_processable, source.name

        if source.state == self.STATES.BUILT and not force:
            ps.update(message='Source {} already built'.format(source.name), state='skipped')
            return

        pl = self.pipeline(source, ps=ps)

        source.state = self.STATES.BUILDING

        # Doing this before hand to get at least some information about the pipline,
        # in case there is an error during the run. It will get overwritten with more information
        # after successful run
        self.log_pipeline(pl)

        try:

            source_name = source.name  # In case the source drops out of the session, which is does.
            s_vid = source.vid

            ps.update(message='Running pipeline {}'.format(pl.name), s_vid=s_vid, item_type='rows', item_count=0)

            @call_interval(5)
            def run_progress_f(sink_pipe, rows):
                (n_records, rate) = sink_pipe.report_progress()
                if n_records > 0:
                    ps.update(message='Running pipeline {}: rate: {}'
                              .format(pl.name, rate),
                              s_vid=s_vid, item_type='rows', item_count=n_records)


            pl.run(callback=run_progress_f)

            # Run the final routines at the end of the pipelin
            for f in pl.final:
                ps.update(message='Run final routine: {}'.format(f.__name__))
                f(pl)

            ps.update(message='Finished building source')

        except:
            self.log_pipeline(pl)
            raise

        self.commit()

        try:
            partitions = list(pl[ambry.etl.PartitionWriter].partitions)
            ps.update(message='Finalizing segment partition',
                      item_type='partitions', item_total=len(partitions), item_count=0)
            for i, p in enumerate(partitions):

                ps.update(message='Finalizing segment partition {}'.format(p.name), item_count=i, p_vid=p.vid)

                try:
                    p.finalize()
                except AttributeError:
                    print(self.table(p.table_name))
                    raise

                # FIXME Shouldn't need to do this commit, but without it, some stats get added multiple
                # times, causing an error later. Probably could be avoided by adding the stats to the
                # collection in the dataset

                self.commit()

        except IndexError:
            self.error("Pipeline didn't have a PartitionWriters, won't try to finalize")

        self.log_pipeline(pl)
        source.state = self.STATES.BUILT

        self.commit()

        return source.name