process processes all the files with clang and extracts all relevant
        nodes from the generated AST
765229
def process(self):
        

        self.index = cindex.Index.create()
        self.headers = {}

        for f in self.files:
            if f in self.processed:
                continue

            print('Processing {0}'.format(os.path.basename(f)))

            tu = self.index.parse(f, self.flags)

            if len(tu.diagnostics) != 0:
                fatal = False

                for d in tu.diagnostics:
                    sys.stderr.write(d.format())
                    sys.stderr.write("\n")

                    if d.severity == cindex.Diagnostic.Fatal or \
                       d.severity == cindex.Diagnostic.Error:
                        fatal = True

                if fatal:
                    sys.stderr.write("\nCould not generate documentation due to parser errors\n")
                    sys.exit(1)

            if not tu:
                sys.stderr.write("Could not parse file %s...\n" % (f,))
                sys.exit(1)

            # Extract comments from files and included files that we are
            # supposed to inspect
            extractfiles = [f]

            for inc in tu.get_includes():
                filename = str(inc.include)
                self.headers[filename] = True

                if filename in self.processed or (not filename in self.files) or filename in extractfiles:
                    continue

                extractfiles.append(filename)

            for e in extractfiles:
                db = comment.CommentsDatabase(e, tu)

                self.add_categories(db.category_names)
                self.commentsdbs[e] = db

            self.visit(tu.cursor.get_children())

            for f in self.processing:
                self.processed[f] = True

            self.processing = {}

        # Construct hierarchy of nodes.
        for node in self.all_nodes:
            q = node.qid

            if node.parent is None:
                par = self.find_parent(node)

                # Lookup categories for things in the root
                if (par is None or par == self.root) and (not node.cursor is None):
                    location = node.cursor.extent.start
                    db = self.commentsdbs[location.file.name]

                    if db:
                        par = self.category_to_node[db.lookup_category(location)]

                if par is None:
                    par = self.root

                par.append(node)

            # Resolve comment
            cm = self.find_node_comment(node)

            if cm:
                node.merge_comment(cm)

        # Keep track of classes to resolve bases and subclasses
        classes = {}

        # Map final qid to node
        for node in self.all_nodes:
            q = node.qid
            self.qid_to_node[q] = node

            if isinstance(node, nodes.Class):
                classes[q] = node

        # Resolve bases and subclasses
        for qid in classes:
            classes[qid].resolve_bases(classes)
639717
function processFile() {
  return (file, callback) => {
    let parsed;
    if (file.meta !== undefined && file.meta.layout !== undefined) {
      const ext = path.extname(file.meta.layout);
      parsed = path.parse(`${trimExtension(file.path)}${ext}`);
    } else {
      parsed = path.parse(file.path);
    }

    const filePath = {
      dir: parsed.dir,
      name: parsed.name,
      ext: parsed.ext,
    };
    const meta = Object.assign({}, file.meta, { filePath });
    async.nextTick(callback, null, Object.assign({}, file, { meta }));
  };
}
105245
function start() {
    console.log(
      'Using ts-node version',
      tsNodeVersion + ', typescript version',
      tsVersion
    )
    var cmd = nodeArgs.concat(wrapper, script, scriptArgs)
    var childHookPath = compiler.getChildHookPath()
    cmd = ['-r', childHookPath].concat(cmd)
    log.debug('Starting child process %s', cmd.join(' '))
    child = fork(cmd[0], cmd.slice(1), {
      cwd: process.cwd(),
      env: process.env
    })
    starting = false
    var compileReqWatcher = filewatcher({ forcePolling: opts.poll })
    var currentCompilePath
    fs.writeFileSync(compiler.getCompileReqFilePath(), '')
    compileReqWatcher.add(compiler.getCompileReqFilePath())
    compileReqWatcher.on('change', function(file) {
      fs.readFile(file, 'utf-8', function(err, data) {
        if (err) {
          log.error('Error reading compile request file', err)
          return
        }
        var split = data.split('\n')
        var compile = split[0]
        var compiledPath = split[1]
        if (currentCompilePath == compiledPath) return
        currentCompilePath = compiledPath
        // console.log('compileReqWatcher file change', compile);
        if (compiledPath) {
          compiler.compile({
            compile: compile,
            compiledPath: compiledPath
          })
        }
      })
    })
    child.on('message', function(message) {
      if (!message.compiledPath || currentCompilePath === message.compiledPath)
        return
      currentCompilePath = message.compiledPath
      compiler.compile(message)
    })

    child.on('exit', function(code) {
      log.debug('Child exited with code %s', code)
      if (!child) return
      if (!child.respawn) process.exit(code)
      child = undefined
    })

    if (cfg.respawn) {
      child.respawn = true
    }

    if (compiler.tsConfigPath) {
      watcher.add(compiler.tsConfigPath)
    }

    // Listen for `required` messages and watch the required file.
    ipc.on(child, 'required', function(m) {
      var isIgnored =
        cfg.ignore.some(isPrefixOf(m.required)) ||
        cfg.ignore.some(isRegExpMatch(m.required))

      if (!isIgnored && (cfg.deps === -1 || getLevel(m.required) <= cfg.deps)) {
        watcher.add(m.required)
      }
    })

    // Upon errors, display a notification and tell the child to exit.
    ipc.on(child, 'error', function(m) {
      notify(m.error, m.message, 'error')
      stop(m.willTerminate)
    })
    compiler.writeReadyFile()
  }
251675
function extractFiles(args) {
  var usage = '\n' +
    cyan('USAGE:') + '   hulk [--wrapper wrapper] [--outputdir outputdir] ' +
    '[--namespace namespace] [--variable variable] FILES\n\n' +
    cyan('OPTIONS:') + ' [-w, --wrapper]   :: wraps the template (i.e. amd)\n' +
    '         [-o,  --outputdir] :: outputs the templates as individual files to a directory\n\n' +
    '         [-n,  --namespace] :: prepend string to template names\n\n' +
    '         [-vn, --variable]  :: variable name for non-amd wrapper\n\n' +
    cyan('EXAMPLE:') + ' hulk --wrapper amd ./templates/*.mustache\n\n' +
    cyan('NOTE:') + '    hulk supports the "*" wildcard and allows you to target specific extensions too\n';
  var files = [];

  if (options.version) {
    console.log(require('../package.json').version);
    process.exit(0);
  }

  if (!args.length || options.help) {
    console.log(usage);
    process.exit(0);
  }

  args.forEach(function(arg) {
    if (/\*/.test(arg)) {
      arg = arg.split('*');
      files = files.concat(
        fs.readdirSync(arg[0] || '.')
          .map(function(f) {
            var file = path.join(arg[0], f);
            return new RegExp(esc(arg[1]) + '$').test(f) && fs.statSync(file).isFile() && file;
          })
          .filter(function(f) {
            return f;
          })
      );
      return files;
    }

    if (fs.statSync(arg).isFile()) files.push(arg);
  });

  return files;
}
722442
private void process( Collection<File> templateFiles ) {

		// Compile them all
		Collection<TemplateEntry> templateEntries = new ArrayList<> ();
		for( File f : templateFiles ) {
			final TemplateEntry templateEntry = compileTemplate( f );
			if( templateEntry != null )
				templateEntries.add( templateEntry );
		}

		// Add all the template entries
		this.lock.writeLock().lock();
		try {
			for( final TemplateEntry te : templateEntries )
				this.fileToTemplate.put( te.getTemplateFile(), te );

		} finally {
			this.lock.writeLock().unlock();
		}

		// Notify the templating manager
		this.manager.processNewTemplates( templateEntries );
	}
440978
function( src, fileName ) {
        var ast = parse(src, fileName);
        walk.simple(ast, {
            ExpressionStatement: function( node ) {
                if( node.expression.type !== 'CallExpression' ) {
                    return;
                }

                var start = node.start;
                var end = node.end;
                node = node.expression;
                var callee = node.callee;
                if( callee.name === "CONSTANT" &&
                    callee.type === "Identifier" ) {

                    if( node.arguments.length !== 2 ) {
                        throw new Error( "Exactly 2 arguments must be passed to CONSTANT\n" +
                            src.substring(start, end)
                        );
                    }

                    if( node.arguments[0].type !== "Identifier" ) {
                        throw new Error( "Can only define identifier as a constant\n" +
                            src.substring(start, end)
                        );
                    }

                    var args = node.arguments;

                    var name = args[0];
                    var nameStr = name.name;
                    var expr = args[1];

                    var e = eval;
                    constants[nameStr] = {
                        identifier: name,
                        value: e(nodeToString(expr))
                    };
                    walk.simple( expr, {
                        Identifier: function( node ) {
                            ignore.push(node);
                        }
                    });
                    global[nameStr] = constants[nameStr].value;
                }
            }
        });
    }
169198
def process_files(self):
        
        file_names = []
        for f in self.files:
            file_names.append(f.process_file())

        if not self.has_thumbnail() and config.THUMBNAILS:
            file_names.append(self.derive_thumbnail())

        # node_file = NodeFile(self.to_dict())
        # self.hashed_file_name = node_file.process_file()
        # file_names.append(self.hashed_file_name)

        return file_names
795561
function _getNodeCompiler(
  nodeFileDir,
  nodeConfigureArgs,
  nodeMakeArgs,
  nodeVCBuildArgs,
  complete,
  type,
) {
  var dir = _getFirstDirectory(nodeFileDir);

  // standard
  var executable = 'node.exe';
  var binary = 'node';

  // iojs specifics.
  if (framework === 'iojs') {
    executable = 'iojs.exe';
    binary = 'iojs';
  }

  if (dir) {
    if (isWin) {
      complete(null, {
        dir: dir,
        version: path.basename(nodeFileDir),
        releasePath: path.join(dir, 'Release', executable),
        make: function(next) {
          // create a new env with minimal impact on old one
          var newEnv = process.env;

          if (isPy !== 'python') {
            // add the dir of the suposed python exe to path
            newEnv.path = process.env.PATH + ';' + path.dirname(isPy);
          }

          if (!nodeVCBuildArgs || !nodeVCBuildArgs.length) {
            nodeVCBuildArgs = ['nosign']; // "release" is already the default config value in VCBuild.bat
          }

          // spawn a vcbuild process with our custom enviroment.
          var vcbuild = spawn('vcbuild.bat', nodeVCBuildArgs, {
            cwd: dir,
            env: newEnv,
          });
          vcbuild.stdout.pipe(process.stdout);
          vcbuild.stderr.pipe(process.stderr);
          vcbuild.on('close', function() {
            next();
          });
        },
      });
    } else {
      complete(null, {
        dir: dir,
        version: path.basename(nodeFileDir),
        releasePath: path.join(dir, 'out', 'Release', binary),
        make: function(next) {
          var cfg = './configure',
            configure;

          var conf = [cfg];

          /* add all nodeConfigureArgs to the ./configure argument array */
          conf = [conf].concat(nodeConfigureArgs);

          // should work for all use cases now.
          configure = spawn(isPy, conf, {
            cwd: dir.toString('ascii'),
          });

          // local function, move to top eventually
          function _loop(dir) {
            /* eventually try every python file */
            var pdir = fs.readdirSync(dir);

            pdir.forEach(function(v, i) {
              var stat = fs.statSync(dir + '/' + v);
              if (stat.isFile()) {
                // only process Makefiles and .mk targets.
                if (v !== 'Makefile' && path.extname(v) !== '.mk') {
                  return;
                }

                _log('patching ' + v);

                /* patch the file */
                var py = fs.readFileSync(dir + '/' + v, {
                  encoding: 'utf8',
                });
                py = py.replace(/([a-z]|\/)*python(\w|)/gm, isPy); // this is definently needed
                fs.writeFileSync(dir + '/' + v, py, {
                  encoding: 'utf8',
                }); // write to file
              } else if (stat.isDirectory()) {
                // must be dir?
                // skip tests because we don't need them here
                if (v !== 'test') {
                  _loop(dir + '/' + v);
                }
              }
            });
          }

          configure.stdout.pipe(process.stdout);
          configure.stderr.pipe(process.stderr);

          // on error
          configure.on('error', function(err) {
            console.log('Error:', err);
            console.log('');
            console.log('Details:');
            console.log('command =', isPy, cfg, conf_args || '');
            console.log('cwd =', dir);

            var configure_path = path.join(dir, 'configure');
            var contains_configure = fs.existsSync(configure_path);

            console.log(
              'cwd contains configure,',
              contains_configure ? colors.green('yes') : colors.red('no'),
            );

            var configure_size = fs.statSync(configure_path).size;

            console.log(
              'configure is non-zero size,',
              configure_size > 0 ? colors.green('yes') : colors.red('no'),
            );

            _log('error', 'failed to launch configure.');
            process.exit(1);
          });

          // when it's finished
          configure.on('close', function() {
            if (isPy !== 'python') {
              /**
               * Originally I thought this only applied to io.js,
               * however I soon found out this affects node.js,
               * so it is now mainstream.
               */
              _log('preparing python');

              // loop over depends
              _loop(dir);
            }

            if (nodeMakeArgs === undefined) {
              nodeMakeArgs = [];
            }

            var platformMake = 'make';
            if (os.platform().match(/bsd$/) != null) {
              platformMake = 'gmake';
            }

            var make = spawn(platformMake, nodeMakeArgs, {
              cwd: dir,
            });
            make.stdout.pipe(process.stdout);
            make.stderr.pipe(process.stderr);
            make.on('error', function(err) {
              console.log(err);
              _log('error', 'failed to run make.');
              process.exit(1);
            });
            make.on('close', function() {
              next();
            });
          });
        },
      });
    }
    return true;
  }

  return false;
}
404488
private void processFiles() {
        String[] fileArgs = getOptionValues(INFILE_SHORT_OPT);
        final List<File> files = new ArrayList<File>(fileArgs.length);
        for (final String fileArg : fileArgs) {
            File file = new File(fileArg);
            if (!file.canRead()) {
                error(INPUT_FILE_UNREADABLE + file);
            } else {
                files.add(file);
            }
        }

        if (files.size() == 0) {
            error(NO_DOCUMENT_FILES);
            failUsage();
        }

        // Create the directory artifact or fail
        artifactPath = createDirectoryArtifact(outputDirectory, DIR_ARTIFACT);
        processFiles(files.toArray(new File[0]));
    }
1163716
def process_all_json_files(build_dir):
    """"""
    html_files = []
    for root, _, files in os.walk(build_dir):
        for filename in fnmatch.filter(files, '*.fjson'):
            if filename in ['search.fjson', 'genindex.fjson',
                            'py-modindex.fjson']:
                continue
            html_files.append(os.path.join(root, filename))
    page_list = []
    for filename in html_files:
        try:
            result = process_file(filename)
            if result:
                page_list.append(result)
        # we're unsure which exceptions can be raised
        except:  # noqa
            pass
    return page_list
1159555
def _walk_src(self, targets, flags, pipeline):
        """"""

        for target in targets:
            for f in glob.iglob(target, flags=flags | glob.S):
                if not os.path.isdir(f):
                    self.log('', 2)
                    self.log('> Processing: %s' % f, 1)
                    if pipeline:
                        try:
                            yield pipeline[0]._run_first(f)
                        except Exception as e:
                            err = self.get_error(e)
                            yield [filters.SourceText('', f, '', '', err)]
                    else:
                        try:
                            if self.default_encoding:
                                encoding = filters.PYTHON_ENCODING_NAMES.get(
                                    self.default_encoding, self.default_encoding
                                ).lower()
                                encoding = codecs.lookup(encoding).name
                            else:
                                encoding = self.default_encoding
                            yield [filters.SourceText('', f, encoding, 'file')]
                        except Exception as e:
                            err = self.get_error(e)
                            yield [filters.SourceText('', f, '', '', err)]
752546
def main():
    """"""
    desc = 'Generate files to benchmark'
    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument(
        '--src',
        dest='src_dir',
        default='src',
        help='The directory containing the templates'
    )
    parser.add_argument(
        '--out',
        dest='out_dir',
        default='generated',
        help='The output directory'
    )
    parser.add_argument(
        '--seed',
        dest='seed',
        default='13',
        help='The random seed (to ensure consistent regeneration)'
    )

    args = parser.parse_args()

    random.seed(int(args.seed))

    mkdir_p(args.out_dir)

    for template in templates_in(args.src_dir):
        modes = template.modes()

        n_range = template.range()
        for n_value in n_range:
            base = template.instantiate(n_value)
            for mode in modes:
                write_file(
                    os.path.join(
                        args.out_dir,
                        out_filename(template, n_value, mode)
                    ),
                    mode.convert_from(base)
                )
        write_file(
            os.path.join(args.out_dir, '{0}.json'.format(template.name)),
            json.dumps({
                'files': {
                    n: {
                        m.identifier: out_filename(template, n, m)
                        for m in modes
                    } for n in n_range
                },
                'name': template.name,
                'x_axis_label': template.property('x_axis_label'),
                'desc': template.property('desc'),
                'modes': {m.identifier: m.description() for m in modes}
            })
        )
856145
def _process_file(self):
        ''''''
        print 'Processing file'
        with open(self._rebase_file, 'r') as f:
            raw = f.readlines()
        names = [line.strip()[3:] for line in raw if line.startswith('<1>')]
        seqs = [line.strip()[3:] for line in raw if line.startswith('<5>')]
        if len(names) != len(seqs):
            raise Exception('Found different number of enzyme names and '
                            'sequences.')
        self._enzyme_dict = {}
        for name, seq in zip(names, seqs):
            if '?' in seq:
                # Is unknown sequence, don't keep it
                pass
            elif seq.startswith('(') and seq.endswith(')'):
                # Has four+ cut sites, don't keep it
                pass
            elif '^' in seq:
                # Has reasonable internal cut sites, keep it
                top_cut = seq.index('^')
                bottom_cut = len(seq) - top_cut - 1
                site = seq.replace('^', '')
                self._enzyme_dict[name] = (site, (top_cut, bottom_cut))
            elif seq.endswith(')'):
                # Has reasonable external cut sites, keep it
                # (4-cutter also starts with '(')
                # separate site and cut locations
                site, cuts = seq.split('(')
                cuts = cuts.replace(')', '')
                top_cut, bottom_cut = [int(x) + len(site) for x in
                                       cuts.split('/')]
                self._enzyme_dict[name] = (site, (top_cut, bottom_cut))
        shutil.rmtree(self._tmpdir)
926026
def _process_extends(self, node, **kwargs):
        

        # find all the blocks in this template
        for b in self.ast.find_all(nodes.Block):

            # if not already in `child_blocks` then this is the first time a
            # block with this name has been encountered.
            if b.name not in self.child_blocks:
                self.child_blocks[b.name] = b
            else:

                # otherwise we have seen this block before, so we need to find the last
                # super_block and add the block from this template to the end.
                block = self.child_blocks.get(b.name)
                while hasattr(block, 'super_block'):
                    block = block.super_block
                block.super_block = b

        # load the parent template
        parent_template = JinjaToJS(template_root=self.template_root,
                                    template_name=node.template.value,
                                    js_module_format=self.js_module_format,
                                    runtime_path=self.runtime_path,
                                    include_prefix=self.include_prefix,
                                    include_ext=self.include_ext,
                                    child_blocks=self.child_blocks,
                                    dependencies=self.dependencies)

        # add the parent templates output to the current output
        self.output.write(parent_template.output.getvalue())

        # Raise an exception so we stop parsing this template
        raise ExtendsException
1319859
function processCodePathToEnter(analyzer, node) {
    let codePath = analyzer.codePath;
    let state = codePath && CodePath.getState(codePath);
    const parent = node.parent;

    switch (node.type) {
        case "Program":
        case "FunctionDeclaration":
        case "FunctionExpression":
        case "ArrowFunctionExpression":
            if (codePath) {

                // Emits onCodePathSegmentStart events if updated.
                forwardCurrentToHead(analyzer, node);
                debug.dumpState(node, state, false);
            }

            // Create the code path of this scope.
            codePath = analyzer.codePath = new CodePath(
                analyzer.idGenerator.next(),
                codePath,
                analyzer.onLooped
            );
            state = CodePath.getState(codePath);

            // Emits onCodePathStart events.
            debug.dump(`onCodePathStart ${codePath.id}`);
            analyzer.emitter.emit("onCodePathStart", codePath, node);
            break;

        case "LogicalExpression":
            if (isHandledLogicalOperator(node.operator)) {
                state.pushChoiceContext(
                    node.operator,
                    isForkingByTrueOrFalse(node)
                );
            }
            break;

        case "ConditionalExpression":
        case "IfStatement":
            state.pushChoiceContext("test", false);
            break;

        case "SwitchStatement":
            state.pushSwitchContext(
                node.cases.some(isCaseNode),
                astUtils.getLabel(node)
            );
            break;

        case "TryStatement":
            state.pushTryContext(Boolean(node.finalizer));
            break;

        case "SwitchCase":

            /*
             * Fork if this node is after the 2st node in `cases`.
             * It's similar to `else` blocks.
             * The next `test` node is processed in this path.
             */
            if (parent.discriminant !== node && parent.cases[0] !== node) {
                state.forkPath();
            }
            break;

        case "WhileStatement":
        case "DoWhileStatement":
        case "ForStatement":
        case "ForInStatement":
        case "ForOfStatement":
            state.pushLoopContext(node.type, astUtils.getLabel(node));
            break;

        case "LabeledStatement":
            if (!astUtils.isBreakableStatement(node.body)) {
                state.pushBreakContext(false, node.label.name);
            }
            break;

        default:
            break;
    }

    // Emits onCodePathSegmentStart events if updated.
    forwardCurrentToHead(analyzer, node);
    debug.dumpState(node, state, false);
}
602828
public void process() {
        processOverview();
        for ( ClassDoc doc : rootDoc.classes() ) {
            packages.add(doc.containingPackage());
            processClass(doc);
        }
        for ( PackageDoc doc : packages ) {
            processPackage(doc);
        }
    }
638247
protected function _processTree($file, $nodes, $coverage, $path = '')
    {
        foreach ($nodes as $node) {
            $this->_processNode($file, $node, $coverage, $path);
        }
    }