Returns the S3 location for reading a parse job. A wildcard is
 used for the segment names. The input filter depends on the output
 file type of the parse job and what type of compression is used.
16713
def set_input_path(input_job)
      job_name = input_job.job_name
      input_filter = job_config['input_filter']

      s3_path = "/data/1-parse/#{job_name}/segments/*/#{input_filter}"
      build_s3_uri(s3_path)
    end
774406
def get(self, url=None, delimiter="/"):
        
        params = {'Delimiter': delimiter}
        bucket, obj_key = _parse_url(url)

        if bucket:
            params['Bucket'] = bucket
        else:
            return self.call("ListBuckets", response_data_key="Buckets")

        if obj_key:
            params['Prefix'] = obj_key

        objects = self.call("ListObjects", response_data_key="Contents",
                            **params)
        if objects:
            for obj in objects:
                obj['url'] = "s3://{0}/{1}".format(bucket, obj['Key'])

        return objects
1012757
protected List<String> getFileLocationsFromSubsystemContent(String subsystemContent) {
        String sc = subsystemContent + ",";
        List<String> files = new ArrayList<String>();
        boolean isFile = false;
        String location = null;
        int strLen = sc.length();
        boolean quoted = false;
        for (int i = 0, pos = 0; i < strLen; i++) {
            char c = sc.charAt(i);
            if (!quoted && (c == ';' || c == ',')) {
                String str = sc.substring(pos, i);
                pos = i + 1;
                if (logger.isLoggable(Level.FINE)) {
                    logger.fine("element : " + str);
                }
                if (str.contains(":=")) {
                    if (getKey(str).equals(HDR_ATTR_LOCATION)) {
                        location = getValue(str);
                    }
                } else if (str.contains("=")) {
                    if (getKey(str).equals(HDR_ATTR_TYPE) && getValue(str).equals(HDR_ATTR_VALUE_FILE)) {
                        isFile = true;
                    }
                }
                if (c == ',') { // the delimiter of each subsystem content.
                    if (isFile && location != null) {
                        if (logger.isLoggable(Level.FINE)) {
                            logger.fine("location : " + location);
                        }
                        files.add(location);
                    }
                    location = null;
                    isFile = false;
                }
            } else if (c == '\"') {
                quoted = !quoted;
            }
        }
        return files;
    }
1143940
@Override
    public void setLocation(String location, Job job)
            throws IOException {
        // The location is assumed to be comma separated paths.
        FileInputFormat.setInputPaths(job, location);

        requiredFieldList = (RequiredFieldList) getFromUDFContext(PRUNE_PROJECTION_INFO);

        // If we encounter a PushDown Projection we strip the requestedFields to only the needed ones
        // This pruning will very effectively push the projection down into the actual parser system.
        if (requiredFieldList != null &&
            originalRequestedFields == null) { // Avoid pruning twice !!
            Set<Integer> requestedFieldIndexes = new HashSet<>();
            for (RequiredField requiredField : requiredFieldList.getFields()) {
                requestedFieldIndexes.add(requiredField.getIndex());
            }
            List<String> prunedRequestedFields = new ArrayList<>(requestedFieldIndexes.size());
            int index = 0;
            for (String field : requestedFields) {
                if (requestedFieldIndexes.contains(index)) {
                    prunedRequestedFields.add(field);
                }
                ++index;
            }
            originalRequestedFields = requestedFields;
            requestedFields = prunedRequestedFields;
        }
    }
951563
def get_s3_uri(m, file_names, config):
    
    bucket_name = m.group(1)
    path = m.group(2)
    bucket = get_s3_bucket(bucket_name, config)
    result = 0
    if (config.start_date or config.date_range) and ("{year" in path or "{month" in path or "{day" in path):
        for tmp_date in date_generator(config.end_date, config.date_range or ((config.end_date - config.start_date).days + 1)):
            # +1 because we want to include end_date
            for key in bucket.list(prefix=path.format(year=tmp_date.year, month=tmp_date.month, day=tmp_date.day)):
                file_names.append("s3://{}/{}".format(bucket_name, key.name))
                result += key.size
    else:
        for key in bucket.list(prefix=path):
            file_names.append("s3://{}/{}".format(bucket_name, key.name))
            result += key.size
    return result
363766
final void buildBloomFilterForBucket(int bucketInSegmentPos, MemorySegment bucket, HashPartition<BT, PT> p) {
		final int count = bucket.getShort(bucketInSegmentPos + HEADER_COUNT_OFFSET);
		if (count <= 0) {
			return;
		}

		int[] hashCodes = new int[count];
		// As the hashcode and bloom filter occupy same bytes, so we read all hashcode out at first and then write back to bloom filter.
		for (int i = 0; i < count; i++) {
			hashCodes[i] = bucket.getInt(bucketInSegmentPos + BUCKET_HEADER_LENGTH + i * HASH_CODE_LEN);
		}
		this.bloomFilter.setBitsLocation(bucket, bucketInSegmentPos + BUCKET_HEADER_LENGTH);
		for (int hashCode : hashCodes) {
			this.bloomFilter.addHash(hashCode);
		}
		buildBloomFilterForExtraOverflowSegments(bucketInSegmentPos, bucket, p);
	}
60405
private String fetchRemoteIfNecessary(String s) {
        if (s.startsWith("s3")) {
            String[] bucketAndKey = parseBucketKeyAndFileName(s);
            String bucket = bucketAndKey[0];
            String key = bucketAndKey[1];
            String fileName = bucketAndKey[2];

            log.info("KeyTool - fetching " + s);
            S3Object obj = s3Client.getObject(bucket, key);
            log.info("KeyTool - fetched " + s);

            quickState.put(s, "fetched");

            File f = new File(_keyDir, fileName);
            FileOutputStream os = null;
            log.info("KeyTool - output file " + f.getAbsolutePath());

            try {

                S3ObjectInputStream is = obj.getObjectContent();

                log.info("KeyTool - fetching " + s);

                os = new FileOutputStream(f);
                IOUtils.copy(is, os);

                log.info("local path now " + f.getAbsolutePath());
                return f.getAbsolutePath();

            } catch (Exception ex) {
                log.info("KeyTool - failed to write to " + f.getAbsolutePath());
                throw new RuntimeException(ex);
            } finally {
                if (os != null) {
                    IOUtils.closeQuietly(os);
                }
            }
        }

        return s;
    }
762754
private FileStatus[] findAllRaw(String segmentName) throws IOException {
        assert segmentName != null && segmentName.length() > 0 : "segmentName must be non-null and non-empty";
        String pattern = String.format(NAME_FORMAT, getPathPrefix(segmentName), SUFFIX_GLOB_REGEX);
        FileStatus[] files = this.fileSystem.globStatus(new Path(pattern));

        if (files.length > 1) {
            throw new IllegalArgumentException("More than one file");
        }
        return files;
    }
1164303
private void findUsefulDissectorsFromField(
            final Set<String> possibleTargets,
            final Set<String> locatedTargets,
            final String subRootType, final String subRootName,
            final boolean thisIsTheRoot) {

        String subRootId = subRootType + ':' + subRootName;

        // When we reach this point we have dissectors to get here.
        // So we store this to later validate if we have everything.
        if (locatedTargets.contains(subRootId)) {
            // We already found this one.
            return; // Avoid infinite recursion
        }
        locatedTargets.add(subRootId);

        LOG.debug("findUsefulDissectors:\"{}\" \"{}\"", subRootType, subRootName);

        for (DissectorPhase dissector: availableDissectors) {

            if (!(dissector.inputType.equals(subRootType))) {
                continue; // Wrong type
            }

            // If it starts with a . it extends.
            // If it doesn't then it starts at the beginning
            Set<String> checkFields = new HashSet<>();

            // If true then this dissector can output any name instead of just one
            boolean isWildCardDissector = dissector.name.equals("*");

            if (isWildCardDissector) {
                // Ok, this is special
                // We need to see if any of the wanted types start with the
                // subRootName (it may have a '.' in the rest of the line !)
                String subRootNameMatch = subRootName + '.';
                for (String possibleTarget : possibleTargets) {
                    if (possibleTarget.startsWith(subRootNameMatch)) {
                        checkFields.add(possibleTarget);
                    }
                }
            } else if (thisIsTheRoot) {
                checkFields.add(dissector.name);
            } else if (dissector.name.isEmpty()) {
                checkFields.add(subRootName);
            } else {
                checkFields.add(subRootName + '.' + dissector.name);
            }

            for (String checkField: checkFields) {
                if (possibleTargets.contains(checkField)
                    && !compiledDissectors.containsKey(dissector.outputType + ":" + checkField)) {

                    Set<DissectorPhase> subRootPhases = compiledDissectors.get(subRootId);
                    if (subRootPhases == null) {
                        // New so we can simply add it.
                        subRootPhases = new HashSet<>();
                        compiledDissectors.put(subRootId, subRootPhases);
                        usefulIntermediateFields.add(subRootName);
                    }

                    Class<? extends Dissector> clazz = dissector.instance.getClass();
                    DissectorPhase dissectorPhaseInstance = findDissectorInstance(subRootPhases, clazz);

                    if (dissectorPhaseInstance == null) {
                        dissectorPhaseInstance =
                                new DissectorPhase(dissector.inputType, dissector.outputType,
                                        checkField, dissector.instance.getNewInstance());
                        subRootPhases.add(dissectorPhaseInstance);
                    }

                    // Tell the dissector instance what to expect
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Informing : ({}){} --> {} --> ({}){}",
                                dissector.inputType, subRootName,
                                dissector.instance.getClass().getName(),
                                dissector.outputType, checkField);
                    }
                    castsOfTargets.put(dissector.outputType + ':' + checkField,
                            dissectorPhaseInstance.instance.prepareForDissect(subRootName, checkField));

                    // Recurse from this point down
                    findUsefulDissectorsFromField(possibleTargets, locatedTargets, dissector.outputType, checkField, false);
                }
            }
        }

        Set<String> mappings = typeRemappings.get(subRootName);
        if (mappings != null) {
            for (String mappedType : mappings) {
                if (!compiledDissectors.containsKey(mappedType + ':' + subRootName)) {
                    // Retyped targets are ALWAYS String ONLY.
                    castsOfTargets.put(mappedType + ':' + subRootName, STRING_ONLY);
                    findUsefulDissectorsFromField(possibleTargets, locatedTargets, mappedType, subRootName, false);
                }
            }
        }

    }
559087
def getcandsfile(d, segment=-1, domock=False):
    
    if domock:
        prefix = 'candsmock_'
    else:
        prefix= 'cands_'

    if d.has_key('segment'):
        return os.path.join(d['workdir'], prefix + d['fileroot'] + '_sc' + str(d['scan']) + 'seg' + str(d['segment']) + '.pkl')
    elif segment >= 0:
        return os.path.join(d['workdir'], prefix + d['fileroot'] + '_sc' + str(d['scan']) + 'seg' + str(segment) + '.pkl')
    else:
        return ''
139532
def load_wildcard(as = nil)
      seg = nesting_segments[enclosing_resources.size] or ResourcesController.raise_resource_mismatch(self)

      segment = seg[:segment]
      singleton = seg[:singleton]

      if resource_specification_map[segment]
        spec = resource_specification_map[segment]
        spec = spec.dup.tap {|s| s.as = as} if as
      else
        spec = Specification.new(singleton ? segment : segment.singularize, :singleton => singleton, :as => as)
      end
      load_enclosing_resource_from_specification(spec)
    end
239172
def parseArgv():
    
    parser = OptionParser()
    parser.usage = r""" cat <TEXT> | %prog [--unit <UNIT>] [--output <SA_FILE>]

Create the suffix array of TEXT with the processing UNIT and optionally store it in SA_FILE for subsequent use.
UNIT may be set to 'byte', 'character' (given an encoding with the --encoding option) or 'word', which is the default.
"""

    parser.add_option("-i", "--input",
                      action="store", type="string", dest="input",
                      default=False,
                      help="Path of the file containing the input text. When '-' is given, read the standard input (default). If the path ends with '.gz', reads the decompressed file.")

    parser.add_option("-o", "--output",
                      action="store", type="string", dest="output",
                      default=False,
                      help="Store the suffix array of the input to the file OUTPUT. When '-' is given, writes to the standard output. If the filename ends with '.gz', the suffix array will be stored  compressed.")

    parser.add_option("", "--load",
                      action="store", type="string", dest="SAFile",
                      default=False,
                      help="Load a suffix array from SAFILE, this option and --input are mutually exclusive.")

    parser.add_option("-u", "--unit",
                      action="store", type="string", dest="unit",
                      default=DEFAULT_UNIT_STR,
                      help="Processing unit used for the creation of the suffix array." + \
                           'Possible values are "byte", "character" and "word". Default is "%s".' % DEFAULT_UNIT_STR + \
                           "This option is ignored when the suffix array is loaded from SAFILE." + \
                           'For characters, the input is decoded according to the encoding set via the option --encoding.')

    parser.add_option("-e", "--encoding",
                      action="store", type="string", dest="encoding",
                      default=DEFAULT_ENCODING,
                      help="Encoding of the input. This information is required only when processing characters. Default is '%s'." % DEFAULT_ENCODING)

    parser.add_option("-p", "--print",
                      action="store_true", dest="printSA",
                      default=False,
                      help="Prints the suffix array in a human readable format to the standard error output.")

    parser.add_option("", "--verbose",
                      action="store_true", dest="verbose",
                      default=False,
                      help="Prints more information.")

    parser.add_option("", "--no-lcps",
                      action="store_true", dest="noLCPs",
                      default=False,
                      help="Switch off the computation of LCPs. By doing so, the find functions are unusable.")

    (options, args) = parser.parse_args(_argv)
    strings = args[1:]
    return (options, strings)
544979
@Override
  public String getStorageDir(DataSegment segment, boolean useUniquePath)
  {
    // This is only called by HdfsDataSegmentPusher.push(), which will always set useUniquePath to false since any
    // 'uniqueness' will be applied not to the directory but to the filename along with the shard number. This is done
    // to avoid performance issues due to excessive HDFS directories. Hence useUniquePath is ignored here and we
    // expect it to be false.
    Preconditions.checkArgument(
        !useUniquePath,
        "useUniquePath must be false for HdfsDataSegmentPusher.getStorageDir()"
    );

    return JOINER.join(
        segment.getDataSource(),
        StringUtils.format(
            "%s_%s",
            segment.getInterval().getStart().toString(ISODateTimeFormat.basicDateTime()),
            segment.getInterval().getEnd().toString(ISODateTimeFormat.basicDateTime())
        ),
        segment.getVersion().replace(':', '_')
    );
  }
1008024
public Iterator<StashTableMetadata> listTableMetadata() {
        final String root = getRootPath();
        final String prefix = String.format("%s/", root);
        final int prefixLength = prefix.length();

        return new AbstractIterator<StashTableMetadata>() {
            PeekingIterator<S3ObjectSummary> _listResponse =
                    Iterators.peekingIterator(Iterators.<S3ObjectSummary>emptyIterator());
            String _marker = null;
            boolean _truncated = true;

            @Override
            protected StashTableMetadata computeNext() {
                String tableDir = null;
                List<StashFileMetadata> files = Lists.newArrayListWithCapacity(16);
                boolean allFilesRead = false;

                while (!allFilesRead) {
                    if (_listResponse.hasNext()) {
                        // Peek at the next record but don't consume it until we verify it's part of the same table
                        S3ObjectSummary s3File = _listResponse.peek();
                        String key = s3File.getKey();

                        // Don't include the _SUCCESS file or any other stray files we may find
                        String[] parentDirAndFile = key.substring(prefixLength).split("/");
                        if (parentDirAndFile.length != 2) {
                            // Consume and skip this row
                            _listResponse.next();
                        } else {
                            String parentDir = parentDirAndFile[0];
                            if (tableDir == null) {
                                tableDir = parentDir;
                            }

                            if (!parentDir.equals(tableDir)) {
                                allFilesRead = true;
                            } else {
                                // Record is part of this table; consume it now
                                _listResponse.next();
                                files.add(new StashFileMetadata(_bucket, key, s3File.getSize()));
                            }
                        }
                    } else if (_truncated) {
                        ObjectListing response = _s3.listObjects(new ListObjectsRequest()
                                .withBucketName(_bucket)
                                .withPrefix(prefix)
                                .withMarker(_marker)
                                .withMaxKeys(1000));

                        _listResponse = Iterators.peekingIterator(response.getObjectSummaries().iterator());
                        _marker = response.getNextMarker();
                        _truncated = response.isTruncated();
                    } else {
                        allFilesRead = true;
                    }
                }

                if (tableDir == null) {
                    // No files read this iteration means all files have been read
                    return endOfData();
                }

                String tablePrefix = prefix + tableDir + "/";
                String tableName = StashUtil.decodeStashTable(tableDir);
                return new StashTableMetadata(_bucket, tablePrefix, tableName, files);
            }
        };
    }
1228476
private static void getparseArticle() throws Exception {
        // Instantiate a CLAVIN GeoParser using the StanfordExtractor
        GeoParser parser = GeoParserFactory.getDefault("./IndexDirectory", new StanfordExtractor(), 1, 1, false);
        
        // Unstructured text file about Somalia to be geoparsed
        File inputFile = new File("src/test/resources/sample-docs/Somalia-doc.txt");
        
        // Grab the contents of the text file as a String
        String inputString = TextUtils.fileToString(inputFile);
        
        // Parse location names in the text into geographic entities
        List<ResolvedLocation> resolvedLocations = parser.parse(inputString);
        
        // Display the ResolvedLocations found for the location names
        for (ResolvedLocation resolvedLocation : resolvedLocations)
            System.out.println(resolvedLocation);
        
    }
1298590
def parse_remote(cls, filename):
        
        parts = filename.split("//")[-1].split("/", 1)
        bucket, key = parts if len(parts) == 2 else (parts[0], None)
        if bucket.find("@") > 0:
            bucket, region = bucket.split("@")
        else:
            region = None

        return cls._REMOTE_FILE("s3", bucket, key, region)
1094137
def get(cls, filename):
        
        path = os.path.join(SparkFiles.getRootDirectory(), filename)
        return os.path.abspath(path)