/*
Extract all common WARC fields into a CaptureSearchResult. This is the
same for all WARC record types:

file, offset, timestamp, digest, urlKey, originalUrl
353393
private CaptureSearchResult genericResult(WARCRecord rec) {

		CaptureSearchResult result = new CaptureSearchResult();

		result.setMimeType(DEFAULT_VALUE);
		result.setHttpCode(DEFAULT_VALUE);
		result.setRedirectUrl(DEFAULT_VALUE);

		ArchiveRecordHeader header = rec.getHeader();

		String file = transformWARCFilename(header.getReaderIdentifier());
		long offset = header.getOffset();
		
		result.setCaptureTimestamp(transformWARCDate(header.getDate()));
		result.setFile(file);
		result.setOffset(offset);
		result.setDigest(transformWARCDigest(header.getHeaderValue(
				WARCRecord.HEADER_KEY_PAYLOAD_DIGEST)));
		
		String origUrl = header.getUrl();
		if(origUrl == null) {
			String type = header.getHeaderValue(WARCConstants.HEADER_KEY_TYPE).toString();
			if(type.equals(WARCConstants.WARCRecordType.warcinfo)) {
				String filename = header.getHeaderValue(
						WARCConstants.HEADER_KEY_FILENAME).toString();
				result.setOriginalUrl("filedesc:"+filename);
				result.setUrlKey("filedesc:"+filename);				
			} else {
				result.setOriginalUrl(DEFAULT_VALUE);
				result.setUrlKey(DEFAULT_VALUE);
			}

			
		} else {
			result.setOriginalUrl(origUrl);
			try {
				String urlKey = canonicalizer.urlStringToKey(origUrl);
				result.setUrlKey(urlKey);
			} catch (URIException e) {
				String shortUrl = 
					(origUrl.length() < 100) 
					? origUrl
					:origUrl.substring(0,100);
				LOGGER.warning("FAILED canonicalize(" + shortUrl + "):" + 
						file + " " + offset);
				result.setUrlKey(origUrl);
			}
		}
		return result;
	}
1208138
def write_record(self, record):
        ''''''
        # FIXME: probably not a good idea to modifiy arguments passed to us
        # TODO: add extra gzip headers that wget uses
        record.fields['WARC-Warcinfo-ID'] = self._warcinfo_record.fields[
            WARCRecord.WARC_RECORD_ID]

        _logger.debug('Writing WARC record {0}.',
                      record.fields['WARC-Type'])

        if self._params.compress:
            open_func = gzip.GzipFile
        else:
            open_func = open

        # Use getsize to get actual file size. Avoid tell() because it may
        # not be the raw file position.
        if os.path.exists(self._warc_filename):
            before_offset = os.path.getsize(self._warc_filename)
        else:
            before_offset = 0

        journal_filename = self._warc_filename + '-wpullinc'

        with open(journal_filename, 'w') as file:
            file.write('wpull-journal-version:1\n')
            file.write('offset:{}\n'.format(before_offset))

        try:
            with open_func(self._warc_filename, mode='ab') as out_file:
                for data in record:
                    out_file.write(data)
        except (OSError, IOError) as error:
            _logger.info(
                _('Rolling back file {filename} to length {length}.'),
                filename=self._warc_filename, length=before_offset
            )
            with open(self._warc_filename, mode='wb') as out_file:
                out_file.truncate(before_offset)

            raise error
        finally:
            os.remove(journal_filename)

        after_offset = os.path.getsize(self._warc_filename)

        if self._cdx_filename:
            raw_file_offset = before_offset
            raw_file_record_size = after_offset - before_offset

            self._write_cdx_field(
                record, raw_file_record_size, raw_file_offset
            )
201479
private Table<URI, URI, MatchResult> obtainMatchResults(Set<URI> origins, MatchType minType, MatchType maxType) {

        log.debug("Obtain match results for {}, with {} <= Match Result <= {}", origins, minType, maxType);

        Table<URI, URI, MatchResult> result = HashBasedTable.create();
        // Exit fast if no data is provided or no matches can be found
        if (origins == null || origins.isEmpty() || minType.compareTo(maxType) > 0)
            return result;

        // Create the query
        String queryStr = new StringBuffer()
                .append(generateQueryHeader())
                .append(generateRangeMatchWhereClause(origins, minType, maxType))
                .append(generateQueryFooter())
                .toString();

        log.debug("SPARQL Query generated: \n {}", queryStr);

        // Query the engine
        Query query = QueryFactory.create(queryStr);
        QueryExecution qe = QueryExecutionFactory.sparqlService(this.sparqlEndpoint.toASCIIString(), query);
        MonitoredQueryExecution qexec = new MonitoredQueryExecution(qe);
        try {
            Stopwatch stopwatch = new Stopwatch().start();
            ResultSet qResults = qexec.execSelect();
            stopwatch.stop();
            log.debug("Obtained matches for {} concepts within range {} - {} in {}", origins.size(), minType, maxType, stopwatch);

            Resource origin;
            Resource destination;
            URI originUri;
            URI matchUri;
            int index = 0;
            // Iterate over the results obtained starting with the matches for class0 onwards
            while (qResults.hasNext()) {
                QuerySolution soln = qResults.nextSolution();
                origin = soln.getResource(ORIGIN_VAR);
                destination = soln.getResource(MATCH_VAR + index);

                if (origin != null && origin.isURIResource() && destination != null && destination.isURIResource()) {
                    originUri = new URI(origin.getURI());
                    matchUri = new URI(destination.getURI());
                    MatchType type = getMatchType(soln);
                    result.put(originUri, matchUri, new AtomicMatchResult(originUri, matchUri, type, this));
                    log.debug("Concept {} was matched to {} with type {}", originUri, matchUri, type);
                } else {
                    log.warn("Skipping result as some URI is null: Origin - {}, Destination - {}", origin, destination);
                    break;
                }
            }
        } catch (URISyntaxException e) {
            log.error("Error obtaining match result. Expected a correct URI", e);
        } finally {
            qexec.close();
        }
        if (minType.compareTo(LogicConceptMatchType.Exact) <= 0 && maxType.compareTo(LogicConceptMatchType.Exact) >= 0) {
            for (URI o : origins) {
                result.put(o, o, new AtomicMatchResult(o, o, LogicConceptMatchType.Exact, this));
            }

        }
        return result;

    }
366732
protected WARCRecord createArchiveRecord(InputStream is, long offset)
    throws IOException {
        return (WARCRecord)currentRecord(new WARCRecord(is,
        	getReaderIdentifier(), offset, isDigest(), isStrict()));
    }
167927
function extractBinaryFields(target, spec, data) {
  var offset = 0;

  for (var x = 0; x < spec.length; x++) {
    var item = spec[x];
    if (item.label) {
      if (item.type === 'string') {
        var bytes = new DataView(data.buffer, offset, item.size);
        var str = textDecoder.decode(bytes);
        target[item.label] = str.replace(/\0/g, '');
      } else {
        target[item.label] = data['get' + item.type + (item.size * 8)](offset);
      }
    }
    offset += item.size;
  }

  return offset;
}
1082237
function (items, query, fields, exact) {
        var self = this;
        var isTarget = function (fieldName) {
          return fields.indexOf(fieldName) !== -1;
        };
        if (query === '') {
          return items;
        }
        fields = fields || ['name', 'owner', 'description', 'keyword'];
        return _.filter(items, function (item) {
          if ((isTarget('name') && self.matchedInString(query, item.name, exact)) ||
              (isTarget('owner') && self.matchedInString(query, item.owner, exact)) ||
              (isTarget('description') && self.matchedInString(query, item.description, exact)) ||
              (isTarget('keyword') && self.matchedInArray(query, item.keywords, exact))) {
            return true;
          }
          return false;
        });
      }
704346
public int filterObject(CaptureSearchResult r) {
		String captureDate = r.getCaptureTimestamp();
		String url = r.getOriginalUrl();
		return isBlocked(url,captureDate) ?	FILTER_EXCLUDE : FILTER_INCLUDE;
	}
486299
public int filterObject(CaptureSearchResult r) {
		recordsScanned++;
		if(recordsScanned > maxRecordsToScan) {
			LOGGER.warning("Hit max results on " + r.getUrlKey() + " " 
					+ r.getCaptureTimestamp());
			return FILTER_ABORT;
		}
		return FILTER_INCLUDE;
	}
2352
def extract_multiple(record, field, tag)
     a = Array.new
     record.fields(field).each do |field|
       a.push field[tag]
     end
     return a
   end
1940
@Override
    public org.fcrepo.server.types.gen.FieldSearchResult resumeFindObjects(String sessionToken) {
        MessageContext ctx = context.getMessageContext();
        Context context = ReadOnlyContext.getSoapContext(ctx);
        assertInitialized();
        try {
            org.fcrepo.server.search.FieldSearchResult result =
                    m_access.resumeFindObjects(context, sessionToken);
            return TypeUtility
                    .convertFieldSearchResultToGenFieldSearchResult(result);
        } catch (Throwable th) {
            LOG.error("Error resuming finding objects", th);
            throw CXFUtility.getFault(th);
        }
    }
784561
public CaptureSearchResult adapt(ARCRecord rec) {
		try {
			return adaptInner(rec);
		} catch (IOException e) {
			e.printStackTrace();
			return null;
		}
	}
539333
def _load_different_url_payload(self, cdx, headers_record,
                                    failed_files, cdx_loader):
        

        ref_target_uri = (headers_record.rec_headers.
                          get_header('WARC-Refers-To-Target-URI'))

        target_uri = headers_record.rec_headers.get_header('WARC-Target-URI')

        # if no target uri, no way to find the original
        if not ref_target_uri:
            raise ArchiveLoadFailed(self.MISSING_REVISIT_MSG)

        ref_target_date = (headers_record.rec_headers.
                           get_header('WARC-Refers-To-Date'))

        if not ref_target_date:
            ref_target_date = cdx['timestamp']
        else:
            ref_target_date = iso_date_to_timestamp(ref_target_date)

        digest = cdx.get('digest', '-')

        try:
            orig_cdx_lines = self.load_cdx_for_dupe(ref_target_uri,
                                                    ref_target_date,
                                                    digest,
                                                    cdx_loader)
        except NotFoundException:
            raise ArchiveLoadFailed(self.MISSING_REVISIT_MSG)

        for orig_cdx in orig_cdx_lines:
            try:
                payload_record = self._resolve_path_load(orig_cdx, False,
                                                         failed_files)
                return payload_record

            except ArchiveLoadFailed as e:
                pass

        raise ArchiveLoadFailed(self.MISSING_REVISIT_MSG)
1233172
def extractFieldsFromResult(data):
    '''
        
    ''' 
    
    entities = []
    
    # Defining the objects to extract
    fieldsRegExp = {}
    fieldsRegExp["i3visio.fullname"] = "<span class=\"fn\">([^<]*)</span>"
    fieldsRegExp["i3visio.name"] = "    por <strong>[^ ]* ([^<]*)</strong>"        
    fieldsRegExp["i3visio.surname"] = "    por <strong>([^ ]*) "     
    fieldsRegExp["i3visio.location.address"] = "itemprop=\"streetAddress\">([^<]*)</span>"
    fieldsRegExp["i3visio.location.city"] = "addressLocality\">([^<]*)</span>"
    fieldsRegExp["i3visio.location.postalcode"] = "postalCode\">([^<]*)</span>"    
    fieldsRegExp["i3visio.phone"] = "document.write\('([0-9]+)'"        

    for field in fieldsRegExp.keys():
        listRecovered = re.findall(fieldsRegExp[field], data)
        if len(listRecovered) >0:
            aux = {}
            aux["type"]= field
            aux["value"] = listRecovered[0].replace('\xa0', ' ')
            aux["attributes"] = []
            entities.append(aux)
            
    return entities
1176437
def fieldgetter(*fieldspecs):
    
    pattern = r'(?P<field>[^.]+)(.(?P<subfield>[^.]+))?'

    def fields(record):
        for s in fieldspecs:
            match = re.match(pattern, s)
            if not match:
                continue
            grp = match.groupdict()
            for field in record.get_fields(grp['field']):
                if grp['subfield']:
                    for value in field.get_subfields(grp['subfield']):
                        yield field, value
                else:
                    if int(grp['field']) < 10:
                        yield field, field.value()
                    else:
                        for value in field.subfields[1::2]:
                            yield field, value
    fields.__doc__ = 'returns a field generator over %s' % (
        ', '.join(fieldspecs))
    return fields
707160
function processRetrievedContent(content, filePrismicMetadata, queryKey, ctx) {
            if (content.results != null && content.results.length > 0) {
                var queryMetadata = filePrismicMetadata[queryKey];
                for (var i = 0; i < content.results.length; i++) {
                    var result = processRetrievedDocument(content.results[i], filePrismicMetadata, queryKey, ctx);
                    filePrismicMetadata[queryKey].results.push(result);
                }
            }
        }
611281
function getImportResult(extractions, url, prev, includedFilesMap, includedPaths) {
  const absolutePath = getImportAbsolutePath(url, prev, includedFilesMap, includedPaths);
  const contents = extractions[absolutePath].injectedData;

  return { file: absolutePath, contents };
}
778479
def get_field_cache(self, cache_type='es'):
        """"""
        if cache_type == 'kibana':
            try:
                search_results = urlopen(self.get_url).read().decode('utf-8')
            except HTTPError:  # as e:
                # self.pr_err("get_field_cache(kibana), HTTPError: %s" % e)
                return []
            index_pattern = json.loads(search_results)
            # Results look like: {"_index":".kibana","_type":"index-pattern","_id":"aaa*","_version":6,"found":true,"_source":{"title":"aaa*","fields":"<what we want>"}}  # noqa
            fields_str = index_pattern['_source']['fields']
            return json.loads(fields_str)
        elif cache_type == 'es' or cache_type.startswith('elastic'):
            search_results = urlopen(self.es_get_url).read().decode('utf-8')
            es_mappings = json.loads(search_results)
            # Results look like: {"<index_name>":{"mappings":{"<doc_type>":{"<field_name>":{"full_name":"<field_name>","mapping":{"<sub-field_name>":{"type":"date","index_name":"<sub-field_name>","boost":1.0,"index":"not_analyzed","store":false,"doc_values":false,"term_vector":"no","norms":{"enabled":false},"index_options":"docs","index_analyzer":"_date/16","search_analyzer":"_date/max","postings_format":"default","doc_values_format":"default","similarity":"default","fielddata":{},"ignore_malformed":false,"coerce":true,"precision_step":16,"format":"dateOptionalTime","null_value":null,"include_in_all":false,"numeric_resolution":"milliseconds","locale":""}}},  # noqa
            # now convert the mappings into the .kibana format
            field_cache = []
            for (index_name, val) in iteritems(es_mappings):
                if index_name != self.index:  # only get non-'.kibana' indices
                    # self.pr_dbg("index: %s" % index_name)
                    m_dict = es_mappings[index_name]['mappings']
                    # self.pr_dbg('m_dict %s' % m_dict)
                    mappings = self.get_index_mappings(m_dict)
                    # self.pr_dbg('mappings %s' % mappings)
                    field_cache.extend(mappings)
            field_cache = self.dedup_field_cache(field_cache)
            return field_cache
        self.pr_err("Unknown cache type: %s" % cache_type)
        return None