Constructs and returns all records in the expansion.
@throws IOException The name or rdata of a record was invalid after
substitutions were performed.
768568
public Record []
expand() throws IOException {
	List list = new ArrayList();
	for (long i = start; i < end; i += step) {
		String namestr = substitute(namePattern, current);
		Name name = Name.fromString(namestr, origin);
		String rdata = substitute(rdataPattern, current);
		list.add(Record.fromString(name, type, dclass, ttl,
					   rdata, origin));
	}
	return (Record []) list.toArray(new Record[list.size()]);
}
68756
private void init() 
            throws IOException, InterruptedException {
        
        // get job config properties
        boolean advancedMode = 
            conf.get(INPUT_MODE, BASIC_MODE).equals(ADVANCED_MODE);
        redactionRuleCol = conf.getStrings(REDACTION_RULE_COLLECTION);
        
        // initialize the total length
        float recToFragRatio = conf.getFloat(RECORD_TO_FRAGMENT_RATIO, 
                getDefaultRatio());
        length = mlSplit.getLength() * recToFragRatio;
        
        // generate the query
        String queryLanguage = null;
        String queryText = null;
        long start = mlSplit.getStart() + 1 + count;
        long end = mlSplit.isLastSplit() ? 
                   Long.MAX_VALUE : start + mlSplit.getLength() - count - 1;
        if (!advancedMode) { 
            StringBuilder buf = new StringBuilder();      
            buf.append("xquery version \"1.0-ml\"; \n");
            if (redactionRuleCol != null) {
            	buf.append(
            		"import module namespace rdt = \"http://marklogic.com/xdmp/redaction\" at \"/MarkLogic/redaction.xqy\";\n");
            }
            String indent = conf.get(INDENTED, "FALSE");
            Indentation ind = Indentation.valueOf(indent);
            buf.append(
                    "declare namespace mlmr=\"http://marklogic.com/hadoop\";\n");
            buf.append(
                    "declare variable $mlmr:splitstart as xs:integer external;\n");
            buf.append(
                    "declare variable $mlmr:splitend as xs:integer external;\n");
           
            buf.append(ind.getStatement());
            buf.append("\n");
            String docExpr = conf.get(DOCUMENT_SELECTOR);
            String ctsQuery = conf.get(QUERY_FILTER);
            String subExpr = conf.get(SUBDOCUMENT_EXPRESSION);
            Collection<String> nsCol = docExpr != null || subExpr != null? 
                    conf.getStringCollection(PATH_NAMESPACE) : null;
            if (ctsQuery != null) {
                buildSearchQuery(docExpr, ctsQuery, nsCol, buf);
                queryText = buf.toString();
            } else if (docExpr != null || subExpr != null) {   
                buildDocExprQuery(docExpr, nsCol, subExpr, buf);
                queryText = buf.toString();
            } else {
                LexiconFunction function = null;
                Class<? extends LexiconFunction> lexiconClass = 
                        conf.getClass(INPUT_LEXICON_FUNCTION_CLASS, null,
                                LexiconFunction.class);
                if (lexiconClass != null) {
                    function = ReflectionUtils.newInstance(lexiconClass, 
                            conf);
                    if (nsCol == null) {
                        nsCol = conf.getStringCollection(PATH_NAMESPACE);
                    }
                    queryText = function.getInputQuery(nsCol, start, 
                            mlSplit.isLastSplit() ? 
                                    Long.MAX_VALUE : (long)length);
                }
            } 
            if (queryText == null) {
                buildDocExprQuery(docExpr, nsCol, subExpr, buf);
                queryText = buf.toString();
            }
        } else {
            queryText = conf.get(MarkLogicConstants.INPUT_QUERY);
            queryLanguage = conf.get(MarkLogicConstants.INPUT_QUERY_LANGUAGE);
            if (queryText == null) {
                throw new IllegalStateException(
                  "Input query is required in advanced mode but not defined.");
            }
        }
        
        if (LOG.isDebugEnabled()) {
            LOG.debug(queryText);
        }


        // set up a connection to the server
        while (retry < maxRetries) {
        try {
            if (retry == 1) {
                LOG.info("Retrying connect");
            }
            String curForestName = "";
            String curHostName = "";
            if (curForest == -1) {
                curForestName = mlSplit.getForestId().toString();
                curHostName = hostNames[0];
            } else {
                curForestName = replicas.get(curForest).getForest();
                curHostName = replicas.get(curForest).getHostName();
            }
            ContentSource cs = InternalUtilities.getInputContentSource(conf, 
                    curHostName);
            session = cs.newSession("#"+curForestName);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Connect to forest "
                    + curForestName + " on "
                    + session.getConnectionUri().getHost());
            }
            AdhocQuery query = session.newAdhocQuery(queryText);
            if (advancedMode) {
                boolean bindSplitRange = 
                        conf.getBoolean(BIND_SPLIT_RANGE, false);
                if (bindSplitRange) {
                    query.setNewIntegerVariable(MR_NAMESPACE, 
                            SPLIT_START_VARNAME, start);
                    query.setNewIntegerVariable(MR_NAMESPACE, 
                            SPLIT_END_VARNAME, end);
                } else {
                    query.setPosition(start);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("split start position: " + start);
                    }        
                    query.setCount(mlSplit.isLastSplit() ?
                            Long.MAX_VALUE : mlSplit.getLength());
                }
            } else {
                query.setNewIntegerVariable(MR_NAMESPACE, 
                        SPLIT_START_VARNAME, start);
                query.setNewIntegerVariable(MR_NAMESPACE, 
                        SPLIT_END_VARNAME, end);
            }
            RequestOptions options = new RequestOptions();
            options.setCacheResult(false);
            if (queryLanguage != null) {
            	options.setQueryLanguage(queryLanguage);
            }
            String ts = conf.get(INPUT_QUERY_TIMESTAMP);
            if (ts != null) {
                options.setEffectivePointInTime(new BigInteger(ts));
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Query timestamp: " + ts);
                }
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Input query: " + query.getQuery());
            }
            query.setOptions(options);       
            result = session.submitRequest(query);
        } catch (XccConfigException e) {
            LOG.error("XccConfigException:" + e);
            throw new IOException(e);
        } catch (QueryException e) {
            LOG.error("QueryException:" + e);
            throw new IOException(e);
        } catch (RequestException e) {
            LOG.error("RequestException:" + e.getMessage());
            if (curForest != -1) {
                if (++retry < maxRetries) {
                    // failover
                    try {
                        InternalUtilities.sleep(sleepTime);
                    } catch (Exception e2) {
                    }
                    sleepTime = Math.min(sleepTime * 2,maxSleepTime);

                    curForest = (curForest+1)%replicas.size();
                    continue;
                 }
                 LOG.info("Exceeded max retry");
            }
            LOG.info("Query: " + queryText);
            if (LOG.isDebugEnabled()) {
                LOG.debug("RequestException: " + e);
            }
            throw new IOException(e);
        }
        break;
        } // retry
    }
337009
private Stream<Quad> fetchAllFromGraph(final String fromGraphName, final IRI toGraphName) {
        final Query q = new Query();
        q.setQuerySelectType();
        q.addResultVar(SUBJECT);
        q.addResultVar(PREDICATE);
        q.addResultVar(OBJECT);

        final ElementPathBlock epb = new ElementPathBlock();
        epb.addTriple(create(SUBJECT, PREDICATE, OBJECT));

        final ElementGroup elg = new ElementGroup();
        elg.addElement(new ElementNamedGraph(createURI(fromGraphName), epb));

        q.setQueryPattern(elg);

        final Stream.Builder<Quad> builder = builder();
        rdfConnection.querySelect(q, qs -> builder.accept(rdf.createQuad(toGraphName,
                        getSubject(qs), getPredicate(qs), getObject(qs))));
        return builder.build();
    }
645823
protected void gotoEOR(ArchiveRecord record) throws IOException {
        if (getIn().available() <= 0) {
            return;
        }
        
        // Remove any trailing LINE_SEPARATOR
        int c = -1;
        while (getIn().available() > 0) {
            if (getIn().markSupported()) {
                getIn().mark(1);
            }
            c = getIn().read();
            if (c != -1) {
                if (c == LINE_SEPARATOR) {
                    continue;
                }
                if (getIn().markSupported()) {
                    // We've overread.  We're probably in next record.  There is
                    // no way of telling for sure. It may be dross at end of
                    // current record. Backup.
                        getIn().reset();
                    break;
                }
                ArchiveRecordHeader h = (getCurrentRecord() != null)?
                    record.getHeader(): null;
                throw new IOException("Read " + (char)c +
                    " when only " + LINE_SEPARATOR + " expected. " + 
                    getReaderIdentifier() + ((h != null)?
                        h.getHeaderFields().toString(): ""));
            }
        }
    }
1373189
public Iterable<Record> values() {
        return () -> new Iterator<Record>(){
            private final Iterator<Record> it = data.records.values().iterator();

            /** {@inheritDoc} */
            @Override
            public boolean hasNext() {
                return it.hasNext();
            }

            /** {@inheritDoc} */
            @Override
            public Record next() {
                return it.next();
            }

            /** {@inheritDoc} */
            @Override
            public void remove() {
                throw new UnsupportedOperationException("This is a read-only iterator, remove operation is not supported.");
            }
        };
    }
1313913
@Override
	public synchronized List<RecordContext> getDataForContext (int contextId) throws DatabaseException {
    	try {
			List<RecordContext> result = new ArrayList<>();
			psGetAllDataForContext.setInt(1, contextId);
			try (ResultSet rs = psGetAllDataForContext.executeQuery()) {
				while (rs.next()) {
					result.add(new RecordContext(rs.getLong(DATAID), rs.getInt(CONTEXTID), rs.getInt(TYPE), rs.getString(DATA)));
				}
			}
			
			return result;
		} catch (SQLException e) {
			throw new DatabaseException(e);
		}
    }
727833
public Set<String> getExpandedRows() {
		Set<String> keys = getComponentModel().expandedRows;
		if (keys == null) {
			return Collections.emptySet();
		} else {
			return Collections.unmodifiableSet(keys);
		}
	}
377599
public String readAll() throws IOException {

        StringBuilder sb = new StringBuilder();
        String line;
        while ((line = readLine()) != null) {
            sb.append(line);
        }

        return sb.toString();
    }
1207470
public byte[] readRecord() throws IOException {
        if (this.debug) {
            System.err.println("ReadRecord: recIdx = " + this.currRecIdx + " blkIdx = " + this.currBlkIdx);
        }

        if (this.inStream == null) {
            throw new IOException("reading from an output buffer");
        }

        if (this.currRecIdx >= this.recsPerBlock) {
            if (!this.readBlock()) {
                return null;
            }
        }

        byte[] result = new byte[this.recordSize];

        System.arraycopy(this.blockBuffer, (this.currRecIdx * this.recordSize), result, 0, this.recordSize);

        this.currRecIdx++;

        return result;
    }
42322
public List<InputRecord> generate() throws Exception {
        List<InputRecord> result = new ArrayList<InputRecord>();
        this.validate();
        for (ValueExpression expression : this.getApplicableExpressions()) {
            try {
                expression.validate();
            } catch (Throwable e) {
                continue;
            }
            result.addAll(expression.generate());
        }
        return result;
    }
96579
public static Iterable<VcfRecord> records(final Readable readable) throws IOException {
        checkNotNull(readable);
        ParseListener parseListener = new ParseListener();
        VcfParser.parse(readable, parseListener);
        return parseListener.getRecords();
    }
1190426
@Override
	public OUT nextRecord(OUT tuple) throws IOException {
		try {
			resultSet.next();
			if (columnTypes == null) {
				extractTypes(tuple);
			}
			addValue(tuple);
			return tuple;
		} catch (SQLException se) {
			close();
			throw new IOException("Couldn't read data - " + se.getMessage(), se);
		} catch (NullPointerException npe) {
			close();
			throw new IOException("Couldn't access resultSet", npe);
		}
	}
268722
public static Iterable<VcfRecord> records(final InputStream inputStream) throws IOException {
        checkNotNull(inputStream);
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
            return records(reader);
        }
    }
1367876
public boolean hasNext() throws DBException
    {
        if (m_iCurrentRecord >= m_iRecordsAccessed)
            return true;
        Object record = this.next();
        if (record == null)
            return false;
        else
        {
            m_iRecordsAccessed--;   // Offically this record has not been accessed
            return true;
        }
    }
812647
public String[] loadAllRealNames ()
        throws PersistenceException
    {
        final ArrayList<String> names = new ArrayList<String>();

        // do the query
        execute(new Operation<Object>() {
            public Object invoke (Connection conn, DatabaseLiaison liaison)
                throws PersistenceException, SQLException
            {
                Statement stmt = conn.createStatement();
                try {
                    String query = "select realname from users";
                    ResultSet rs = stmt.executeQuery(query);
                    while (rs.next()) {
                        names.add(rs.getString(1));
                    }

                    // nothing to return
                    return null;

                } finally {
                    JDBCUtil.close(stmt);
                }
            }
        });

        // finally construct our result
        return names.toArray(new String[names.size()]);
    }
941224
@Override
    public List<InputRecord> generate() throws Exception {
        List<InputRecord> result = new ArrayList<InputRecord>();
        // Generated valid string
        Generex generex = new Generex(this.getInput().getValue());
        String value = "";
        InputRecord testRecord = this.getInput();
        for (int i = 0; i < 2; i++) {
            value = generex.random(1);
            result.add(
                    new InputRecord(
                            testRecord.getName(),
                            testRecord.getType(),
                            value,
                            testRecord.getCondition(),
                            value.matches(testRecord.getValue()))
            );
        }
        // Empty string
        value = "";
        result.add(
                new InputRecord(
                        testRecord.getName(),
                        testRecord.getType(),
                        value,
                        testRecord.getCondition(),
                        value.matches(testRecord.getValue()))
        );
        // Doubled string (repetitive pattern)
        value = generex.random(1);
        value = value.concat(value);
        result.add(
                new InputRecord(
                        testRecord.getName(),
                        testRecord.getType(),
                        value,
                        testRecord.getCondition(),
                        value.matches(testRecord.getValue()))
        );
        // Pattern itself
        result.add(
                new InputRecord(
                    testRecord.getName(),
                    testRecord.getType(),
                    testRecord.getValue(),
                    testRecord.getCondition(),
                    testRecord.getValue().matches(testRecord.getValue()))
                );
        return result;
    }
1226500
public void parseHeaders () throws IOException {
		if(!parsedHeader) {
			arcRecord.skipHttpHeader();
			// copy all HTTP headers to metaData, prefixing with 
			// HTTP_HEADER_PREFIX
			Header[] headers = arcRecord.getHttpHeaders();
			if (headers != null) {
				for (int i = 0; i < headers.length; i++) {
					String value = headers[i].getValue();
					String name = headers[i].getName();
					metaData.put(HTTP_HEADER_PREFIX + name,value);
					if(name.toUpperCase().contains(
							HttpHeaderOperation.HTTP_TRANSFER_ENC_HEADER)) {
						if(value.toUpperCase().contains(
								HttpHeaderOperation.HTTP_CHUNKED_ENCODING_HEADER)) {
							setChunkedEncoding();
						}
					}
				}
			}

			// copy all ARC record header fields to metaData, prefixing with 
			// ARC_META_PREFIX
			Map<String,Object> headerMetaMap = arcRecord.getMetaData().getHeaderFields();
			Set<String> keys = headerMetaMap.keySet();
			Iterator<String> itr = keys.iterator();
			while(itr.hasNext()) {
				String metaKey = itr.next();
				Object value = headerMetaMap.get(metaKey);
				String metaValue = "";
				if(value != null) {
					metaValue = value.toString();
				}
				metaData.put(ARC_META_PREFIX + metaKey,metaValue);
			}
		
			parsedHeader = true;			
		}
	}