Reads out an arc record, formats it and returns it
345514
def _read_arc_record(self):
        ""
        #XXX:Noufal Stream payload here rather than just read it
        # r = self.fileobj.readline() # Drop the initial newline
        # if r == "":
        #     return None
        # header = self.fileobj.readline()

        # Strip the initial new lines and read first line
        header = self.fileobj.readline()
        while header and header.strip() == "":
            header = self.fileobj.readline()

        if header == "":
            return None

        if int(self.version) == 1:
            arc_header_re = ARC1_HEADER_RE
        elif int(self.version) == 2:
            arc_header_re = ARC2_HEADER_RE

        matches = arc_header_re.search(header)
        headers = matches.groupdict()
        arc_header = ARCHeader(**headers)

        payload = self.fileobj.read(int(headers['length']))

        self.fileobj.readline() # Munge the separator newline.

        return ARCRecord(header = arc_header, payload = payload)
1262912
def __shxRecords(self):
        """"""
        f = self.__getFileObj(self.shx)
        f.seek(100)
        for i in range(len(self._shapes)):
            f.write(pack(">i", self._offsets[i] // 2))
            f.write(pack(">i", self._lengths[i]))
1191851
def from_bytes(cls, bitstream):
        '''
        
        '''
        record = cls()

        # Convert to ConstBitStream (if not already provided)
        if not isinstance(bitstream, ConstBitStream):
            if isinstance(bitstream, Bits):
                bitstream = ConstBitStream(auto=bitstream)
            else:
                bitstream = ConstBitStream(bytes=bitstream)

        # Read the record TTL
        record.ttl = bitstream.read('uint:32')

        # Store the locator record count until we need it
        locator_record_count = bitstream.read('uint:8')

        # Store the EID prefix mask length until we need it
        eid_prefix_len = bitstream.read('uint:8')

        # Read the Negative Map_Reply action
        record.action = bitstream.read('uint:3')

        # Read the flag
        record.authoritative = bitstream.read('bool')

        # Read reserved bits
        record._reserved1 = bitstream.read(12 + 4)

        # Read the map version
        record.map_version = bitstream.read('uint:12')

        # Read the EID prefix
        record.eid_prefix = read_afi_address_from_bitstream(bitstream,
                                                            eid_prefix_len)

        # Read the locator records
        for dummy in range(locator_record_count):
            locator_record = LocatorRecord.from_bytes(bitstream)
            record.locator_records.append(locator_record)

        # Verify that the properties make sense
        record.sanitize()

        return record
864081
def from_bytes(cls, bitstream):
        '''
        
        '''
        record = cls()

        # Convert to ConstBitStream (if not already provided)
        if not isinstance(bitstream, ConstBitStream):
            if isinstance(bitstream, Bits):
                bitstream = ConstBitStream(auto=bitstream)
            else:
                bitstream = ConstBitStream(bytes=bitstream)

        # Read the priorities and weights
        (record.priority, record.weight, record.m_priority,
         record.m_weight) = bitstream.readlist('4*uint:8')

        # Read over unused flags
        record._reserved1 = bitstream.read(13)

        # Read the flags
        (record.local,
         record.probed_locator,
         record.reachable) = bitstream.readlist('3*bool')

        # Read the locator
        record.address = read_afi_address_from_bitstream(bitstream)

        # Verify that the properties make sense
        record.sanitize()

        return record
210342
def from_record(cls, record, crs):
        """"""
        if 'type' not in record:
            raise TypeError("The data isn't a valid record.")

        return cls(to_shape(record), crs)
380137
def _detect_type_load_headers(self, stream,
                                  statusline=None, known_format=None):
        

        if known_format != 'arc':
            # try as warc first
            try:
                rec_headers = self.warc_parser.parse(stream, statusline)
                return 'warc', rec_headers
            except StatusAndHeadersParserException as se:
                if known_format == 'warc':
                    msg = 'Invalid WARC record, first line: '
                    raise ArchiveLoadFailed(msg + str(se.statusline))

                statusline = se.statusline
                pass

        # now try as arc
        try:
            rec_headers = self.arc_parser.parse(stream, statusline)
            return self.arc_parser.get_rec_type(), rec_headers
        except StatusAndHeadersParserException as se:
            if known_format == 'arc':
                msg = 'Invalid ARC record, first line: '
            else:
                msg = 'Unknown archive format, first line: '
            raise ArchiveLoadFailed(msg + str(se.statusline))
854050
function splitArc(arc, point) {
    if (!point) {
        point = findMidPoint(arc);
    }

    var subArcs = [];

    var oldDistance = geomUtils.findDistanceBetweenTwoPoints(arc.start, arc.end);
    var halfOldDistance = oldDistance / 2;

    var arcHeight = Math.abs(arc.height);
    var radius = (Math.pow(arcHeight, 2) + Math.pow(halfOldDistance, 2)) / (2 * arcHeight);

    var newDistance1 = geomUtils.findDistanceBetweenTwoPoints(arc.start, point);
    var newDistance2 = geomUtils.findDistanceBetweenTwoPoints(arc.end, point);

    var halfNewDistance1 = newDistance1 / 2;
    var halfNewDistance2 = newDistance2 / 2;

    var angleDistance1 = geomUtils.solveAngle(oldDistance, newDistance2, newDistance1);
    var angleDistance2 = geomUtils.solveAngle(oldDistance, newDistance1, newDistance2);

    var angle1Total = angleDistance1 * 2;
    var angle2Total = angleDistance2 * 2;
    var isAngle1GreaterThan180 = angle1Total > 180;
    var isAngle2GreaterThan180 = angle2Total > 180;

    var newArcHeight1 = findArcHeight(halfNewDistance1, radius, arcHeight, isAngle1GreaterThan180);
    var newArcHeight2 = findArcHeight(halfNewDistance2, radius, arcHeight, isAngle2GreaterThan180);

    if (arc.height < 0) {
        newArcHeight1 = -1 * newArcHeight1;
        newArcHeight2 = -1 * newArcHeight2;
    }

    // Remember an Arc is {start, end, height} -- because requiring arc.js in
    // here would be recursive, we avoid doing that and just hack around it
    subArcs.push({start: arc.start, end: point, height: newArcHeight1 || arc.height});
    subArcs.push({start: point, end: arc.end, height: newArcHeight2 || 0});

    // debug("orig:", arc, "new:", subArcs);
    return subArcs;
}
508483
def readline(self, size=-1):
        ""
        try:
            record = next(self.reader)
            return self.outdel.join(self.process_line(record)) + '\n'
        except StopIteration:
            return ''
507474
def readline(self, size=-1):
        ""
        while True:
            try:
                record = next(self.reader)
            except StopIteration:
                break

            # Ensure this is a valid record
            if checks.record_is_valid(record):
                if self.use_cache:
                    # Ensures the cache is updated and available
                    self.variants.ensure_cache(record)

                # Calculate the MD5 of the variant itself (not the record)
                md5 = calculate_md5(record)

                # Ensure this variant is not already loaded
                if not self.use_cache or md5 not in self.variants:
                    cleaned = self.process_line(record)
                    cleaned.append(md5)
                    return self.outdel.join(cleaned) + '\n'

        return ''
1309592
private InputStream readHttpHeader() throws IOException {
    	
    	// this can be helpful when simply iterating over records, 
    	// looking for problems.
        Logger logger = Logger.getLogger(this.getClass().getName());
    	ArchiveRecordHeader h = this.getHeader();
    	
        // If judged a record that doesn't have an http header, return
        // immediately.
        String url = getHeader().getUrl();
        if(!url.startsWith("http") ||
            getHeader().getLength() <= MIN_HTTP_HEADER_LENGTH) {
            return null;
        }
        
        String statusLine;
        byte[] statusBytes;
        int eolCharCount = 0;
        int errOffset = 0;
        
        // Read status line, skipping any errant http headers found before it
        // This allows a larger number of 'corrupt' arcs -- where headers were accidentally
        // inserted before the status line to be readable
        while (true) {
        	statusBytes = LaxHttpParser.readRawLine(getIn());
        	eolCharCount = getEolCharsCount(statusBytes);
        	if (eolCharCount <= 0) {
        		throw new RecoverableIOException(
                "Failed to read http status where one was expected: " 
                + ((statusBytes == null) ? "" : new String(statusBytes)));
        	}
        
        	statusLine = EncodingUtil.getString(statusBytes, 0,
        			statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING);
        	
        	// If a null or DELETED break immediately
        	if ((statusLine == null) || statusLine.startsWith("DELETED")) {
        		break;
        	}
        	
        	// If it's actually the status line, break, otherwise continue skipping any
        	// previous header values
        	if (!statusLine.contains(":") && StatusLine.startsWithHTTP(statusLine)) {
        		break;
        	}
        	
        	// Add bytes read to error "offset" to add to position
        	errOffset += statusBytes.length;
        }
        
        if (errOffset > 0) {
            this.incrementPosition(errOffset);
        }
        
        if ((statusLine == null) ||
                !StatusLine.startsWithHTTP(statusLine)) {
            if (statusLine.startsWith("DELETED")) {
                // Some old ARCs have deleted records like following:
                // http://vireo.gatech.edu:80/ebt-bin/nph-dweb/dynaweb/SGI_Developer/SGITCL_PG/@Generic__BookTocView/11108%3Btd%3D2 130.207.168.42 19991010131803 text/html 29202
                // DELETED_TIME=20000425001133_DELETER=Kurt_REASON=alexalist
                // (follows ~29K spaces)
                // For now, throw a RecoverableIOException so if iterating over
                // records, we keep going.  TODO: Later make a legitimate
                // ARCRecord from the deleted record rather than throw
                // exception.
                throw new DeletedARCRecordIOException(statusLine);
            } else {
            	this.errors.add(ArcRecordErrors.HTTP_STATUS_LINE_INVALID);
            }
        }

        try {
        	this.httpStatus = new StatusLine(statusLine);
        } catch(IOException e) {
        	logger.warning(e.getMessage() + " at offset: " + h.getOffset());
        	this.errors.add(ArcRecordErrors.HTTP_STATUS_LINE_EXCEPTION);
        }
        
        // Save off all bytes read.  Keep them as bytes rather than
        // convert to strings so we don't have to worry about encodings
        // though this should never be a problem doing http headers since
        // its all supposed to be ascii.
        ByteArrayOutputStream baos =
            new ByteArrayOutputStream(statusBytes.length + 4 * 1024);
        baos.write(statusBytes);
        
        // Now read rest of the header lines looking for the separation
        // between header and body.
        for (byte [] lineBytes = null; true;) {
            lineBytes = LaxHttpParser.readRawLine(getIn());
            eolCharCount = getEolCharsCount(lineBytes);
            if (eolCharCount <= 0) {
            	if (getIn().available() == 0) {
            		httpHeaderBytesRead += statusBytes.length;
                	logger.warning("HTTP header truncated at offset: " + h.getOffset());
            		this.errors.add(ArcRecordErrors.HTTP_HEADER_TRUNCATED);
            		this.setEor(true);
            		break;
            	} else {
            		throw new IOException("Failed reading http headers: " +
            				((lineBytes != null)? new String(lineBytes): null));
            	}
            } else {
            	httpHeaderBytesRead += lineBytes.length;
            }
            // Save the bytes read.
            baos.write(lineBytes);
            if ((lineBytes.length - eolCharCount) <= 0) {
                // We've finished reading the http header.
                break;
            }
        }
        
        byte [] headerBytes = baos.toByteArray();
        // Save off where body starts.
        this.getMetaData().setContentBegin(headerBytes.length);
        ByteArrayInputStream bais =
            new ByteArrayInputStream(headerBytes);
        if (!bais.markSupported()) {
            throw new IOException("ByteArrayInputStream does not support mark");
        }
        bais.mark(headerBytes.length);
        // Read the status line.  Don't let it into the parseHeaders function.
        // It doesn't know what to do with it.
        bais.read(statusBytes, 0, statusBytes.length);
        this.httpHeaders = LaxHttpParser.parseHeaders(bais,
            ARCConstants.DEFAULT_ENCODING);
        this.getMetaData().setStatusCode(Integer.toString(getStatusCode()));
        bais.reset();
        return bais;
    }
409756
def shapeRecord(self, i=0):
        """"""
        i = self.__restrictIndex(i)
        return ShapeRecord(shape=self.shape(i), record=self.record(i))
517771
def loadRecord(self, domain, rtype, callback=None, errback=None):
        
        rec = Record(self, domain, rtype)
        return rec.load(callback=callback, errback=errback)
73234
def _par_read(dirname, compressed=True):
    
    templates = []
    if compressed:
        arc = tarfile.open(dirname, "r:*")
        members = arc.getmembers()
        _parfile = [member for member in members
                    if member.name.split(os.sep)[-1] ==
                    'template_parameters.csv']
        if len(_parfile) == 0:
            arc.close()
            raise MatchFilterError(
                'No template parameter file in archive')
        parfile = arc.extractfile(_parfile[0])
    else:
        parfile = open(dirname + '/' + 'template_parameters.csv', 'r')
    for line in parfile:
        t_in = Template()
        for key_pair in line.rstrip().split(','):
            if key_pair.split(':')[0].strip() == 'name':
                t_in.__dict__[key_pair.split(':')[0].strip()] = \
                    key_pair.split(':')[-1].strip()
            elif key_pair.split(':')[0].strip() == 'filt_order':
                try:
                    t_in.__dict__[key_pair.split(':')[0].strip()] = \
                        int(key_pair.split(':')[-1])
                except ValueError:
                    pass
            else:
                try:
                    t_in.__dict__[key_pair.split(':')[0].strip()] = \
                        float(key_pair.split(':')[-1])
                except ValueError:
                    pass
        templates.append(t_in)
    parfile.close()
    if compressed:
        arc.close()
    return templates
1105338
def __shxRecord(self, offset, length):
         """"""
         f = self.__getFileObj(self.shx)
         f.write(pack(">i", offset // 2))
         f.write(pack(">i", length))
1084451
function stringify(arc) {
  let fmtTbl = obj=> {
    let name = Object.keys(obj)[0]
    let keys = Object.keys(obj[name])
    let result = `${name}\n`
    keys.forEach(key=> {
      let val = obj[name][key]
      result += `  ${key} ${val}\n`
    })
    return result
  }
  let str = `@app\n${arc.app[0]}\n`
  /////////////////////////////////
  if (arc.http.length > 0)
    str += `\n@http\n` + arc.http.map(tuple=> tuple.join(' ') + '\n').join('')

  if (arc.events.length > 0)
    str += `\n@events\n` + arc.events.join('\n') + '\n'

  if (arc.queues.length > 0)
    str += `\n@queues\n` + arc.queues.join('\n') + '\n'

  if (arc.scheduled.length > 0)
    str += `\n@scheduled\n` + arc.scheduled.map(v=> v.join(' ')).join('\n') + '\n'

  if (arc.tables.length > 0)
    str += `\n@tables\n` + arc.tables.map(fmtTbl).join('\n')

  if (arc.aws.length > 0)
    str += `\n@aws\n` + arc.aws.map(tuple=> tuple.join(' ') + '\n').join('')
  //////////
  return str
}
168819
def read_sheets(archive):
    """"""
    xml_source = archive.read(ARC_WORKBOOK)
    tree = fromstring(xml_source)
    for element in safe_iterator(tree, '{%s}sheet' % SHEET_MAIN_NS):
        attrib = element.attrib
        attrib['id'] = attrib["{%s}id" % REL_NS]
        del attrib["{%s}id" % REL_NS]
        if attrib['id']:
            yield attrib
785058
def read(path='', mode='r', record_size=None, record_seperator=None, offset=0, autostrip=False):
    '''  '''
    assert isinstance(path, str), 'path needs to be a string, not: {}'.format(repr(path))
    assert isinstance(mode, str), 'mode needs to be a string, not: {}'.format(repr(mode))
    assert ((record_size is None)+(record_seperator is None))>0 , 'generators.read does not support both record_seperator and record_size options being used at the same time: {} {}'.format(record_size, record_seperator)
    assert record_size is None or type(record_size) is int, 'record_size can only be defined as an int, not: {}'.format(repr(record_size))
    assert record_seperator is None or isinstance(record_seperator, str) and len(record_seperator), 'record_seperator can only be defined as a non-empty string, not: {}'.format(repr(record_seperator))
    assert type(offset) == int and offset>=0, 'offset needs to be a positive int, not: {}'.format(repr(offset))
    assert type(autostrip) == bool, 'autostrip needs to be a boolean, not: {}'.format(repr(autostrip))

    stop_value = b'' if mode == 'rb' else ''
    if autostrip:
        autostrip = lambda i:i.strip()
    else:
        autostrip = lambda i:i

    if path=='': # if path is empty, use stdin
        if record_seperator is None:
            for line in stdin:
                yield autostrip(line)
        else:
            for entry in read_with_seperator(stdin, record_seperator, stop_value):
                yield autostrip(entry)
    else: # otherwise, open path
        with open(path, mode) as f:
            if record_size is None and record_seperator is None:  # no record_size or record_seperator? iterate over lines
                for line in f:
                    if offset > 0: # offset in line reading mode offsets per each line
                        offset -= 1
                    else:
                        yield autostrip(line)
            elif record_seperator is not None:
                f.seek(offset)
                for record in read_with_seperator(f, record_seperator, stop_value):
                    yield autostrip(record)
            else:  # if record_size is specified, iterate over records at that size
                f.seek(offset)
                for record in iter(partial(f.read, record_size), stop_value):
                    yield autostrip(record)