Helper routine that converts a Sentence protobuf to a string from
    its tokens.
351967
def to_text(sentence):
    
    text = ""
    for i, tok in enumerate(sentence.token):
        if i != 0:
            text += tok.before
        text += tok.word
    return text
995254
def tokenize_sentences(self, untokenized_string: str):
        
        # load tokenizer
        assert isinstance(untokenized_string, str), \
            'Incoming argument must be a string.'

        if self.language == 'latin':
            self.models_path = self._get_models_path(self.language)
            try:
                self.model = open_pickle(
                    os.path.expanduser(os.path.join(self.models_path, 'latin_punkt.pickle')))
            except FileNotFoundError as err:
                raise type(err)(TokenizeSentence.missing_models_message + self.models_path)
            tokenizer = self.model
            tokenizer._lang_vars = self.lang_vars
        elif self.language == 'greek': # Workaround for regex tokenizer
            self.sent_end_chars=GreekLanguageVars.sent_end_chars
            self.sent_end_chars_regex = '|'.join(self.sent_end_chars)
            self.pattern = rf'(?<=[{self.sent_end_chars_regex}])\s'
        else:
            tokenizer = open_pickle(self.tokenizer_path)
            tokenizer = self._setup_tokenizer(tokenizer)

        # mk list of tokenized sentences
        if self.language == 'latin':
            return tokenizer.tokenize(untokenized_string)
        elif self.language == 'greek':
            return re.split(self.pattern, untokenized_string)
        else:
            tokenized_sentences = [sentence for sentence in
                                   tokenizer.sentences_from_text(untokenized_string,
                                                                 realign_boundaries=True)]
            return tokenized_sentences
1306433
def p_out(p):
    
    p[0] = make_sentence('OUT',
                         make_typecast(TYPE.uinteger, p[2], p.lineno(3)),
                         make_typecast(TYPE.ubyte, p[4], p.lineno(4)))
18236
def from_tokens(cls, text, toks):
        
        sentence_pb = CoreNLP_pb2.Sentence()
        sentence_pb.characterOffsetBegin = 0
        sentence_pb.characterOffsetEnd = len(text)
        sentence_pb.sentenceIndex = 0
        sentence_pb.tokenOffsetBegin = 0
        sentence_pb.tokenOffsetEnd = len(toks)

        # Track progress in sentence and tokens.
        char_idx = 0
        tok_idx = 0

        buf = ""
        token_pb = None
        while char_idx < len(text):
            tok = toks[tok_idx]
            # Scan to the beginning of the token.
            if text[char_idx] != tok[0]:
                buf += text[char_idx]
                char_idx += 1
            # Aha! we have found the token. Assert that they match.
            else:
                assert text[char_idx:char_idx+len(tok)] == tok, "text did not match a token"
                # Create a new Token from this text.
                if token_pb: token_pb.after = buf
                token_pb = sentence_pb.token.add()
                token_pb.before = buf

                token_pb.beginChar = char_idx
                token_pb.endChar = char_idx + len(tok)
                # TODO(chaganty): potentially handle -LRB-?
                token_pb.value = tok
                token_pb.word = tok
                token_pb.originalText = text[char_idx:char_idx+len(tok)]

                buf = ""
                char_idx += len(tok)
                tok_idx += 1
        if token_pb: token_pb.after = buf
        assert tok_idx == len(toks), "text does not match all tokens"
        return AnnotatedSentence.from_pb(sentence_pb)
715239
def AsPrimitiveProto(self):
    """"""
    if self.protobuf:
      result = self.protobuf()
      result.ParseFromString(self.SerializeToString())
      return result
968080
def from_string(proto_str):
    '''
    
    '''
    _, proto_file = tempfile.mkstemp(suffix='.proto')

    with open(proto_file, 'w+') as proto_f:
        proto_f.write(proto_str)

    return from_file(proto_file)
893697
function bufferToString (buf) {
  const a = bufferToTuples(buf)
  const b = tuplesToStringTuples(a)
  return stringTuplesToString(b)
}
716071
def _parse_proto(prototxt_fname):
    
    proto = caffe_parser.read_prototxt(prototxt_fname)

    # process data layer
    input_name, input_dim, layers = _get_input(proto)
    # only support single input, so always use `data` as the input data
    mapping = {input_name: 'data'}
    need_flatten = {input_name: False}
    symbol_string = "import mxnet as mx\ndata = mx.symbol.Variable(name='data')\n"

    flatten_count = 0
    output_name = ""
    prev_name = None
    _output_name = {}

    # convert reset layers one by one
    for i, layer in enumerate(layers):
        type_string = ''
        param_string = ''
        skip_layer = False
        name = re.sub('[-/]', '_', layer.name)
        for k in range(len(layer.bottom)):
            if layer.bottom[k] in _output_name:
                _output_name[layer.bottom[k]]['count'] = _output_name[layer.bottom[k]]['count']+1
            else:
                _output_name[layer.bottom[k]] = {'count':0}
        for k in range(len(layer.top)):
            if layer.top[k] in _output_name:
                _output_name[layer.top[k]]['count'] = _output_name[layer.top[k]]['count']+1
            else:
                _output_name[layer.top[k]] = {'count':0, 'name':name}
        if layer.type == 'Convolution' or layer.type == 4:
            type_string = 'mx.symbol.Convolution'
            param_string = _convert_conv_param(layer.convolution_param)
            need_flatten[name] = True
        if layer.type == 'Deconvolution' or layer.type == 39:
            type_string = 'mx.symbol.Deconvolution'
            param_string = _convert_conv_param(layer.convolution_param)
            need_flatten[name] = True
        if layer.type == 'Pooling' or layer.type == 17:
            type_string = 'mx.symbol.Pooling'
            param_string = _convert_pooling_param(layer.pooling_param)
            need_flatten[name] = True
        if layer.type == 'ReLU' or layer.type == 18:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='relu'"
            param = layer.relu_param
            if hasattr(param, 'negative_slope'):
                if param.negative_slope > 0:
                    type_string = 'mx.symbol.LeakyReLU'
                    param_string = "act_type='leaky', slope=%f" % param.negative_slope
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'TanH' or layer.type == 23:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='tanh'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Sigmoid' or layer.type == 19:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='sigmoid'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'LRN' or layer.type == 15:
            type_string = 'mx.symbol.LRN'
            param = layer.lrn_param
            param_string = "alpha=%f, beta=%f, knorm=%f, nsize=%d" % (
                param.alpha, param.beta, param.k, param.local_size)
            need_flatten[name] = True
        if layer.type == 'InnerProduct' or layer.type == 14:
            type_string = 'mx.symbol.FullyConnected'
            param = layer.inner_product_param
            param_string = "num_hidden=%d, no_bias=%s" % (
                param.num_output, not param.bias_term)
            need_flatten[name] = False
        if layer.type == 'Dropout' or layer.type == 6:
            type_string = 'mx.symbol.Dropout'
            param = layer.dropout_param
            param_string = "p=%f" % param.dropout_ratio
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Softmax' or layer.type == 20:
            type_string = 'mx.symbol.SoftmaxOutput'
        if layer.type == 'Flatten' or layer.type == 8:
            type_string = 'mx.symbol.Flatten'
            need_flatten[name] = False
        if layer.type == 'Split' or layer.type == 22:
            type_string = 'split'  # will process later
        if layer.type == 'Concat' or layer.type == 3:
            type_string = 'mx.symbol.Concat'
            need_flatten[name] = True
        if layer.type == 'Crop':
            type_string = 'mx.symbol.Crop'
            need_flatten[name] = True
            param_string = 'center_crop=True'
        if layer.type == 'BatchNorm':
            type_string = 'mx.symbol.BatchNorm'
            param = layer.batch_norm_param
            # CuDNN requires eps to be greater than 1e-05
            # We compensate for this change in convert_model
            epsilon = param.eps
            if (epsilon <= 1e-05):
                epsilon = 1e-04
            # if next layer is scale, don't fix gamma
            fix_gamma = layers[i+1].type != 'Scale'
            param_string = 'use_global_stats=%s, fix_gamma=%s, eps=%f' % (
                param.use_global_stats, fix_gamma, epsilon)
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Scale':
            assert layers[i-1].type == 'BatchNorm'
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
            skip_layer = True
            prev_name = re.sub('[-/]', '_', layers[i-1].name)
        if layer.type == 'PReLU':
            type_string = 'mx.symbol.LeakyReLU'
            param = layer.prelu_param
            param_string = "act_type='prelu', slope=%f" % param.filler.value
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Eltwise':
            type_string = 'mx.symbol.broadcast_add'
            param = layer.eltwise_param
            param_string = ""
            need_flatten[name] = False
        if layer.type == 'Reshape':
            type_string = 'mx.symbol.Reshape'
            need_flatten[name] = False
            param = layer.reshape_param
            param_string = "shape=(%s)" % (','.join(param.shape.dim),)
        if layer.type == 'AbsVal':
            type_string = 'mx.symbol.abs'
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]

        if skip_layer:
            assert len(layer.bottom) == 1
            symbol_string += "%s = %s\n" % (name, prev_name)
        elif type_string == '':
            raise ValueError('Unknown layer %s!' % layer.type)
        elif type_string != 'split':
            bottom = layer.bottom
            if param_string != "":
                param_string = ", " + param_string
            if len(bottom) == 1:
                if need_flatten[mapping[bottom[0]]] and type_string == 'mx.symbol.FullyConnected':
                    flatten_name = "flatten_%d" % flatten_count
                    symbol_string += "%s=mx.symbol.Flatten(name='%s', data=%s)\n" % (
                        flatten_name, flatten_name, mapping[bottom[0]])
                    flatten_count += 1
                    need_flatten[flatten_name] = False
                    bottom[0] = flatten_name
                    mapping[bottom[0]] = bottom[0]
                symbol_string += "%s = %s(name='%s', data=%s %s)\n" % (
                    name, type_string, name, mapping[bottom[0]], param_string)
            else:
                if layer.type == 'Eltwise' and param.operation == 1 and len(param.coeff) > 0:
                    symbol_string += "%s = " % name
                    symbol_string += " + ".join(["%s * %s" % (
                        mapping[bottom[i]], param.coeff[i]) for i in range(len(param.coeff))])
                    symbol_string += "\n"
                else:
                    symbol_string += "%s = %s(name='%s', *[%s] %s)\n" % (
                        name, type_string, name, ','.join(
                            [mapping[x] for x in bottom]), param_string)
        for j in range(len(layer.top)):
            mapping[layer.top[j]] = name
        output_name = name
    output_name = []
    for i in _output_name:
        if 'name' in _output_name[i] and _output_name[i]['count'] == 0:
            output_name.append(_output_name[i]['name'])

    return symbol_string, output_name, input_dim
140952
def _create_sentence_objects(self):
        """"""
        sentence_objects = []
        sentences = sent_tokenize(self.raw, tokenizer=self.tokenizer)
        char_index = 0  # Keeps track of character index within the blob
        for sent in sentences:
            # Compute the start and end indices of the sentence
            # within the blob. This only works if the sentence splitter
            # does not perform any character replacements or changes to
            # white space.
            # Working: NLTKPunktTokenizer
            # Not working: PatternTokenizer
            try:
                start_index = self.raw.index(sent, char_index)
                char_index += len(sent)
                end_index = start_index + len(sent)
            except ValueError:
                start_index = None
                end_index = None
            # Sentences share the same models as their parent blob
            s = Sentence(
                sent,
                start_index=start_index,
                end_index=end_index,
                tokenizer=self.tokenizer,
                np_extractor=self.np_extractor,
                pos_tagger=self.pos_tagger,
                analyzer=self.analyzer,
                parser=self.parser,
                classifier=self.classifier)
            sentence_objects.append(s)
        return sentence_objects
214226
def sentence_bytes(self, sentence):
        
        result = [TOKENS[sentence[0]]]

        for i in sentence[1:]:  # Remaining bytes
            if isinstance(i, str):
                result.extend(self.literal(i))
            elif isinstance(i, float) or isinstance(i, int):  # A number?
                result.extend(self.number(i))
            else:
                result.extend(i)  # Must be another thing

        return result
411598
def _ToString(x):
    """"""
    # Some cross-language values for primitives.  This is tested in
    # jsontemplate_test.py.
    if x is None:
        return 'null'
    if isinstance(x, six.string_types):
        return x
    return pprint.pformat(x)
579638
function parseBulkString (parser) {
  const length = parseLength(parser)
  if (length === undefined) {
    return
  }
  if (length < 0) {
    return null
  }
  const offset = parser.offset + length
  if (offset + 2 > parser.buffer.length) {
    parser.bigStrSize = offset + 2
    parser.totalChunkSize = parser.buffer.length
    parser.bufferCache.push(parser.buffer)
    return
  }
  const start = parser.offset
  parser.offset = offset + 2
  if (parser.optionReturnBuffers === true) {
    return parser.buffer.slice(start, offset)
  }
  return parser.buffer.toString('utf8', start, offset)
}
1057674
def from_bytes(self, string):
        """"""
        msg = srsly.msgpack_loads(gzip.decompress(string))
        self.attrs = msg["attrs"]
        self.strings = set(msg["strings"])
        lengths = numpy.fromstring(msg["lengths"], dtype="int32")
        flat_spaces = numpy.fromstring(msg["spaces"], dtype=bool)
        flat_tokens = numpy.fromstring(msg["tokens"], dtype="uint64")
        shape = (flat_tokens.size // len(self.attrs), len(self.attrs))
        flat_tokens = flat_tokens.reshape(shape)
        flat_spaces = flat_spaces.reshape((flat_spaces.size, 1))
        self.tokens = NumpyOps().unflatten(flat_tokens, lengths)
        self.spaces = NumpyOps().unflatten(flat_spaces, lengths)
        for tokens in self.tokens:
            assert len(tokens.shape) == 2, tokens.shape
        return self
994905
private final String bufferToString(byte[] str) {
		String string = null;
		if (str != null) {
			string = AMF.CHARSET.decode(ByteBuffer.wrap(str)).toString();
			log.debug("String: {}", string);
		} else {
			log.warn("ByteBuffer was null attempting to read String");
		}
		return string;
	}
91927
function tokenize (str) {
  let pos = 0
  let row = 1
  let col = 1
  let newline = true
  let state = CONTENT
  let start = 0
  let consumingSameLineContent = false
  let escapedInlineCounter = 0
  let escapedParamCounter = 0
  let consumingEscapedParams = false
  let consumingUnparsed = false // true when in an @@ block
  let lastInlineContentEndPos = void (0)
  let unparsedIndentStart = void (0)

  const indent = [0]
  const tokens = []

  function emit (type, value) {
    if (arguments.length > 1) {
      tokens.push({type: type, value: value})
    } else {
      tokens.push({type: type})
    }
  }

  function consume (type, next) {
    emit(type, str.substring(start, pos))
    state = next
    start = pos + 1
  }

  function consumeIfNonEmpty (type, next, escaped) {
    let v = str.substring(start, pos)
    if (v.length > 0) {
      if (escaped) {
        const l = escaped.length
        for (let i = 0; i < l; i++) {
          const es = escaped[i]
          v = v.replace(es, es[1])
        }
      }
      emit(type, v)
    }
    state = next
    start = pos + 1
  }

  function shuffleIfNext (character) {
    if (str[pos + 1] === character) {
      pos++
      start++
      return true
    } else {
      return false
    }
  }

  function err (msg) {
    let start = str.lastIndexOf('\n', pos - 1)
    start = str.lastIndexOf('\n', start - 1)
    start = str.lastIndexOf('\n', start - 1)

    const nextNewline = str.indexOf('\n', pos)
    const errorLineEnd = nextNewline === -1 ? pos : nextNewline

    let end = str.indexOf('\n', errorLineEnd + 1)
    if (end === -1) {
      end = str.length - 1
    } else {
      end = str.indexOf('\n', end + 1)
      if (end === -1) {
        end = str.length - 1
      } else {
        end = str.indexOf('\n', end + 1)
        if (end === -1) {
          end = str.length - 1
        }
      }
    }

    const indentSpaces = (new Array(col)).join(' ')

    const message = 'Syntax error at line ' + row + ', col ' + col + ': ' + msg
    const context = str.substring(start, errorLineEnd) + '\n' + indentSpaces + '^^^^^^^^\n' + str.substring(errorLineEnd + 1, end)

    throw new ParseError(message + '\n' + context, context, str, row, col, msg, pos)
  }

  while (pos < str.length) {
    // indentation and comment handling.
    if (newline) {
      while (newline && pos < str.length) {
        let ind = 0
        while (str[pos + ind] === ' ' && pos + ind < str.length) {
          ind++
        }

        // if it was an empty content line, then add it to the list of empty
        // content to emit on the next non empty line
        if (str[pos + ind] === '\n') {
          pos += ind + 1
          row++
          emit('EMPTY_CONTENT', str.substring(start, pos - 1))
        } else if (str[pos + ind] !== '#' || consumingUnparsed) {
          if (ind > indent[indent.length - 1]) {
            emit('INDENT', ind - indent[indent.length - 1])
            indent.push(ind)
          } else if (ind < indent[indent.length - 1]) {
            while (indent[indent.length - 1] > ind) {
              const prev = indent.pop()
              emit('DEDENT', prev - indent[indent.length - 1])

              if (consumingUnparsed && unparsedIndentStart >= indent[indent.length - 1]) {
                consumingUnparsed = false
              }
            }

            if (indent.length > 0 && (indent[indent.length - 1] !== ind)) {
              err('indentation mismatch: this line dedents with an indentation of ' + ind + ' spaces, but an indentation of ' + indent[indent.length - 1] + ' spaces was expected')
            }
          }
          pos += ind

          if (str[pos] === '\\' && str[pos + 1] === '#') {
            pos++
          }
          newline = false
        } else {
          while (str[pos] !== '\n' && pos < str.length) {
            pos++
          }
          emit('COMMENT', str.substring(start + ind + 1, pos))
          pos++
          row++
        }

        start = pos
      }
    }

    if (pos >= str.length) {
      return tokens
    }

    const s = str[pos]

    if (state === TYPE) {
      if (s === ' ') {
        consume('TYPE', PARAMS)
        while (shuffleIfNext(' ') && pos < str.length) {}
      } else if (s === ':') {
        consume('TYPE', CONTENT)
        shuffleIfNext(' ')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '[') {
        consume('TYPE', INLINE_CONTENT)
        emit('START_INLINE_CONTENT')
      } else if (s === '(') {
        consume('TYPE', INLINE_PARAMS)
      } else if (s === '\n') {
        consume('TYPE', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === PARAMS) {
      if (s === '[') {
        escapedParamCounter++
        consumingEscapedParams = true
      } else if (consumingEscapedParams && (s === ']')) {
        escapedParamCounter--
        if (escapedParamCounter === 0) {
          consumingEscapedParams = false
        }
      } else if (s === ':' && !consumingEscapedParams) {
        consume('PARAMS', CONTENT)
        shuffleIfNext(' ')
        shuffleIfNext(']')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '\n') {
        escapedParamCounter = 0
        consumingEscapedParams = false
        consume('PARAMS', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === CONTENT) {
      if (s === '@' && !consumingUnparsed) {
        consumeIfNonEmpty('CONTENT', TYPE)
        if (str[pos + 1] === '@') {
          consumingUnparsed = true
          unparsedIndentStart = indent[indent.length - 1]
          pos++
          start++
        }
      } else if (s === '\n') {
        if (consumingSameLineContent) {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
          consumingUnparsed = false
        } else {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
        }
      }
    } else if (state === INLINE_PARAMS) {
      if (s === ')') {
        if (str[pos + 1] === '[') {
          consume('PARAMS', INLINE_CONTENT)
          emit('START_INLINE_CONTENT')
          start++
          pos++
        } else {
          consume('PARAMS', CONTENT)
        }
      }
    } else { // if (state === INLINE_CONTENT) {
      if (s === '\\' && (str[pos + 1] === '[' || str[pos + 1] === ']')) {
        pos++
      } else if (s === '[') {
        escapedInlineCounter++
      } else if (s === ']') {
        if (escapedInlineCounter === 0) {
          consumeIfNonEmpty('CONTENT', CONTENT, ['\\]', '\\['])
          emit('END_INLINE_CONTENT')
          lastInlineContentEndPos = pos + 1
        } else {
          escapedInlineCounter--
        }
      } else if (s === '\n') {
        consume('CONTENT', INLINE_CONTENT)
      }
    }

    if (s === '\n') {
      col = 0
      row++
      newline = true
    } else {
      col++
    }

    pos++
  }

  if (state === TYPE) {
    emit('TYPE', str.substring(start, pos))
  } else if (state === PARAMS) {
    emit('PARAMS', str.substring(start, pos))
  } else if (state === INLINE_PARAMS) {
    err('missing closing ) bracket?')
  } else if (state === INLINE_CONTENT) {
    err('missing closing ] bracket?')
  } else { // if (state === CONTENT) {
    if (start < pos) {
      emit('CONTENT', str.substring(start, pos))
    }
  }

  return tokens
}
1207743
def _converttobitstring(cls, bs, offset=0, cache={}):
        
        if isinstance(bs, Bits):
            return bs
        try:
            return cache[(bs, offset)]
        except KeyError:
            if isinstance(bs, basestring):
                b = cls()
                try:
                    _, tokens = tokenparser(bs)
                except ValueError as e:
                    raise CreationError(*e.args)
                if tokens:
                    b._append(Bits._init_with_token(*tokens[0]))
                    b._datastore = offsetcopy(b._datastore, offset)
                    for token in tokens[1:]:
                        b._append(Bits._init_with_token(*token))
                assert b._assertsanity()
                assert b.len == 0 or b._offset == offset
                if len(cache) < CACHE_SIZE:
                    cache[(bs, offset)] = b
                return b
        except TypeError:
            # Unhashable type
            pass
        return cls(bs)
1106698
def from_string(cls, string):
        
        fields = string.split(None, 2)
        if len(fields) < 2:
            msg = "Not enough fields for public blob: {}"
            raise ValueError(msg.format(fields))
        key_type = fields[0]
        key_blob = decodebytes(b(fields[1]))
        try:
            comment = fields[2].strip()
        except IndexError:
            comment = None
        # Verify that the blob message first (string) field matches the
        # key_type
        m = Message(key_blob)
        blob_type = m.get_text()
        if blob_type != key_type:
            deets = "key type={!r}, but blob type={!r}".format(
                key_type, blob_type
            )
            raise ValueError("Invalid PublicBlob contents: {}".format(deets))
        # All good? All good.
        return cls(type_=key_type, blob=key_blob, comment=comment)