Initialize the integer encoder and the one-hot encoder, fitting them to the vocabulary
        of the corpus.

        NB:
        From here on out,
            - 'ie' stands for 'integer encoded', and
            - 'ohe' stands for 'one-hot encoded'
179161
def init_encoders(self, config=DataConfig()):
        
        self.log('info', 'Initializing the encoders ...')

        # create the integer encoder and fit it to our corpus' vocab
        self.ie = LabelEncoder()
        self.ie_vocab = self.ie.fit_transform(self.vocab_list)

        self.pad_u_index = self.ie.transform([self.pad_u])[0]

        # create the OHE encoder and fit it to our corpus' vocab
        self.ohe = OneHotEncoder(sparse=False)
        self.ohe_vocab = self.ohe.fit_transform(self.ie_vocab.reshape(len(self.ie_vocab), 1))

        return
390476
def init(self, fle=None):
        
        if fle is None:
            fname = self.kwargs.get('gmpe_table', self.GMPE_TABLE)
            if fname is None:
                raise ValueError('You forgot to set GMPETable.GMPE_TABLE!')
            elif os.path.isabs(fname):
                self.GMPE_TABLE = fname
            else:
                # NB: (hackish) GMPE_DIR must be set externally
                self.GMPE_TABLE = os.path.abspath(
                    os.path.join(self.GMPE_DIR, fname))
            fle = h5py.File(self.GMPE_TABLE, "r")
        try:
            # this is the format inside the datastore
            self.distance_type = fle["distance_type"].value
        except KeyError:
            # this is the original format outside the datastore
            self.distance_type = decode(fle["Distances"].attrs["metric"])
        self.REQUIRES_DISTANCES = set([self.distance_type])
        # Load in magnitude
        self.m_w = fle["Mw"][:]
        # Load in distances
        self.distances = fle["Distances"][:]
        # Load intensity measure types and levels
        self.imls = hdf_arrays_to_dict(fle["IMLs"])
        self.DEFINED_FOR_INTENSITY_MEASURE_TYPES = set(self._supported_imts())
        if "SA" in self.imls and "T" not in self.imls:
            raise ValueError("Spectral Acceleration must be accompanied by "
                             "periods")
        # Get the standard deviations
        self._setup_standard_deviations(fle)
        if "Amplification" in fle:
            self._setup_amplification(fle)
1275105
def prepare_question_encoder(inputs, hparams):
  
  encoder_input = inputs
  # Usual case - not a packed dataset.
  encoder_padding = common_attention.embedding_to_padding(encoder_input)
  ignore_padding = common_attention.attention_bias_ignore_padding(
      encoder_padding)
  encoder_self_attention_bias = ignore_padding
  if hparams.pos == "timing":
    encoder_input = common_attention.add_timing_signal_1d(encoder_input)
  elif hparams.pos == "emb":
    encoder_input = common_attention.add_positional_embedding(
        encoder_input, hparams.max_length, "inputs_positional_embedding",
        None)
  return (encoder_input, encoder_self_attention_bias)
769053
function encoding(rtype, value, conf) {
  var enc;

  //console.dir('rtype: ' + rtype);
  //console.dir(value);

  switch(rtype) {
    case RTYPE.STRING:
      enc = ENCODING.RAW;
      // integer encoding
      if(typeof value === 'number'
        && parseInt(value) === value
        || (typeof value === 'string'
          && /^-?\d+$/.test(value))
        || (value instanceof Buffer) && isIntegerBuffer(value)) {
        enc = ENCODING.INT;
      }
      break;
    case RTYPE.LIST:
      // no ziplist support yet
      enc = ENCODING.LINKEDLIST;
      break;
    case RTYPE.SET:
      // no intset support yet
      enc = ENCODING.HASHTABLE;
      break;
    case RTYPE.HASH:
      // no zipmap support yet
      enc = ENCODING.HASHTABLE;
      break;
    case RTYPE.ZSET:
      // no ziplist support yet
      enc = ENCODING.SKIPLIST;
      break;
  }

  return enc;
}
64250
def decode(cls, byteIter):
        
        iei = next(byteIter)
        ieLen = next(byteIter)
        ieData = []
        for i in xrange(ieLen):
            ieData.append(next(byteIter))
        return InformationElement(iei, ieLen, ieData)
265059
def assign(self, value, termenc):
        
        if self._termenc != termenc:
            self._decoder = codecs.getincrementaldecoder(termenc)(errors='replace')
            self._termenc = termenc
        self._data = self._decoder.decode(value)
802501
def calculate_one_hot_encoder_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_numbers(operator, input_count_range=1, output_count_range=1)

    if operator.inputs[0].type.shape[1] != 1 or len(operator.inputs[0].type.shape) > 2:
        raise RuntimeError('Input must be [N, 1]-tensor')

    int_categories = operator.raw_operator.oneHotEncoder.int64Categories.vector
    str_categories = operator.raw_operator.oneHotEncoder.stringCategories.vector

    N = operator.inputs[0].type.shape[0]

    if len(int_categories) > 0:
        operator.outputs[0].type = FloatTensorType([N, len(int_categories)],
                                                   doc_string=operator.outputs[0].type.doc_string)
    elif len(str_categories) > 0 and type(operator.inputs[0].type) == StringTensorType:
        operator.outputs[0].type = FloatTensorType([N, len(str_categories)],
                                                   doc_string=operator.outputs[0].type.doc_string)
    else:
        raise ValueError('Categorical indexes are missing')
795392
def convert_examples_to_features(examples, label_list, max_seq_length,
                                 tokenizer, output_mode):
    """"""

    label_map = {label : i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d of %d" % (ex_index, len(examples)))

        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambiguously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        segment_ids = [0] * len(tokens)

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            segment_ids += [1] * (len(tokens_b) + 1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        if output_mode == "classification":
            label_id = label_map[example.label]
        elif output_mode == "regression":
            label_id = float(example.label)
        else:
            raise KeyError(output_mode)

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % (example.guid))
            logger.info("tokens: %s" % " ".join(
                    [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
            logger.info("label: %s (id = %d)" % (example.label, label_id))

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_mask=input_mask,
                              segment_ids=segment_ids,
                              label_id=label_id))
    return features
792707
def _position_encoding_init(max_length, dim):
    
    position_enc = np.arange(max_length).reshape((-1, 1)) \
                   / (np.power(10000, (2. / dim) * np.arange(dim).reshape((1, -1))))
    # Apply the cosine to even columns and sin to odds.
    position_enc[:, 0::2] = np.sin(position_enc[:, 0::2])  # dim 2i
    position_enc[:, 1::2] = np.cos(position_enc[:, 1::2])  # dim 2i+1
    return position_enc
433399
private void initParameters()  {
		
		int idx = requestPath.indexOf("?");
		if(idx != -1){
			
			String strParams = null;
			try {
				strParams = URLDecoder.decode(requestPath.substring(idx+1), "UTF-8");
			} catch (UnsupportedEncodingException neverHappens) {
				/*URLEncoder:how not to use checked exceptions...*/
				throw new RuntimeException("Something went unexpectedly wrong while decoding a URL for a generator. ",
											neverHappens);
			}
			
			String[] params = strParams.split("&");
			for (int i = 0; i < params.length; i++) {
				String[] param = params[i].split("=");
				parameters.put(param[0], param[1]);
			}
		}else{ // No parameters
			parameters.clear();
		}
	}
411513
def _init_decoder(self):
        
        # Note: content-encoding value should be case-insensitive, per RFC 7230
        # Section 3.2
        content_encoding = self.headers.get('content-encoding', '').lower()
        if self._decoder is None and content_encoding in self.CONTENT_DECODERS:
            self._decoder = _get_decoder(content_encoding)
971271
function TextEncoder(label, options) {
    // Web IDL conventions
    if (!(this instanceof TextEncoder))
      throw TypeError('Called as a function. Did you forget \'new\'?');
    options = ToDictionary(options);

    // A TextEncoder object has an associated encoding and encoder.

    /** @private */
    this._encoding = null;
    /** @private @type {?Encoder} */
    this._encoder = null;

    // Non-standard
    /** @private @type {boolean} */
    this._do_not_flush = false;
    /** @private @type {string} */
    this._fatal = Boolean(options['fatal']) ? 'fatal' : 'replacement';

    // 1. Let enc be a new TextEncoder object.
    var enc = this;

    // 2. Set enc's encoding to UTF-8's encoder.
    if (Boolean(options['NONSTANDARD_allowLegacyEncoding'])) {
      // NONSTANDARD behavior.
      label = label !== undefined ? String(label) : DEFAULT_ENCODING;
      var encoding = getEncoding(label);
      if (encoding === null || encoding.name === 'replacement')
        throw RangeError('Unknown encoding: ' + label);
      if (!encoders[encoding.name]) {
        throw Error('Encoder not present.' +
                    ' Did you forget to include encoding-indexes.js?');
      }
      enc._encoding = encoding;
    } else {
      // Standard behavior.
      enc._encoding = getEncoding('utf-8');

      if (label !== undefined && 'console' in global) {
        console.warn('TextEncoder constructor called with encoding label, '
                     + 'which is ignored.');
      }
    }

    // For pre-ES5 runtimes:
    if (!Object.defineProperty)
      this.encoding = enc._encoding.name.toLowerCase();

    // 3. Return enc.
    return enc;
  }
341211
function onLanguageInitialized(sLanguage, resolve, hyphenateMethod) {
		oHyphenateMethods[sLanguage] = hyphenateMethod;
		oHyphenationInstance.bIsInitialized = true;
		if (aLanguagesQueue.length > 0) {
			aLanguagesQueue.forEach(function (oElement) {
				initializeLanguage(oElement.sLanguage, oElement.oConfig, oElement.resolve);
			});
			aLanguagesQueue = [];
		}
		oHyphenationInstance.bLoading = false;
		resolve(
			getLanguageFromPattern(sLanguage)
		);
	}
740058
function encodeXREF( encoder, id ) {

	// Write opcode for 16-bit lookup 
	encoder.log(LOG.XREF, "xref="+id+" [" + encoder.stringLookup[id] + "]")
	encoder.stream8.write( pack1b( PRIM_OP.IMPORT, false ) );
	encoder.stream16.write( pack2b( id, false ) );
	encoder.counters.op_xref+=3;

}
1333989
def inet_aton(text):
    

    #
    # Our aim here is not something fast; we just want something that works.
    #

    if text == '::':
        text = '0::'
    #
    # Get rid of the icky dot-quad syntax if we have it.
    #
    m = _v4_ending.match(text)
    if not m is None:
        text = "%s:%04x:%04x" % (m.group(1),
                                 int(m.group(2)) * 256 + int(m.group(3)),
                                 int(m.group(4)) * 256 + int(m.group(5)))
    #
    # Try to turn '::<whatever>' into ':<whatever>'; if no match try to
    # turn '<whatever>::' into '<whatever>:'
    #
    m = _colon_colon_start.match(text)
    if not m is None:
        text = text[1:]
    else:
        m = _colon_colon_end.match(text)
        if not m is None:
            text = text[:-1]
    #
    # Now canonicalize into 8 chunks of 4 hex digits each
    #
    chunks = text.split(':')
    l = len(chunks)
    if l > 8:
        raise dns.exception.SyntaxError
    seen_empty = False
    canonical = []
    for c in chunks:
        if c == '':
            if seen_empty:
                raise dns.exception.SyntaxError
            seen_empty = True
            for i in xrange(0, 8 - l + 1):
                canonical.append('0000')
        else:
            lc = len(c)
            if lc > 4:
                raise dns.exception.SyntaxError
            if lc != 4:
                c = ('0' * (4 - lc)) + c
            canonical.append(c)
    if l < 8 and not seen_empty:
        raise dns.exception.SyntaxError
    text = ''.join(canonical)

    #
    # Finally we can go to binary.
    #
    try:
        return text.decode('hex_codec')
    except TypeError:
        raise dns.exception.SyntaxError
50754
def initialize(self, useRandomEncoder):
    
    self.setRandomSeed(self.seed)

    self.dim = numpy.shape(self.spatialConfig)[-1]

    self.spatialMap = dict( zip( map(tuple, list(self.spatialConfig)),
                              self.sensoryInputElements))

    self.lengthMotorInput1D = (2*self.maxDisplacement + 1) * \
                                                    self.numActiveBitsMotorInput

    uniqueSensoryElements = list(set(self.sensoryInputElementsPool))

    if useRandomEncoder:
      self.sensoryEncoder = SDRCategoryEncoder(n=1024,
                                w=self.numActiveBitsSensoryInput,
                                categoryList=uniqueSensoryElements,
                                forced=True)
      self.lengthSensoryInput = self.sensoryEncoder.getWidth()

    else:
      self.lengthSensoryInput = (len(self.sensoryInputElementsPool)+1) * \
                                          self.numActiveBitsSensoryInput

      self.sensoryEncoder = CategoryEncoder(w=self.numActiveBitsSensoryInput,
                            categoryList=uniqueSensoryElements, forced=True)

    motorEncoder1D =  ScalarEncoder(n=self.lengthMotorInput1D,
                                    w=self.numActiveBitsMotorInput,
                                    minval=-self.maxDisplacement,
                                    maxval=self.maxDisplacement,
                                    clipInput=True,
                                    forced=True)

    self.motorEncoder = VectorEncoder(length=self.dim, encoder=motorEncoder1D)
193323
def _init_decoder(self):
        
        # Note: content-encoding value should be case-insensitive, per RFC 7230
        # Section 3.2
        content_encoding = self.headers.get('content-encoding', '').lower()
        if self._decoder is None:
            if content_encoding in self.CONTENT_DECODERS:
                self._decoder = _get_decoder(content_encoding)
            elif ',' in content_encoding:
                encodings = [e.strip() for e in content_encoding.split(',') if e.strip() in self.CONTENT_DECODERS]
                if len(encodings):
                    self._decoder = _get_decoder(content_encoding)