Old format taking DataDescriptor
345745
public static WpdCollection get(DataDescriptor dd, boolean recursive) {
        if (disabled)
            return null;
        return getStats(findModuleItem(dd), recursive);
    }
1140623
function parseField(identifier, descriptor) {
    switch(identifier) {
    // Literal (uninterpreted) fields
    case 'content':
    case 'number':
    case 'sampleUnits':
        break;
    // Integers
    case 'dimension':
    case 'blockSize':
    case 'lineSkip':
    case 'byteSkip':
    case 'spaceDimension':
        descriptor = parseNRRDInteger(descriptor);
        break;
    // Floats
    case 'min':
    case 'max':
    case 'oldMin':
    case 'oldMax':
        descriptor = parseNRRDFloat(descriptor);
        break;
    // Vectors
    case 'spaceOrigin':
        descriptor = parseNRRDVector(descriptor);
        break;
    // Lists of strings
    case 'labels':
    case 'units':
    case 'spaceUnits':
        descriptor = parseNRRDWhitespaceSeparatedList(descriptor, parseNRRDQuotedString);
        break;
    // Lists of integers
    case 'sizes':
        descriptor = parseNRRDWhitespaceSeparatedList(descriptor, parseNRRDInteger);
        break;
    // Lists of floats
    case 'spacings':
    case 'thicknesses':
    case 'axisMins':
    case 'axisMaxs':
        descriptor = parseNRRDWhitespaceSeparatedList(descriptor, parseNRRDFloat);
        break;
    // Lists of vectors
    case 'spaceDirections':
    case 'measurementFrame':
        descriptor = parseNRRDWhitespaceSeparatedList(descriptor, parseNRRDVector);
        break;
    // One-of-a-kind fields
    case 'type':
        descriptor = parseNRRDType(descriptor);
        break;
    case 'encoding':
        descriptor = parseNRRDEncoding(descriptor);
        break;
    case 'endian':
        descriptor = parseNRRDEndian(descriptor);
        break;
    case 'dataFile':
        descriptor = parseNRRDDataFile(descriptor);
        break;
    case 'centers':
        descriptor = parseNRRDWhitespaceSeparatedList(descriptor, parseNRRDCenter);
        break;
    case 'kinds':
        descriptor = parseNRRDWhitespaceSeparatedList(descriptor, parseNRRDKind);
        break;
    case 'space':
        descriptor = parseNRRDSpace(descriptor);
        break;
    // Something unknown
    default:
        console.warn("Unrecognized NRRD field: " + identifier);
    }
    return descriptor;
}
321217
def _parse_data_desc(data_names, label_names, data_shapes, label_shapes):
    """"""
    data_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in data_shapes]
    _check_names_match(data_names, data_shapes, 'data', True)
    if label_shapes is not None:
        label_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in label_shapes]
        _check_names_match(label_names, label_shapes, 'label', False)
    else:
        _check_names_match(label_names, [], 'label', False)
    return data_shapes, label_shapes
220988
def graph_format(new_mem, old_mem, is_firstiteration=True):
    """"""
    if is_firstiteration:
        output = "  n/a   "
    elif new_mem - old_mem > 50000000:
        output = "   +++++"
    elif new_mem - old_mem > 20000000:
        output = "   ++++ "
    elif new_mem - old_mem > 5000000:
        output = "   +++  "
    elif new_mem - old_mem > 1000000:
        output = "   ++   "
    elif new_mem - old_mem > 50000:
        output = "   +    "
    elif old_mem - new_mem > 10000000:
        output = "---     "
    elif old_mem - new_mem > 2000000:
        output = " --     "
    elif old_mem - new_mem > 100000:
        output = "  -     "
    else:
        output = "        "
    return output
551807
def AdjustDescriptor(self, fields):
    """"""

    for f in fields:
      if f.name == "args_rdf_name":
        f.name = "payload_type"

      if f.name == "args":
        f.name = "payload"

    return fields
823323
def _ConvertDataType(self, dataType):
      
      if dataType:
         vmodlName = dataType.name
         wsdlName = dataType.wsdlName
         version = dataType.version
         parent = dataType.base[0]
         props = self._Filter(self._ConvertDataPropertyType, dataType.property)
         doType = (vmodlName, wsdlName, parent, version, props)
      else:
         doType = None
      return doType
1099242
def deserialize_dtype(d):
    
    if isinstance(d['descr'], six.string_types):
        return np.dtype(d['descr'])
    descr = []
    for col in d['descr']:
        col_descr = []
        for c in col:
            if isinstance(c, six.string_types):
                col_descr.append(str(c))
            elif type(c) is list:
                col_descr.append(tuple(c))
            else:
                col_descr.append(c)
        descr.append(tuple(col_descr))
    return np.dtype(descr)
586819
def update_dataset(self,dataStr,flatten=False):
        '''
        
        '''

        #Load keys and dimensions
        #########################
        dataDim = dataStr.pop('_dimensions',{})
        attrStr = dataStr.pop('_attributes',{})
        ndims = dataDim.pop('_ndims',0)
        dimensions = [dataDim.keys(),dataDim.values()]
        
        keys = dataStr.keys()
        if len(keys) == 0:
            self.warning(2, 'No data loaded')
            return
        self.message(2, 'Loaded variables : '+str(keys))
        
        #Check what is the current variable type
        isStructure = True if isinstance(dataStr[keys[0]],dict) else False
        
#        datalen = [np.size(dataStr[key]) for key in keys]
        datalen = [list(np.shape(dataStr[key]['data'])[::-1]) for key in keys] if isStructure else [list(np.shape(dataStr[key])[::-1]) for key in keys] #Shape is inverted wrt to order of dimensions to be consistent with check_variable
        if isStructure :
            varDim = [list(dataStr[key]['_dimensions'])[1:] for key in keys]
            ind = [where_list(vDim,dimensions[0]) for vDim in varDim] #Dimensions indices from actual variables' dimensions
            
            #Check dimension lengths
#             dimOk = np.array([enum[1][0] == dimensions[1][ind[enum[0]][0]] for enum in enumerate(datalen)])
            dimOk = [any([enum[1][ii] == dimensions[1][jj] for ii,jj in enumerate(ind[enum[0]])]) for enum in enumerate(datalen)]
            

            if any([not d for d in dimOk]) : 
                notOk = np.where(~np.array(dimOk))[0]
                print datalen
                
                self.Error('Problem with {0} variables : {1}'.format(len(notOk),','.join(np.array(dataStr.keys())[notOk])))
        else :
            ind = [where_list(dlen,dimensions[1]) for dlen in datalen] #Dimensions indices from variable length
            if (np.array(ind).sum() == -1)!= 0 : self.Error('At least one variable have not been properly defined')
            
        dimname = [np.array(dimensions[0])[i].tolist() for i in ind]  #Get correspondance between data structure dimensions and variables
                
        curDim, nself=self.get_currentDim()
        createDim=np.array([np.array([w == -1 for w in where_list(j, curDim[0])]) for i,j in enumerate(dimname) ])
        createDim=np.squeeze(createDim)
#        curInd = atools.where_list(dimname_reduced,curDim[0]) #Get correspondance between data structure dimensions and object dimensions
        
#        createDim = (np.array(curInd) == -1) #Get dimensions to be created   
        toCreate = np.array([not self.__dict__.has_key(key) for key in keys])
        
        updateDim=[]  
        
        self.message(2, 'Updating object with '+str(['{0}({1}:{2})'.format(i[0],i[1],i[2]) for i in zip(*(keys,dimname,datalen))]))
        
        #Update variables available in files
        for enum in enumerate(keys) :
            
            ind=enum[0]
            key=enum[1]
            
            #Load variable
            ##############
#            var=dataStr.get(key)
            dum=dataStr.get(key).pop('data') if isStructure else copy.deepcopy(dataStr.get(key)) 
            if flatten :
                if isinstance(dum,dict) :dum['data']=dum['data'].flatten()
                else : dum=dum.flatten()
            
            if not isStructure :
                dum={'_dimensions':dum._dimensions if hasattr(dum,'_dimensions') else {},
                      '_attributes':dum._attributes if hasattr(dum,'_attributes') else {},
                      'data':dum}
            else :
                dumStr=dataStr.get(key)
                dumStr.update({'data':dum})
                dum=dumStr

            dumDim=dimStr(dimname[ind],datalen[ind])

#             if dataStr[key].has_key('_attributes'):
#                 dum.update(dataStr[key]['_attributes'])
#            if isinstance(dum,np.ma.masked_array) : 
            
#            #Get associated dimensions
#            ##################################
#            datalen = datalen[ind]#[len(dataStr[key]) for key in keys]
#            ind = atools.where_list([datalen],dimensions[1])[0]
#            if (ind == -1) : self.Error('Dimensions of current variable ('+key+') have not been properly defined')
#            dimname = dimensions[
            
            #Initialize variable if required
#            if toCreate :
#            updateDim.append(self.create_Variable(key, dum, dimensions={dimname[ind]:datalen[ind]},toCreate=toCreate[ind],createDim=createDim[ind]))
            updateDim.append(self.create_Variable(key, dum, dimensions=dumDim,toCreate=toCreate[ind],createDim=createDim[ind]))

        
        #Extend missing variables
#         missing__keys = list(set(self.par_list).difference(keys))
#         for enum in enumerate(missing__keys) :
#             ind=enum[0]
#             key=enum[1]
#             updateDim.append(self.create_Variable(key, np.ma.repeat(self.dist_to_coast_leuliette.fill_value), dimensions=dumDim,toCreate=False,createDim=False))
        
        
        #Final sequence
        
        zipped_upd=zip(*(np.hstack(dimname)[~np.hstack(createDim)],np.hstack(datalen)[~np.hstack(createDim)]))
        updateDim_List = np.array(list(set(tuple(i) for i in np.array(zipped_upd,dtype='|S16').tolist()))) #2D unique
#        updateDim_List = np.unique(np.array(zipped_upd,dtype='|S16')) #[str(i) for i in datalen]
#        if updateDim_List.size > 0 : updateDim_List.resize((2,updateDim_List.size/2))
#        updateDim_List = np.unique(zip(*(np.array(dimname)[~createDim],np.array(datalen)[~createDim]))) #[str(i) for i in datalen]

        zipped_dims=zip(*(np.hstack(dimname)[np.hstack(createDim)],np.hstack(datalen)[np.hstack(createDim)]))
        createDim_list = np.array(list(set(tuple(i) for i in np.array(zipped_dims,dtype='|S16').tolist()))) #2D unique
#        clist, inv = np.unique(np.array(zipped_dims,dtype='|S16'),return_inverse=True) #RQ : THIS WILL FAIL IF NUMBERS HAVE MORE THAN 16 DIGITS #[str(i) for i in datalen]
#        if createDim_list.size > 0 : createDim_list.resize((2,createDim_list.size/2))
        
#        createDim_list = np.unique(zip(*(np.array(dimname)[createDim],np.array(datalen)[createDim]))) #[str(i) for i in datalen]
        
        for dname,dim in createDim_list :
            self.create_Dim(dname, np.int(dim))
        
        for dname,dim in updateDim_List:
            self.update_Dim(dname, np.int(dim))
666355
def primitive_datatype_nostring(self, t: URIRef, v: Optional[str] = None) -> Optional[URIRef]:
        
        vt = self.primitive_datatype(t)
        if self.fhir_dates and vt == XSD.dateTime and v:
            return XSD.gYear if len(v) == 4 else XSD.gYearMonth if len(v) == 7 \
                else XSD.date if (len(v) == 10 or (len(v) > 10 and v[10] in '+-')) else XSD.dateTime
        # For some reason the oid datatype is represented as a string as well
        if self.fhir_oids and vt == XSD.anyURI:
            vt = None
        return None if vt == XSD.string else vt
116938
def _str_desc(self, reader):
        """"""
        data_version = reader.data_version
        if data_version is not None:
            data_version = data_version.replace("releases/", "")
        desc = "{OBO}: fmt({FMT}) rel({REL}) {N:,} GO Terms".format(
            OBO=reader.obo_file, FMT=reader.format_version,
            REL=data_version, N=len(self))
        if reader.optobj:
            desc = "{D}; optional_attrs({A})".format(D=desc, A=" ".join(sorted(reader.optobj.optional_attrs)))
        return desc
436841
def _convert_type(self, data_type):
        '''
        
        '''
        if (data_type == 1) or (data_type == 41):
            dt_string = 'b'
        elif data_type == 2:
            dt_string = 'h'
        elif data_type == 4:
            dt_string = 'i'
        elif (data_type == 8) or (data_type == 33):
            dt_string = 'q'
        elif data_type == 11:
            dt_string = 'B'
        elif data_type == 12:
            dt_string = 'H'
        elif data_type == 14:
            dt_string = 'I'
        elif (data_type == 21) or (data_type == 44):
            dt_string = 'f'
        elif (data_type == 22) or (data_type == 45) or (data_type == 31):
            dt_string = 'd'
        elif (data_type == 32):
            dt_string = 'd'
        elif (data_type == 51) or (data_type == 52):
            dt_string = 's'
        return dt_string
272851
<V> V fromJson(final DataTypeDescriptor<V> descriptor, String value) {
		if (value == null) {
			return null;
		}

		TypeEnum type = descriptor.getType();
		if (type == TypeEnum.STRING || type == TypeEnum.DATETIME || type == TypeEnum.ENUM) {
			if (!value.startsWith("\"") && !value.endsWith("\"")) {
				// Return the quotes to parse a value as valid json strings.
				value = "\"" + value + "\"";
			}
		}

		return format.read(value, descriptor);
	}
1140248
private void decodeDictionaryIds(
      int rowId,
      int num,
      WritableColumnVector column,
      WritableColumnVector dictionaryIds) {
    switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
      case INT32:
        if (column.dataType() == DataTypes.IntegerType ||
            DecimalType.is32BitDecimalType(column.dataType())) {
          for (int i = rowId; i < rowId + num; ++i) {
            if (!column.isNullAt(i)) {
              column.putInt(i, dictionary.decodeToInt(dictionaryIds.getDictId(i)));
            }
          }
        } else if (column.dataType() == DataTypes.ByteType) {
          for (int i = rowId; i < rowId + num; ++i) {
            if (!column.isNullAt(i)) {
              column.putByte(i, (byte) dictionary.decodeToInt(dictionaryIds.getDictId(i)));
            }
          }
        } else if (column.dataType() == DataTypes.ShortType) {
          for (int i = rowId; i < rowId + num; ++i) {
            if (!column.isNullAt(i)) {
              column.putShort(i, (short) dictionary.decodeToInt(dictionaryIds.getDictId(i)));
            }
          }
        } else {
          throw constructConvertNotSupportedException(descriptor, column);
        }
        break;

      case INT64:
        if (column.dataType() == DataTypes.LongType ||
            DecimalType.is64BitDecimalType(column.dataType()) ||
            originalType == OriginalType.TIMESTAMP_MICROS) {
          for (int i = rowId; i < rowId + num; ++i) {
            if (!column.isNullAt(i)) {
              column.putLong(i, dictionary.decodeToLong(dictionaryIds.getDictId(i)));
            }
          }
        } else if (originalType == OriginalType.TIMESTAMP_MILLIS) {
          for (int i = rowId; i < rowId + num; ++i) {
            if (!column.isNullAt(i)) {
              column.putLong(i,
                DateTimeUtils.fromMillis(dictionary.decodeToLong(dictionaryIds.getDictId(i))));
            }
          }
        } else {
          throw constructConvertNotSupportedException(descriptor, column);
        }
        break;

      case FLOAT:
        for (int i = rowId; i < rowId + num; ++i) {
          if (!column.isNullAt(i)) {
            column.putFloat(i, dictionary.decodeToFloat(dictionaryIds.getDictId(i)));
          }
        }
        break;

      case DOUBLE:
        for (int i = rowId; i < rowId + num; ++i) {
          if (!column.isNullAt(i)) {
            column.putDouble(i, dictionary.decodeToDouble(dictionaryIds.getDictId(i)));
          }
        }
        break;
      case INT96:
        if (column.dataType() == DataTypes.TimestampType) {
          if (!shouldConvertTimestamps()) {
            for (int i = rowId; i < rowId + num; ++i) {
              if (!column.isNullAt(i)) {
                Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
                column.putLong(i, ParquetRowConverter.binaryToSQLTimestamp(v));
              }
            }
          } else {
            for (int i = rowId; i < rowId + num; ++i) {
              if (!column.isNullAt(i)) {
                Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
                long rawTime = ParquetRowConverter.binaryToSQLTimestamp(v);
                long adjTime = DateTimeUtils.convertTz(rawTime, convertTz, UTC);
                column.putLong(i, adjTime);
              }
            }
          }
        } else {
          throw constructConvertNotSupportedException(descriptor, column);
        }
        break;
      case BINARY:
        // TODO: this is incredibly inefficient as it blows up the dictionary right here. We
        // need to do this better. We should probably add the dictionary data to the ColumnVector
        // and reuse it across batches. This should mean adding a ByteArray would just update
        // the length and offset.
        for (int i = rowId; i < rowId + num; ++i) {
          if (!column.isNullAt(i)) {
            Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
            column.putByteArray(i, v.getBytes());
          }
        }
        break;
      case FIXED_LEN_BYTE_ARRAY:
        // DecimalType written in the legacy mode
        if (DecimalType.is32BitDecimalType(column.dataType())) {
          for (int i = rowId; i < rowId + num; ++i) {
            if (!column.isNullAt(i)) {
              Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
              column.putInt(i, (int) ParquetRowConverter.binaryToUnscaledLong(v));
            }
          }
        } else if (DecimalType.is64BitDecimalType(column.dataType())) {
          for (int i = rowId; i < rowId + num; ++i) {
            if (!column.isNullAt(i)) {
              Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
              column.putLong(i, ParquetRowConverter.binaryToUnscaledLong(v));
            }
          }
        } else if (DecimalType.isByteArrayDecimalType(column.dataType())) {
          for (int i = rowId; i < rowId + num; ++i) {
            if (!column.isNullAt(i)) {
              Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(i));
              column.putByteArray(i, v.getBytes());
            }
          }
        } else {
          throw constructConvertNotSupportedException(descriptor, column);
        }
        break;

      default:
        throw new UnsupportedOperationException(
          "Unsupported type: " + descriptor.getPrimitiveType().getPrimitiveTypeName());
    }
  }
1161220
@Deprecated
  private DatanodeDescriptor getDatanodeByIndex(int index) {
    int i = 0;
    for (DatanodeDescriptor node : datanodeMap.values()) {
      if (i == index) {
        return node;
      }
      i++;
    }
    return null;
  }
1238577
def simple_separated_format(separator):
    
    return TableFormat(None, None, None, None,
                       headerrow=DataRow('', separator, ''),
                       datarow=DataRow('', separator, ''),
                       padding=0, with_header_hide=None)
542161
private void transformDescriptorIntrospectors(PrintWriter writer) {
		
    	
    	Map<String, RequestProbeTransformDescriptor> registeredTransformDescriptors =  RequestProbeBCIManagerImpl.getRequestProbeTransformDescriptors();
        List<String> transformDescriptorRefs = new ArrayList<String>() {{add("Transform Descriptor");  add("");}};
        List<String> tdDetails = new ArrayList<String>() {{add("ClassName.MethodName(Description)");  add("");}};
        List<Integer> indentationLength = new ArrayList<Integer>();
        int maxSpaceRequired = 0;
       
        for(Entry<String, RequestProbeTransformDescriptor> transformDescriptor : registeredTransformDescriptors.entrySet()) {
        	
        	transformDescriptorRefs.add(transformDescriptor.getValue().toString());
        	if(transformDescriptor.getValue().getMethodDesc().equals("all"))
        		 tdDetails.add(transformDescriptor.getValue().getClassName() + "." + transformDescriptor.getValue().getMethodName() + "("+ transformDescriptor.getValue().getMethodDesc() + ")" );
        	else
        		 tdDetails.add(transformDescriptor.getValue().getClassName() + "." + transformDescriptor.getValue().getMethodName() +  transformDescriptor.getValue().getMethodDesc() );
        }
        
        for(String transformDescriptorRef : transformDescriptorRefs) {
        	if(transformDescriptorRef.length() > maxSpaceRequired) {
        		maxSpaceRequired = transformDescriptorRef.length();
        	}
        }
        indentationLength.add(maxSpaceRequired+ EXTRA_SPACE_REQUIRED);

        writer.println();
        writer.println("------------------------------------------------------------------------------");
        writer.println("              Registered Transform Descriptors ");
        writer.println("------------------------------------------------------------------------------");
        
        for(int i = 0 ; i < transformDescriptorRefs.size(); i ++) {
        	writer.println(String.format("%-" + indentationLength.get(0) + "s%s",transformDescriptorRefs.get(i) ,tdDetails.get(i)));
        }
        
        if(registeredTransformDescriptors.size() ==  0) {
          	 writer.println("----- No transform descriptors are registered ----- ");
           }

	}
626923
def _prototype_to_generator(descriptor, cls):
    ''
    _fd = D.FieldDescriptor
    generator = None

    ints32 = [_fd.TYPE_INT32, _fd.TYPE_UINT32, _fd.TYPE_FIXED32,
              _fd.TYPE_SFIXED32, _fd.TYPE_SINT32]
    ints64 = [_fd.TYPE_INT64, _fd.TYPE_UINT64, _fd.TYPE_FIXED64,
              _fd.TYPE_SFIXED64, _fd.TYPE_SINT64]
    ints_signed = [_fd.TYPE_INT32, _fd.TYPE_SFIXED32, _fd.TYPE_SINT32,
                   _fd.TYPE_INT64, _fd.TYPE_SFIXED64, _fd.TYPE_SINT64]

    if descriptor.type in ints32+ints64:
        bitwidth = [32, 64][descriptor.type in ints64]
        unsigned = descriptor.type not in ints_signed
        generator = _int_generator(descriptor, bitwidth, unsigned)
    elif descriptor.type == _fd.TYPE_DOUBLE:
        generator = _float_generator(descriptor, 64)
    elif descriptor.type == _fd.TYPE_FLOAT:
        generator = _float_generator(descriptor, 32)
    elif descriptor.type == _fd.TYPE_STRING:
        generator = _string_generator(descriptor)
    elif descriptor.type == _fd.TYPE_BYTES:
        generator = _bytes_generator(descriptor)
    elif descriptor.type == _fd.TYPE_BOOL:
        generator = gen.IterValueGenerator(descriptor.name, [True, False])
    elif descriptor.type == _fd.TYPE_ENUM:
        generator = _enum_generator(descriptor)
    elif descriptor.type == _fd.TYPE_MESSAGE:
        generator = descriptor_to_generator(descriptor.message_type, cls)
        generator.set_name(descriptor.name)
    else:
        raise RuntimeError("type {} unsupported".format(descriptor.type))

    return generator