Checks that if a variable exists for the time series id it has the appropriate attributes

        :param netCDF4.Dataset dataset: An open netCDF dataset
760040
def check_timeseries_id(self, dataset):
        '''
        
        '''
        timeseries_ids = dataset.get_variables_by_attributes(cf_role='timeseries_id')
        # No need to check
        if not timeseries_ids:
            return
        test_ctx = TestCtx(BaseCheck.MEDIUM, 'Recommended attributes for the timeSeries variable')
        timeseries_variable = timeseries_ids[0]
        test_ctx.assert_true(
            getattr(timeseries_variable, 'long_name', '') != "",
            "long_name attribute should exist and not be empty"
        )
        return test_ctx.to_result()
625428
def check_dataset(dataset, mode):
  """"""

  names = [x['name'] for x in dataset.schema]
  types = [x['type'] for x in dataset.schema]
  if mode == 'train':
    if (set(['image_url', 'label']) != set(names) or any(t != 'STRING' for t in types)):
      raise ValueError('Invalid dataset. Expect only "image_url,label" STRING columns.')
  else:
    if (set(['image_url']) != set(names) and set(['image_url', 'label']) != set(names)) or \
            any(t != 'STRING' for t in types):
      raise ValueError('Invalid dataset. Expect only "image_url" or "image_url,label" ' +
                       'STRING columns.')
586819
def update_dataset(self,dataStr,flatten=False):
        '''
        
        '''

        #Load keys and dimensions
        #########################
        dataDim = dataStr.pop('_dimensions',{})
        attrStr = dataStr.pop('_attributes',{})
        ndims = dataDim.pop('_ndims',0)
        dimensions = [dataDim.keys(),dataDim.values()]
        
        keys = dataStr.keys()
        if len(keys) == 0:
            self.warning(2, 'No data loaded')
            return
        self.message(2, 'Loaded variables : '+str(keys))
        
        #Check what is the current variable type
        isStructure = True if isinstance(dataStr[keys[0]],dict) else False
        
#        datalen = [np.size(dataStr[key]) for key in keys]
        datalen = [list(np.shape(dataStr[key]['data'])[::-1]) for key in keys] if isStructure else [list(np.shape(dataStr[key])[::-1]) for key in keys] #Shape is inverted wrt to order of dimensions to be consistent with check_variable
        if isStructure :
            varDim = [list(dataStr[key]['_dimensions'])[1:] for key in keys]
            ind = [where_list(vDim,dimensions[0]) for vDim in varDim] #Dimensions indices from actual variables' dimensions
            
            #Check dimension lengths
#             dimOk = np.array([enum[1][0] == dimensions[1][ind[enum[0]][0]] for enum in enumerate(datalen)])
            dimOk = [any([enum[1][ii] == dimensions[1][jj] for ii,jj in enumerate(ind[enum[0]])]) for enum in enumerate(datalen)]
            

            if any([not d for d in dimOk]) : 
                notOk = np.where(~np.array(dimOk))[0]
                print datalen
                
                self.Error('Problem with {0} variables : {1}'.format(len(notOk),','.join(np.array(dataStr.keys())[notOk])))
        else :
            ind = [where_list(dlen,dimensions[1]) for dlen in datalen] #Dimensions indices from variable length
            if (np.array(ind).sum() == -1)!= 0 : self.Error('At least one variable have not been properly defined')
            
        dimname = [np.array(dimensions[0])[i].tolist() for i in ind]  #Get correspondance between data structure dimensions and variables
                
        curDim, nself=self.get_currentDim()
        createDim=np.array([np.array([w == -1 for w in where_list(j, curDim[0])]) for i,j in enumerate(dimname) ])
        createDim=np.squeeze(createDim)
#        curInd = atools.where_list(dimname_reduced,curDim[0]) #Get correspondance between data structure dimensions and object dimensions
        
#        createDim = (np.array(curInd) == -1) #Get dimensions to be created   
        toCreate = np.array([not self.__dict__.has_key(key) for key in keys])
        
        updateDim=[]  
        
        self.message(2, 'Updating object with '+str(['{0}({1}:{2})'.format(i[0],i[1],i[2]) for i in zip(*(keys,dimname,datalen))]))
        
        #Update variables available in files
        for enum in enumerate(keys) :
            
            ind=enum[0]
            key=enum[1]
            
            #Load variable
            ##############
#            var=dataStr.get(key)
            dum=dataStr.get(key).pop('data') if isStructure else copy.deepcopy(dataStr.get(key)) 
            if flatten :
                if isinstance(dum,dict) :dum['data']=dum['data'].flatten()
                else : dum=dum.flatten()
            
            if not isStructure :
                dum={'_dimensions':dum._dimensions if hasattr(dum,'_dimensions') else {},
                      '_attributes':dum._attributes if hasattr(dum,'_attributes') else {},
                      'data':dum}
            else :
                dumStr=dataStr.get(key)
                dumStr.update({'data':dum})
                dum=dumStr

            dumDim=dimStr(dimname[ind],datalen[ind])

#             if dataStr[key].has_key('_attributes'):
#                 dum.update(dataStr[key]['_attributes'])
#            if isinstance(dum,np.ma.masked_array) : 
            
#            #Get associated dimensions
#            ##################################
#            datalen = datalen[ind]#[len(dataStr[key]) for key in keys]
#            ind = atools.where_list([datalen],dimensions[1])[0]
#            if (ind == -1) : self.Error('Dimensions of current variable ('+key+') have not been properly defined')
#            dimname = dimensions[
            
            #Initialize variable if required
#            if toCreate :
#            updateDim.append(self.create_Variable(key, dum, dimensions={dimname[ind]:datalen[ind]},toCreate=toCreate[ind],createDim=createDim[ind]))
            updateDim.append(self.create_Variable(key, dum, dimensions=dumDim,toCreate=toCreate[ind],createDim=createDim[ind]))

        
        #Extend missing variables
#         missing__keys = list(set(self.par_list).difference(keys))
#         for enum in enumerate(missing__keys) :
#             ind=enum[0]
#             key=enum[1]
#             updateDim.append(self.create_Variable(key, np.ma.repeat(self.dist_to_coast_leuliette.fill_value), dimensions=dumDim,toCreate=False,createDim=False))
        
        
        #Final sequence
        
        zipped_upd=zip(*(np.hstack(dimname)[~np.hstack(createDim)],np.hstack(datalen)[~np.hstack(createDim)]))
        updateDim_List = np.array(list(set(tuple(i) for i in np.array(zipped_upd,dtype='|S16').tolist()))) #2D unique
#        updateDim_List = np.unique(np.array(zipped_upd,dtype='|S16')) #[str(i) for i in datalen]
#        if updateDim_List.size > 0 : updateDim_List.resize((2,updateDim_List.size/2))
#        updateDim_List = np.unique(zip(*(np.array(dimname)[~createDim],np.array(datalen)[~createDim]))) #[str(i) for i in datalen]

        zipped_dims=zip(*(np.hstack(dimname)[np.hstack(createDim)],np.hstack(datalen)[np.hstack(createDim)]))
        createDim_list = np.array(list(set(tuple(i) for i in np.array(zipped_dims,dtype='|S16').tolist()))) #2D unique
#        clist, inv = np.unique(np.array(zipped_dims,dtype='|S16'),return_inverse=True) #RQ : THIS WILL FAIL IF NUMBERS HAVE MORE THAN 16 DIGITS #[str(i) for i in datalen]
#        if createDim_list.size > 0 : createDim_list.resize((2,createDim_list.size/2))
        
#        createDim_list = np.unique(zip(*(np.array(dimname)[createDim],np.array(datalen)[createDim]))) #[str(i) for i in datalen]
        
        for dname,dim in createDim_list :
            self.create_Dim(dname, np.int(dim))
        
        for dname,dim in updateDim_List:
            self.update_Dim(dname, np.int(dim))
185679
def evaluate(self, dataset):
        
        if not isinstance(dataset, DataFrame):
            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
        java_glr_summary = self._call_java("evaluate", dataset)
        return GeneralizedLinearRegressionSummary(java_glr_summary)
1128622
def _raise_error_if_not_sarray(dataset, variable_name="SArray"):
    
    err_msg = "Input %s is not an SArray."
    if not isinstance(dataset, _SArray):
      raise ToolkitError(err_msg % variable_name)
241896
def data_validation(self, in_nc):
        """"""
        data_nc = Dataset(in_nc)

        dims = list(data_nc.dimensions)

        if dims not in self.dims_oi:
            data_nc.close()
            raise Exception("{0} {1}".format(self.error_messages[1], dims))

        nc_vars = list(data_nc.variables)

        if nc_vars == self.vars_oi[0]:
            self.runoff_vars = [self.vars_oi[0][-1]]
        elif nc_vars == self.vars_oi[1]:
            self.runoff_vars = [self.vars_oi[1][-1]]
        else:
            data_nc.close()
            raise Exception("{0} {1}".format(self.error_messages[2], nc_vars))
        data_nc.close()
853358
def dataset_exists(dataset_name):
    ''''''
    dataset_dir = os.path.join(LIB_DIR, 'datasets')
    dataset_path = os.path.join(dataset_dir, dataset_name)
    
    return dataset_path if os.path.isdir(dataset_path) else None
947129
def verify(self, dataset, publication_date, source, refernce_url):
        """"""

        path = '/api/1.0/meta/verifydataset'
        req = definition.DatasetVerifyRequest(dataset, publication_date, source, refernce_url)
        result = self._api_post(definition.DatasetVerifyResponse, path, req)
        if result.status == 'failed':
            ver_err = '\r\n'.join(result.errors)
            msg = 'Dataset has not been verified, because of the following error(s): {}'.format(ver_err)
            raise ValueError(msg)
1269058
def exists(self):
        """"""
        result = TimeSeries(default=False if self.default is None else True)
        for t, v in self:
            result[t] = False if v is None else True
        return result
813785
def _validate_dataset(dataset):
    
    if not (isinstance(dataset, _SFrame)):
        raise TypeError("Input 'dataset' must be an SFrame.")

    if dataset.num_rows() == 0 or dataset.num_columns() == 0:
        raise ValueError("Input 'dataset' has no data.")
317815
def evaluate(self, dataset):
        
        if not isinstance(dataset, DataFrame):
            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
        java_blr_summary = self._call_java("evaluate", dataset)
        return BinaryLogisticRegressionSummary(java_blr_summary)
419912
def validate_dataset_string(self, dataset):
        
        if dataset:
            if '/' not in dataset:
                raise ValueError('Dataset must be specified in the form of '
                                 '\'{username}/{dataset-slug}\'')

            split = dataset.split('/')
            if not split[0] or not split[1]:
                raise ValueError('Invalid dataset specification ' + dataset)
510149
def addDataset(self, dataset):
        
        item = XChartDatasetItem()
        self.addItem(item)
        item.setDataset(dataset)
        return item
637483
def create(self, dataset_id):
        
        from google.cloud.bigquery import Dataset

        if self.exists(dataset_id):
            raise DatasetCreationError(
                "Dataset {0} already " "exists".format(dataset_id)
            )

        dataset = Dataset(self.client.dataset(dataset_id))

        if self.location is not None:
            dataset.location = self.location

        try:
            self.client.create_dataset(dataset)
        except self.http_error as ex:
            self.process_http_error(ex)
1316746
def data_available(dataset_name=None):
    """"""
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    dr = data_resources[dataset_name]
    zip_urls = (dr['files'], )
    if 'save_names' in dr: zip_urls += (dr['save_names'], )
    else: zip_urls += ([],)

    for file_list, save_list in zip_longest(*zip_urls, fillvalue=[]):
        for f, s in zip_longest(file_list, save_list, fillvalue=None):
            if s is not None: f=s # If there is a save_name given, use that one
            if not os.path.exists(os.path.join(data_path, dataset_name, f)):
                return False
    return True
879198
def evaluate_with_predictions(data_file, predictions):
    '''
    
    '''
    expected_version = '1.1'
    with open(data_file) as dataset_file:
        dataset_json = json.load(dataset_file)
        if dataset_json['version'] != expected_version:
            print('Evaluation expects v-' + expected_version +
                  ', but got dataset with v-' + dataset_json['version'],
                  file=sys.stderr)
        dataset = dataset_json['data']
    result = _evaluate(dataset, predictions)
    return result['exact_match']
1185168
def std_check_in(dataset, name, allowed_vals):
    
    if not hasattr(dataset, name):
        return 0

    ret_val = 1
    if getattr(dataset, name) in allowed_vals:
        ret_val += 1

    return ret_val