Convert between NOAA and LiPD file formats.
    :return:
178070
def noaa_prompt():
    
    logger_noaa.info("enter noaa")
    # Run lpd_noaa or noaa_lpd ?
    print("Which conversion?\n1. LPD to NOAA\n2. NOAA to LPD\n")
    mode = input("Option: ")
    logger_noaa.info("chose option: {}".format(mode))

    return mode
750270
def convert(self, infile, item=None):
        
        if not os.path.isfile(infile):
            raise IOError('{} is not a valid file'.format(infile))

        if item is None:
            item = os.path.splitext(infile)[0]

        ext = os.path.splitext(infile)[1]
        if ext == '.npz':
            self.npz_convert(infile, item)
        elif ext == '.mat':
            self.mat_convert(infile, item)
        elif ext == '.h5':
            self.h5features_convert(infile)
        else:
            raise IOError('Unknown file format for {}'.format(infile))
42230
def _parse_supl(content):
        
        col_names = ['year', 'month', 'day', 'hour', 'minute',
                     'hourly_low_pressure', 'hourly_low_pressure_time',
                     'hourly_high_wind', 'hourly_high_wind_direction',
                     'hourly_high_wind_time']

        col_units = {'hourly_low_pressure': 'hPa',
                     'hourly_low_pressure_time': None,
                     'hourly_high_wind': 'meters/second',
                     'hourly_high_wind_direction': 'degrees',
                     'hourly_high_wind_time': None,
                     'time': None}

        df = pd.read_table(StringIO(content), comment='#', na_values='MM',
                           names=col_names, sep=r'\s+')

        df['time'] = pd.to_datetime(df[['year', 'month', 'day', 'hour', 'minute']], utc=True)

        df['hours'] = np.floor(df['hourly_low_pressure_time'] / 100)
        df['minutes'] = df['hourly_low_pressure_time'] - df['hours'] * 100
        df['hours'] = df['hours'].replace(99, np.nan)
        df['minutes'] = df['minutes'].replace(99, np.nan)
        df['hourly_low_pressure_time'] = pd.to_datetime(df[['year', 'month', 'day', 'hours',
                                                            'minutes']], utc=True)

        df['hours'] = np.floor(df['hourly_high_wind_time'] / 100)
        df['minutes'] = df['hourly_high_wind_time'] - df['hours'] * 100
        df['hours'] = df['hours'].replace(99, np.nan)
        df['minutes'] = df['minutes'].replace(99, np.nan)
        df['hourly_high_wind_time'] = pd.to_datetime(df[['year', 'month', 'day',
                                                         'hours', 'minutes']], utc=True)
        df = df.drop(columns=['year', 'month', 'day', 'hour', 'minute', 'hours', 'minutes'])
        df.units = col_units
        return df
209220
def convert(self):
        
        try:
            log('Processing %s ...' % self.rapid_output_file_list[0])
            time_start_conversion = datetime.utcnow()

            # Validate the raw netCDF file
            log('validating input netCDF file', 'INFO')
            id_len, time_len = self._validate_raw_nc()

            # Initialize the output file (create dimensions and variables)
            log('initializing output', 'INFO')
            self._initialize_output(time_len, id_len)

            self._generate_time_values()

            # copy river ids over
            self.cf_nc.variables[self.output_id_dim_name][:] = \
                self.raw_nc_list[0].get_river_id_array()

            # Populate comid, lat, lon, z
            log('writing comid lat lon z')
            lookup_start = datetime.now()
            self._write_comid_lat_lon_z()
            duration = str((datetime.now() - lookup_start).total_seconds())
            log('Lookup Duration (s): ' + duration)

            # Create a variable for streamflow. This is big, and slows down
            # previous steps if we do it earlier.
            self._copy_streamflow_values()

            # close files
            for raw_nc in self.raw_nc_list:
                raw_nc.close()
            self.cf_nc.close()

            # delete original RAPID output
            remove_files(*self.rapid_output_file_list)

            # rename nc compliant file to original name
            os.rename(self.cf_compliant_file, self.rapid_output_file_list[0])
            log('Time to process %s' %
                (datetime.utcnow()-time_start_conversion))
        except Exception:
            # delete cf RAPID output
            remove_files(self.cf_compliant_file)
            raise
835996
def mat_convert(self, infile, item):
        """"""
        data = sio.loadmat(infile)
        labels = self._labels(data)
        features = data['features']
        self._write(item, labels, features)
515405
def _format_lon(self, lon):
        '''  '''
        if self.ppd in [4, 16, 64, 128]:
            return None
        else:
            return map(lambda x: "{0:0>3}".format(int(x)), self._map_center('long', lon))
207357
def loads(self, noaa_string):
    '''  '''
    self.raw = noaa_string
    self.weather_station = noaa_string[4:10]
    self.wban = noaa_string[10:15]
    expected_length = int(noaa_string[0:4]) + self.PREAMBLE_LENGTH
    actual_length = len(noaa_string)
    if actual_length != expected_length:
      msg = "Non matching lengths. Expected %d, got %d" % (expected_length,
                                                           actual_length)
      raise ish_reportException(msg)

    try:
      self.datetime = datetime.strptime(noaa_string[15:27], '%Y%m%d%H%M')
    except ValueError:
      ''' some cases, we get 2400 hours, which is really the next day, so 
      this is a workaround for those cases '''
      time = noaa_string[15:27]
      time = time.replace("2400", "2300")
      self.datetime = datetime.strptime(time, '%Y%m%d%H%M')
      self.datetime += timedelta(hours=1)

    self.datetime = self.datetime.replace(tzinfo=pytz.UTC)

    self.report_type = ReportType(noaa_string[41:46].strip())

    self.latitude = float(noaa_string[28:34]) / self.GEO_SCALE
    self.longitude = float(noaa_string[34:41]) / self.GEO_SCALE
    self.elevation = int(noaa_string[46:51])

    ''' other mandatory fields '''
    self.wind_direction = Direction(noaa_string[60:63],
                                    Direction.RADIANS,
                                    noaa_string[63:64])
    self.wind_observation_direction_type = noaa_string[64:64]
    self.wind_speed = Speed(int(noaa_string[65:69]) / float(self.SPEED_SCALE),
                            Speed.METERSPERSECOND,
                            noaa_string[69:70])
    self.sky_ceiling = Distance(int(noaa_string[70:75]),
                                Distance.METERS,
                                noaa_string[75:76])
    self.sky_ceiling_determination = noaa_string[76:77]
    self.visibility_distance = Distance(int(noaa_string[78:84]),
                                        Distance.METERS,
                                        noaa_string[84:85]) 
    self.visibility_variability = noaa_string[85:86]
    self.visibility_variability_quality = noaa_string[86:87]

    self.air_temperature = Temperature(int(noaa_string[87:92]) / self.TEMPERATURE_SCALE,
                                           Units.CELSIUS,
                                           noaa_string[92:93])
    self.dew_point = Temperature(int(noaa_string[93:98]) / self.TEMPERATURE_SCALE,
                                 Units.CELSIUS,
                                 noaa_string[98:99])

    self.humidity = Humidity(str(self.air_temperature), str(self.dew_point))
    self.sea_level_pressure = Pressure(int(noaa_string[99:104])/self.PRESSURE_SCALE,
                                       Pressure.HECTOPASCALS,
                                       noaa_string[104:104])

    ''' handle the additional fields '''
    additional = noaa_string[105:108]
    if additional == 'ADD':
      position = 108
      while position < expected_length:
        try:
          (position, (addl_code, addl_string)) = self._get_component(noaa_string,
                                                                     position)
          self._additional[addl_code] = addl_string
        except ish_reportException as err:
          ''' this catches when we move to remarks section '''
          break

    ''' handle the remarks section if it exists '''
    try:
      position = noaa_string.index('REM', 108) 
      self._get_remarks_component(noaa_string, position)
    except (ish_reportException, ValueError) as err:
      ''' this catches when we move to EQD section '''

    return self
496074
def export(self, nidm_version, export_dir):
        

        # In FSL we have a single thresholding (extent, height) applied to all
        # contrasts
        # FIXME: Deal with two-tailed inference?
        atts = (
            (PROV['type'], self.type),
            (PROV['label'], self.label),
            (NIDM_HAS_ALTERNATIVE_HYPOTHESIS, self.tail))

        if self.partial_degree is not None:
            atts += (
                (SPM_PARTIAL_CONJUNCTION_DEGREE, self.partial_degree),)

        self.add_attributes(atts)
1321768
def lipd_to_df(metadata, csvs):
    
    dfs = {}
    logger_dataframes.info("enter lipd_to_df")

    # Flatten the dictionary, but ignore the chron data items
    dict_in_dotted = {}
    logger_dataframes.info("enter dot_notation")
    _dotnotation_for_nested_dictionary(metadata, '', dict_in_dotted)
    dict_in_dotted = collections.OrderedDict(sorted(dict_in_dotted.items()))

    # Create one data frame for metadata items
    dfs["metadata"] = pd.DataFrame(list(dict_in_dotted.items()), columns=["Key", "Value"])

    # Create data frames for paleo data and chron data items. This does not use LiPD data, it uses the csv data
    dfs.update(_get_dfs(csvs))

    return dfs
434220
def rsdl(self):
        """"""

        diff = self.Xf - self.Yfprv
        return sl.rfl2norm2(diff, self.X.shape, axis=self.cri.axisN)
25431
def get_noalt_contigs(data):
    
    alts = set([])
    alt_files = [f for f in tz.get_in(["reference", "bwa", "indexes"], data, []) if f.endswith("alt")]
    if alt_files:
        for alt_file in alt_files:
            with open(alt_file) as in_handle:
                for line in in_handle:
                    if not line.startswith("@"):
                        alts.add(line.split()[0].strip())
    else:
        for contig in ref.file_contigs(dd.get_ref_file(data)):
            if ("_alt" in contig.name or "_decoy" in contig.name or
                  contig.name.startswith("HLA-") or ":" in contig.name):
                alts.add(contig.name)
    return [c for c in ref.file_contigs(dd.get_ref_file(data)) if c.name not in alts]
520819
def _readbin(fid, fmt='i', nwords=1, file64=False, unpack=True):
    
    if fmt in 'if':
        fmt += '8' if file64 else '4'
    elts = np.fromfile(fid, fmt, nwords)
    if unpack and len(elts) == 1:
        elts = elts[0]
    return elts
107054
def __reorganize(self):
        
        logger_lpd_noaa.info("enter reorganize")
        # NOAA files are organized in sections differently than NOAA. try to translate these sections.
        for key, value in self.lipd_data.items():
            # if this key has a noaa match, it'll be returned. otherwise, empty string for no match
            noaa_key = self.__get_noaa_key(key)
            # check if this lipd key is in the NOAA_KEYS conversion dictionary.
            # if it's not, then stash it in our ignore list.
            if key not in LIPD_NOAA_MAP_FLAT:
                self.noaa_data_sorted["Ignore"][noaa_key] = value
            # studyName is placed two times in file. Line #1, and under the 'title' section
            elif noaa_key == "Study_Name":
                # study name gets put in two locations
                self.noaa_data_sorted["Top"][noaa_key] = value
                self.noaa_data_sorted["Title"][noaa_key] = value
            # put archiveType in self, because we'll reuse it later for the 9-part-variables as well
            elif noaa_key == "Archive":
                self.lsts_tmp["archive"].append(value)
            # Dataset_DOI is a repeatable element. the key could be a single DOI, or a list of DOIs.
            elif noaa_key == "Dataset_DOI":
                self.__parse_dois(value)

            # all other keys. determine which noaa section they belong in.
            else:
                # noaa keys are sorted by section.
                for header, content in NOAA_KEYS_BY_SECTION.items():
                    try:
                        # if our key is a noaa header key, then that means it's the ONLY key in the section.
                        # set value directly
                        if noaa_key == header:
                            self.noaa_data_sorted[header] = value
                        # all other cases, the key is part of the section
                        elif noaa_key in content:
                            self.noaa_data_sorted[header][noaa_key] = value
                    except KeyError:
                        # this shouldn't ever really happen, but just in case
                        logger_lpd_noaa.warn("lpd_noaa: reorganize: KeyError: {}".format(noaa_key))
        return
1273291
def _readFile(self, sldir, fileName, sep):
        '''
        

        '''

        if sldir.endswith(os.sep):
            fileName = str(sldir)+str(fileName)
        else:
            fileName = str(sldir)+os.sep+str(fileName)


        fileLines=[] #list of lines in the file
        header=[]    #list of Header lines
        dataCols=[]  #Dictionary of data column names
        data=[]      #List of Data lists
        cols=[]      #List of column names

        f=open(fileName,'r')
        fileLines=f.readlines()
        i=0
        if self.datatype != 'trajectory':

            while i<len(fileLines):
                if fileLines[i].startswith(self.header_char):
                    tmp=fileLines[i].lstrip(self.header_char)
                    header.append(tmp.strip())
                else:
                    break
                i+=1

            cols=fileLines[i].split(sep)

            tmp=[]
            tmp1=[]
            for j in range(len(cols)):
                tmp1=cols[j].strip()
                if tmp1 !='':
                    tmp.append(tmp1)
            cols=tmp
            i+=1
        else:
            header={}
            while fileLines[i].startswith('#') or '=' in fileLines[i]:
                if fileLines[i].startswith('#') and cols==[]:
                    cols=fileLines[i].strip('#')
                    cols=cols.strip()
                    cols=cols.split()
                elif fileLines[i].startswith('#'):
                    tmp1=fileLines[i].strip('#')
                    tmp1=tmp1.strip()
                    self.headerLines.append(tmp1)
                elif not fileLines[i].startswith('#'):
                    tmp=fileLines[i].split('=')
                    tmp[0]=tmp[0].strip()
                    tmp[1]=tmp[1].strip()
                    if header=={}:
                        header={str(tmp[0]):str(tmp[1])}
                    else:
                        header[str(tmp[0])]=str(tmp[1])
                i+=1
        while i<len(fileLines):
            if fileLines[i].startswith('#'):
                i=i+1
            else:
                tmp=fileLines[i].split()
                for j in range(len(tmp)):
                    tmp[j]=tmp[j].strip()
                data.append(tmp)
                i+=1
        tmp=[]
        tmp1=[]
        for j in range(len(data)):
            for k in range(len(data[j])):
                tmp1=data[j][k].strip()
                if tmp1 !='':
                    tmp.append(tmp1)
            data[j]=tmp
            tmp=[]
        tmp=[]

        for j in range(len(cols)):
            for k in range(len(data)):
                try:
                    a=float(data[k][j])
                    tmp.append(a)
                except ValueError:
                    tmp.append(data[k][j])
                #else:
                 #   tmp.append(float(data[k][j])) # previously tmp.append(float(data[k][j]))
            tmp=array(tmp)

            if j == 0:
                dataCols={cols[j]:tmp}
            else:
                dataCols[cols[j]]=tmp
            tmp=[]

        return header,dataCols
540245
def to_vobject(self, filename=None, uid=None):
        
        self._update()
        return self._to_vcard(self._book[uid.split('@')[0]])
615072
def formatted(self):
    '''  '''

    return """
Weather Station: %s (%s, %s)
Elevation: %s m
Time: %s UTC
Air Temperature: %s C (%s F)
Wind Speed: %s m/s (%s mph)
Wind Direction: %s
Present Weather Obs: %s
Precipitation: %s
Cloud Coverage: %s oktas
Cloud Summation: %s
Solar Irradiance: %s 
    """ % (self.weather_station, self.latitude, self.longitude,
           self.elevation, self.datetime, self.air_temperature,
           self.air_temperature.get_fahrenheit(), self.wind_speed,
           self.wind_speed.get_miles(), self.wind_direction,
           str(self.present_weather), str(self.precipitation),
           str(self.sky_cover), str(self.sky_cover_summation),
           str(self.solar_irradiance))
210327
def msvd(m):
  
  u, s, vdgr = np.linalg.svd(m)
  order = s.argsort()
  # reverse the n first columns of u
  s = s[order]
  u= u[:,order]
  vdgr = vdgr[order]
  return u, s, vdgr.conj().T