Fetch data and parse it to build a list of cable entries.
165386
def fetch_entries(self):
        """"""
        data = []
        for row in self.get_rows():
            # Stop fetching data if limit has been met
            if exceeded_limit(self.limit, len(data)):
                break

            entry = row.find_all('td')
            entry_dict = {}

            show = entry[0].string
            net = entry[1].string
            if not self._match_query(show, net):
                continue

            entry_dict['show'] = show
            entry_dict['net'] = net
            entry_dict['time'] = entry[2].string

            if ',' in entry[3].string:
                entry_dict['viewers'] = entry[3].string.replace(',', '.')
            else:
                entry_dict['viewers'] = '0.' + entry[3].string
            entry_dict['rating'] = entry[4].string

            # Add data to create cable entry
            data.append(Entry(**entry_dict))

        return data
159741
def __parseDatasets(self):
        """"""
        datasets = []
        if self.__dataItem.has_key('dataSets'):
            for dataset in self.__dataItem['dataSets']:
                datasets.append(DataSet(Sitools2Abstract.getBaseUrl(self), dataset))
        return datasets
941396
def _collect_data(directory, input_ext, transcription_ext):
  """"""
  # Directory from string to tuple pair of strings
  # key: the filepath to a datafile including the datafile's basename. Example,
  #   if the datafile was "/path/to/datafile.wav" then the key would be
  #   "/path/to/datafile"
  # value: a pair of strings (media_filepath, label)
  data_files = {}
  for root, _, filenames in os.walk(directory):
    transcripts = [filename for filename in filenames
                   if transcription_ext in filename]
    for transcript in transcripts:
      transcript_path = os.path.join(root, transcript)
      with open(transcript_path, "r") as transcript_file:
        for transcript_line in transcript_file:
          line_contents = transcript_line.strip().split(" ", 1)
          media_base, label = line_contents
          key = os.path.join(root, media_base)
          assert key not in data_files
          media_name = "%s.%s"%(media_base, input_ext)
          media_path = os.path.join(root, media_name)
          data_files[key] = (media_base, media_path, label)
  return data_files
376964
def start(self):
        """"""
        LOG.info('Listing options...')
        with indicator.Spinner(**self.indicator_options):
            objects_list = self._list_contents()
            if not objects_list:
                return

        # Index items
        self._index_objects(objects_list=objects_list)
        # Create the underlying structure
        self._make_directory_structure()

        # Download everything
        LOG.debug('Download Items: %s', self.download_items)
        self._multi_processor(
            self._get,
            items=[i for i in self.download_items.values() for i in i]
        )
389750
def load_and_parse(self):
        """"""

        archives = []
        to_return = {}

        for name, project in self.all_projects.items():
            archives = archives + self.parse_archives_from_project(project)

        # We're going to have a similar issue with parsed nodes, if we want to
        # make parse_node return those.
        for a in archives:
            # archives have a config, but that would make for an invalid
            # UnparsedNode, so remove it and pass it along to parse_node as an
            # argument.
            archive_config = a.pop('config')
            archive = UnparsedNode(**a)
            node_path = self.get_path(archive.resource_type,
                                      archive.package_name,
                                      archive.name)

            to_return[node_path] = self.parse_node(
                archive,
                node_path,
                self.all_projects.get(archive.package_name),
                archive_config=archive_config)

        return to_return
449784
def parse(self, raw):
        """"""
        self._raw = raw
        self.hub_name = self._parse("userData", "hubName", converter=base64_to_unicode)
        self.ip = self._parse("userData", "ip")
        self.ssid = self._parse("userData", "ssid")
167730
def parse(feedparser, args):
    """"""
    entries = []
    args = irc3.utils.Config(args)
    max_date = datetime.datetime.now() - datetime.timedelta(days=2)

    for filename in args['filenames']:
        try:
            with open(filename + '.updated') as fd:
                updated = fd.read().strip()
        except (OSError, IOError):
            updated = '0'

        feed = feedparser.parse(filename)
        for e in feed.entries:
            if e.updated <= updated:
                # skip already sent entries
                continue
            try:
                updated_parsed = e.updated_parsed
            except AttributeError:
                continue
            if datetime.datetime(*updated_parsed[:7]) < max_date:
                # skip entries older than 2 days
                continue
            e['filename'] = filename
            e['feed'] = args
            entries.append((e.updated, e))
        if entries:
            entries = sorted(entries, key=itemgetter(0))
            with open(filename + '.updated', 'w') as fd:
                fd.write(str(entries[-1][0]))
    return entries
367410
def build_data_list(lst):
    
    txt = '<H3>' + List + '<H3><UL>'
    for i in lst:
        txt += '<LI>' + i + '</LI>'
        
    txt += '<UL>'
    return txt
1072443
def user_parse(data):
        """"""
        yield 'id', data.get('id')
        yield 'email', data.get('email')
        first_name, _, last_name = (data.get('name') or '').partition(' ')
        yield 'first_name', first_name
        yield 'last_name', last_name
        yield 'username', data.get('login')
        yield 'picture', data.get('avatar_url')
        yield 'link', data.get('html_url')
        location = data.get('location', '')
        if location:
            split_location = location.split(',')
            yield 'country', split_location[0].strip()
            if len(split_location) > 1:
                yield 'city', split_location[1].strip()
98948
def _fetch_data(self, dataset, query={}, include_inactive_stations=False):
        
        data = []
        parameter = dataset
        station_dim = dataset.dimensions["station"]
        all_stations = station_dim.allowed_values
        # Step 1: Prepare query
        if "station" not in query:
            if include_inactive_stations:
                # Get all stations
                query["station"] = list(all_stations)
            else:
                # Get only active stations
                query["station"] = list(station_dim.active_stations())
        else:
            if not isinstance(query["station"], list):
                query["station"] = [query["station"]]
            # Make sure that the queried stations actually exist
            query["station"] = [ all_stations.get_by_label(x) for x in query["station"]]

        if "period" not in query:
            # TODO: I'd prepare to do dataset.get("period").allowed_values here
            query["period"] = PERIODS

        elif not isinstance(query["period"], list):
            query["period"] = [query["period"]]

        for period in query["period"]:
            if period not in PERIODS:
                msg = u"{} is not an allowed period".format(period)
                raise Exception(msg)


        # Step 3: Get data
        n_queries = len(query["station"]) * len(query["period"])
        counter = 0
        print("Fetching data with {} queries.".format(n_queries))
        for station in query["station"]:
            for period in query["period"]:
                url = dataset.url\
                    .replace(".json", "/station/{}/period/{}/data.csv"\
                        .format(station.key, period))
                print("/GET {} ".format(url))
                r = requests.get(url)

                if r.status_code == 200:
                    raw_data = DataCsv().from_string(r.content).to_dictlist()

                    # TODO: This is a very hard coded parse function
                    # Expects fixed start row and number of cols
                    for row in raw_data:
                        #timepoint = datetime.strptime(timepoint_str, "%Y-%m-%d %H:%M:%S")
                        value_col = parameter.id.split(",")[0]
                        value = float(row[value_col])

                        row["parameter"] = parameter.id
                        row["station"] = station.label
                        row["station_key"] = station.key
                        row["period"] = period

                        row.pop(value_col,None)

                        datapoint = Result(value, row)

                        yield datapoint

                elif r.status_code == 404:
                    print("Warning no data at {}".format(url))
                else:
                    raise Exception("Connection error for {}".format(url))
608433
def process_data(lines)  
      data = {}
      lines.each do |line|
        line = line.split
        key = line.first.split('.value').first
        data[key] = line.last
      end
      data
    end
129147
def fetch_and_parse(method, uri, params_prefix=None, **params):
    """"""
    response = fetch(method, uri, params_prefix, **params)
    return _parse(json.loads(response.text))
1223634
def _parse(res, params, n, api, **kwds):
    """"""
    cursor = "cursor" in params
    if not cursor:
        start = params["start"]
    if n == 0:
        return ""
    _json = res.get('search-results', {}).get('entry', [])
    # Download the remaining information in chunks
    while n > 0:
        n -= params["count"]
        if cursor:
            pointer = res['search-results']['cursor'].get('@next')
            params.update({'cursor': pointer})
        else:
            start += params["count"]
            params.update({'start': start})
        res = download(url=URL[api], params=params, accept="json", **kwds).json()
        _json.extend(res.get('search-results', {}).get('entry', []))
    return _json
725380
def fetch_normal(item, request):
    """"""
    try:
        conn = urllib2.urlopen(request, timeout=item.timeout_seconds)
    except urllib2.HTTPError, e:
        conn = e
    except (urllib2.URLError, ssl.SSLError), e:
        # TODO: Make this status more clear
        item.status_code = 400
        return item

    try:
        item.status_code = conn.getcode()
        item.content_type = conn.info().gettype()
        if item.result_path:
            with open(item.result_path, 'wb') as result_file:
                shutil.copyfileobj(conn, result_file)
        else:
            item.data = conn.read()
    except socket.timeout, e:
        # TODO: Make this status more clear
        item.status_code = 400
        return item
    finally:
        conn.close()

    return item
942993
def feed_data(self, data):
        """"""
        if data:
            self._bytes.append(data)
            if self.parser.stream:
                self.parser.stream(self)
            else:
                self.parser.buffer.extend(data)
654488
def process_raw_data(cls, raw_data):
        """"""
        properties = raw_data["properties"]

        routes = []
        raw_routes = properties.get("routes", [])
        for raw_route in raw_routes:
            raw_route["parentResourceID"] = raw_data["resourceId"]
            routes.append(Routes.from_raw_data(raw_route))
        properties["routes"] = routes

        subnets = []
        raw_subnets = properties.get("subnets", [])
        for raw_subnet in raw_subnets:
            subnets.append(Resource.from_raw_data(raw_subnet))
        properties["subnets"] = subnets

        return super(RouteTables, cls).process_raw_data(raw_data)
793367
def load_data(self, pdbid):
        """"""
        f = urlopen("http://projects.biotec.tu-dresden.de/plip-rest/pdb/%s?format=xml" % pdbid.lower())
        self.doc = etree.parse(f)