nodes: 0, module; 1, function_definition; 2, function_name:load_data_source; 3, parameters; 4, block; 5, identifier:local_path; 6, identifier:remote_source_list; 7, identifier:open_method; 8, default_parameter; 9, default_parameter; 10, default_parameter; 11, expression_statement; 12, try_statement; 13, return_statement; 14, identifier:open_method_kwargs; 15, call; 16, identifier:remote_kwargs; 17, call; 18, identifier:verbose; 19, True; 20, string; 21, block; 22, comment:#except FileNotFoundError:  # this is a more specific exception in Python 3; 23, except_clause; 24, expression_list; 25, identifier:dict; 26, argument_list; 27, identifier:dict; 28, argument_list; 29, string_content:Flexible data retreiver to download and cache the data files locally.
  
      Usage example (this makes a local copy of the ozone data file):
  
      :Example:
  
          .. code-block:: python
  
              from climlab.utils.data_source import load_data_source
              from xarray import open_dataset
              ozonename = 'apeozone_cam3_5_54.nc'
              ozonepath = 'http://thredds.atmos.albany.edu:8080/thredds/fileServer/CLIMLAB/ozone/' + ozonename
              data, path = load_data_source(local_path=ozonename,
                                            remote_source_list=[ozonepath],
                                            open_method=open_dataset)
              print(data)
  
  
      The order of operations is
  
      1. Try to read the data directly from ``local_path``
      2. If the file doesn't exist then iterate through ``remote_source_list``.
         Try to download and save the file to ``local_path`` using http request
         If that works then open the data from ``local_path``.
      3. As a last resort, try to read the data remotely from URLs in ``remote_source_list``
  
      In all cases the file is opened and read by the user-supplied ``open_method``
      (e.g. ``xarray.open_dataset``), with additional keyword arguments supplied
      as a dictionary through ``open_method_kwargs``.
      These are passed straight through to ``open_method``.
  
      Additional keyword arguments in ``remote_kwargs``
      are only passed to ``open_method`` in option 3 above
      (remote access, e.g. through OpenDAP)
  
      Quiet all output by passing ``verbose=False``.
  
      Returns:
  
      - ``data`` is the data object returned by the successful call to ``open_method``
      - ``path`` is the path that resulted in a successful call to ``open_method``.; 30, expression_statement; 31, expression_statement; 32, if_statement; 33, identifier:IOError; 34, comment:# works for Py2.7 and Py3.x; 35, comment:#  First try to load from remote sources and cache the file locally; 36, block; 37, identifier:data; 38, identifier:path; 39, assignment; 40, assignment; 41, identifier:verbose; 42, block; 43, for_statement; 44, identifier:path; 45, identifier:local_path; 46, identifier:data; 47, call; 48, expression_statement; 49, identifier:source; 50, identifier:remote_source_list; 51, block; 52, else_clause; 53, identifier:open_method; 54, argument_list; 55, call; 56, try_statement; 57, comment:# as a final resort, try opening the source remotely; 58, block; 59, identifier:path; 60, dictionary_splat; 61, identifier:print; 62, argument_list; 63, block; 64, except_clause; 65, for_statement; 66, identifier:open_method_kwargs; 67, call; 68, expression_statement; 69, expression_statement; 70, if_statement; 71, break_statement; 72, identifier:Exception; 73, block; 74, identifier:source; 75, identifier:remote_source_list; 76, block; 77, else_clause; 78, attribute; 79, argument_list; 80, assignment; 81, assignment; 82, identifier:verbose; 83, block; 84, continue_statement; 85, expression_statement; 86, try_statement; 87, block; 88, string; 89, identifier:format; 90, identifier:path; 91, identifier:response; 92, call; 93, identifier:data; 94, call; 95, expression_statement; 96, assignment; 97, comment:# This works fine for Python >= 3.5; 98, comment:#data = open_method(path, **open_method_kwargs, **remote_kwargs); 99, block; 100, except_clause; 101, raise_statement; 102, string_content:Opened data from {}; 103, identifier:_download_and_cache; 104, argument_list; 105, identifier:open_method; 106, argument_list; 107, call; 108, identifier:path; 109, identifier:source; 110, expression_statement; 111, if_statement; 112, break_statement; 113, identifier:Exception; 114, block; 115, call; 116, identifier:source; 117, identifier:local_path; 118, identifier:local_path; 119, dictionary_splat; 120, identifier:print; 121, argument_list; 122, assignment; 123, identifier:verbose; 124, block; 125, continue_statement; 126, identifier:Exception; 127, argument_list; 128, identifier:open_method_kwargs; 129, call; 130, identifier:data; 131, call; 132, expression_statement; 133, string; 134, attribute; 135, argument_list; 136, identifier:open_method; 137, argument_list; 138, call; 139, string_content:All data access methods have failed.; 140, string; 141, identifier:format; 142, identifier:source; 143, identifier:path; 144, dictionary_splat; 145, identifier:print; 146, argument_list; 147, string_content:Data retrieved from {} and saved locally.; 148, call; 149, call; 150, identifier:merge_two_dicts; 151, argument_list; 152, attribute; 153, argument_list; 154, identifier:open_method_kwargs; 155, identifier:remote_kwargs; 156, string; 157, identifier:format; 158, identifier:source; 159, string_content:Opened data remotely from {}
code: def load_data_source(local_path,
                 remote_source_list,
                 open_method,
                 open_method_kwargs=dict(),
                 remote_kwargs=dict(),
                 verbose=True):
      '''Flexible data retreiver to download and cache the data files locally.
  
      Usage example (this makes a local copy of the ozone data file):
  
      :Example:
  
          .. code-block:: python
  
              from climlab.utils.data_source import load_data_source
              from xarray import open_dataset
              ozonename = 'apeozone_cam3_5_54.nc'
              ozonepath = 'http://thredds.atmos.albany.edu:8080/thredds/fileServer/CLIMLAB/ozone/' + ozonename
              data, path = load_data_source(local_path=ozonename,
                                            remote_source_list=[ozonepath],
                                            open_method=open_dataset)
              print(data)
  
  
      The order of operations is
  
      1. Try to read the data directly from ``local_path``
      2. If the file doesn't exist then iterate through ``remote_source_list``.
         Try to download and save the file to ``local_path`` using http request
         If that works then open the data from ``local_path``.
      3. As a last resort, try to read the data remotely from URLs in ``remote_source_list``
  
      In all cases the file is opened and read by the user-supplied ``open_method``
      (e.g. ``xarray.open_dataset``), with additional keyword arguments supplied
      as a dictionary through ``open_method_kwargs``.
      These are passed straight through to ``open_method``.
  
      Additional keyword arguments in ``remote_kwargs``
      are only passed to ``open_method`` in option 3 above
      (remote access, e.g. through OpenDAP)
  
      Quiet all output by passing ``verbose=False``.
  
      Returns:
  
      - ``data`` is the data object returned by the successful call to ``open_method``
      - ``path`` is the path that resulted in a successful call to ``open_method``.
      '''
      try:
          path = local_path
          data = open_method(path, **open_method_kwargs)
          if verbose:
              print('Opened data from {}'.format(path))
      #except FileNotFoundError:  # this is a more specific exception in Python 3
      except IOError:  # works for Py2.7 and Py3.x
          #  First try to load from remote sources and cache the file locally
          for source in remote_source_list:
              try:
                  response = _download_and_cache(source, local_path)
                  data = open_method(local_path, **open_method_kwargs)
                  if verbose:
                      print('Data retrieved from {} and saved locally.'.format(source))
                  break
              except Exception:
                  continue
          else:
              # as a final resort, try opening the source remotely
              for source in remote_source_list:
                  path = source
                  try:
                      # This works fine for Python >= 3.5
                      #data = open_method(path, **open_method_kwargs, **remote_kwargs)
                      data = open_method(path, **merge_two_dicts(open_method_kwargs, remote_kwargs))
                      if verbose:
                          print('Opened data remotely from {}'.format(source))
                      break
                  except Exception:
                      continue
              else:
                  raise Exception('All data access methods have failed.')
      return data, path