We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
check_input_data not supporting input files outside of input data repo
@erichlf has asked for support for the following use case:
* Create a partition file in his home directory
* Point namelist files to use this partition file
* check_input_data should find this file but it doesn't because it insists on looking in the input data repo

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `utils/python/CIME/check_input_data.py`
Content:
```
1 """
2 API for checking input for testcase
3 """
4 
5 from CIME.XML.standard_module_setup import *
6 from CIME.utils import get_model, SharedArea
7 
8 import fnmatch, glob, shutil
9 
10 logger = logging.getLogger(__name__)
11 
12 # Should probably be in XML somewhere
13 SVN_LOCS = {
14     "acme" : "https://acme-svn2.ornl.gov/acme-repo/acme/inputdata",
15     "cesm" : "https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata"
16 }
17 
18 def find_files(rootdir, pattern):
19     """
20     recursively find all files matching a pattern
21     """
22     result = []
23     for root, _, files in os.walk(rootdir):
24         for filename in files:
25             if (fnmatch.fnmatch(filename, pattern)):
26                 result.append(os.path.join(root, filename))
27 
28     return result
29 
30 def download_if_in_repo(svn_loc, input_data_root, rel_path):
31     """
32     Return True if successfully downloaded
33     """
34     rel_path = rel_path.strip('/')
35     full_url = os.path.join(svn_loc, rel_path)
36 
37     full_path = os.path.join(input_data_root, rel_path)
38     logging.info("Trying to download file: '%s' to path '%s'" % (full_url, full_path))
39     # Make sure local path exists, create if it does not
40     if(not os.path.exists(os.path.dirname(full_path))):
41         os.makedirs(os.path.dirname(full_path))
42 
43     stat, out, err = run_cmd("svn --non-interactive --trust-server-cert ls %s" % full_url)
44     if (stat != 0):
45         logging.warning("FAIL: SVN repo '%s' does not have file '%s'\nReason:%s\n%s\n" % (svn_loc, full_url, out, err))
46         return False
47     else:
48         # Use umask to make sure files are group read/writable. As long as parent directories
49         # have +s, then everything should work.
50         with SharedArea():
51             stat, output, errput = \
52                 run_cmd("svn --non-interactive --trust-server-cert export %s %s" % (full_url, full_path))
53             if (stat != 0):
54                 logging.warning("svn export failed with output: %s and errput %s\n" % (output, errput))
55                 return False
56             else:
57                 logging.info("SUCCESS\n")
58                 return True
59 
60 ###############################################################################
61 def check_all_input_data(case):
62 ###############################################################################
63 
64     success = check_input_data(case=case, download=True)
65     expect(success, "Failed to download input data")
66 
67     get_refcase  = case.get_value("GET_REFCASE")
68     run_type     = case.get_value("RUN_TYPE")
69     continue_run = case.get_value("CONTINUE_RUN")
70 
71     # We do not fully populate the inputdata directory on every
72     # machine and do not expect every user to download the 3TB+ of
73     # data in our inputdata repository. This code checks for the
74     # existence of inputdata in the local inputdata directory and
75     # attempts to download data from the server if it's needed and
76     # missing.
77     if get_refcase and run_type != "startup" and not continue_run:
78         din_loc_root = case.get_value("DIN_LOC_ROOT")
79         run_refdate  = case.get_value("RUN_REFDATE")
80         run_refcase  = case.get_value("RUN_REFCASE")
81         run_refdir   = case.get_value("RUN_REFDIR")
82         rundir       = case.get_value("RUNDIR")
83 
84         refdir = os.path.join(din_loc_root, run_refdir, run_refcase, run_refdate)
85         expect(os.path.isdir(refdir),
86 """
87 *****************************************************************
88 prestage ERROR: $refdir is not on local disk
89 obtain this data from the svn input data repository
90 > mkdir -p %s
91 > cd %s
92 > cd ..
93 > svn export --force https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/%s
94 or set GET_REFCASE to FALSE in env_run.xml
95 and prestage the restart data to $RUNDIR manually
96 *****************************************************************""" % (refdir, refdir, refdir))
97 
98         logger.info(" - Prestaging REFCASE (%s) to %s" % (refdir, rundir))
99 
100         # prestage the reference case's files.
101 
102         if (not os.path.exists(rundir)):
103             logger.debug("Creating run directory: %s"%rundir)
104             os.makedirs(rundir)
105 
106         for rcfile in glob.iglob(os.path.join(refdir,"*%s*"%run_refcase)):
107             logger.debug("Staging file %s"%rcfile)
108             rcbaseline = os.path.basename(rcfile)
109             if not os.path.exists("%s/%s" % (rundir, rcbaseline)):
110                 os.symlink(rcfile, "%s/%s" % ((rundir, rcbaseline)))
111 
112         # copy the refcases' rpointer files to the run directory
113         for rpointerfile in  glob.iglob(os.path.join("%s","*rpointer*") % (refdir)):
114             logger.debug("Copy rpointer %s"%rpointerfile)
115             shutil.copy(rpointerfile, rundir)
116 
117 
118         for cam2file in  glob.iglob(os.path.join("%s","*.cam2.*") % rundir):
119             camfile = cam2file.replace("cam2", "cam")
120             os.symlink(cam2file, camfile)
121 
122 def check_input_data(case, svn_loc=None, input_data_root=None, data_list_dir="Buildconf", download=False):
123     """
124     Return True if no files missing
125     """
126     # Fill in defaults as needed
127     svn_loc = SVN_LOCS[get_model()] if svn_loc is None else svn_loc
128     input_data_root = case.get_value("DIN_LOC_ROOT") if input_data_root is None else input_data_root
129 
130     expect(os.path.isdir(input_data_root), "Invalid input_data_root directory: '%s'" % input_data_root)
131     expect(os.path.isdir(data_list_dir), "Invalid data_list_dir directory: '%s'" % data_list_dir)
132 
133     data_list_files = find_files(data_list_dir, "*.input_data_list")
134     expect(data_list_files, "No .input_data_list files found in dir '%s'" % data_list_dir)
135 
136     no_files_missing = True
137     for data_list_file in data_list_files:
138         logging.info("Loading input file: '%s'" % data_list_file)
139         with open(data_list_file, "r") as fd:
140             lines = fd.readlines()
141 
142         for line in lines:
143             line = line.strip()
144             if (line and not line.startswith("#")):
145                 tokens = line.split('=')
146                 description, full_path = tokens[0].strip(), tokens[1].strip()
147                 if(full_path):
148                     # expand xml variables
149                     full_path = case.get_resolved_value(full_path)
150                     rel_path  = full_path.replace(input_data_root, "")
151 
152                     # There are some special values of rel_path that
153                     # we need to ignore - some of the component models
154                     # set things like 'NULL' or 'same_as_TS' -
155                     # basically if rel_path does not contain '/' (a
156                     # directory tree) you can assume it's a special
157                     # value and ignore it (perhaps with a warning)
158                     if ("/" in rel_path and not os.path.exists(full_path)):
159                         model = os.path.basename(data_list_file).split('.')[0]
160                         logging.warning("Model %s missing file %s = '%s'" % (model,description,full_path))
161 
162                         if (download):
163                             success = download_if_in_repo(svn_loc, input_data_root, rel_path)
164                             if (not success):
165                                 # If ACME, try CESM repo as backup
166                                 if (get_model() == "acme" and svn_loc != SVN_LOCS["cesm"]):
167                                     success = download_if_in_repo(SVN_LOCS["cesm"], input_data_root, rel_path)
168                                     if (not success):
169                                         no_files_missing = False
170                                 else:
171                                     no_files_missing = False
172                         # if not download
173                         else:
174                             no_files_missing = False
175                     else:
176                         logging.debug("Already had input file: '%s'" % full_path)
177 
178                 else:
179                     model = os.path.basename(data_list_file).split('.')[0]
180                     logging.warning("Model %s no file specified for %s"%(model,description))
181 
182     return no_files_missing
183
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/utils/python/CIME/check_input_data.py b/utils/python/CIME/check_input_data.py
--- a/utils/python/CIME/check_input_data.py
+++ b/utils/python/CIME/check_input_data.py
@@ -148,32 +148,45 @@
                     # expand xml variables
                     full_path = case.get_resolved_value(full_path)
                     rel_path  = full_path.replace(input_data_root, "")
+                    model = os.path.basename(data_list_file).split('.')[0]
 
-                    # There are some special values of rel_path that
-                    # we need to ignore - some of the component models
-                    # set things like 'NULL' or 'same_as_TS' -
-                    # basically if rel_path does not contain '/' (a
-                    # directory tree) you can assume it's a special
-                    # value and ignore it (perhaps with a warning)
-                    if ("/" in rel_path and not os.path.exists(full_path)):
-                        model = os.path.basename(data_list_file).split('.')[0]
-                        logging.warning("Model %s missing file %s = '%s'" % (model,description,full_path))
-
-                        if (download):
-                            success = download_if_in_repo(svn_loc, input_data_root, rel_path)
-                            if (not success):
-                                # If ACME, try CESM repo as backup
-                                if (get_model() == "acme" and svn_loc != SVN_LOCS["cesm"]):
-                                    success = download_if_in_repo(SVN_LOCS["cesm"], input_data_root, rel_path)
-                                    if (not success):
-                                        no_files_missing = False
-                                else:
-                                    no_files_missing = False
-                        # if not download
-                        else:
+                    if ("/" in rel_path and rel_path == full_path):
+                        # User pointing to a file outside of input_data_root, we cannot determine
+                        # rel_path, and so cannot download the file. If it already exists, we can
+                        # proceed
+                        if not os.path.exists(full_path):
+                            logging.warning("  Model %s missing file %s = '%s'" % (model, description, full_path))
+                            if download:
+                                logging.warning("    Cannot download file since it lives outside of the input_data_root '%s'" % input_data_root)
                             no_files_missing = False
+                        else:
+                            logging.info("  Found input file: '%s'" % full_path)
+
                     else:
-                        logging.debug("Already had input file: '%s'" % full_path)
+                        # There are some special values of rel_path that
+                        # we need to ignore - some of the component models
+                        # set things like 'NULL' or 'same_as_TS' -
+                        # basically if rel_path does not contain '/' (a
+                        # directory tree) you can assume it's a special
+                        # value and ignore it (perhaps with a warning)
+                        if ("/" in rel_path and not os.path.exists(full_path)):
+                            logging.warning("  Model %s missing file %s = '%s'" % (model,description,full_path))
+
+                            if (download):
+                                success = download_if_in_repo(svn_loc, input_data_root, rel_path)
+                                if (not success):
+                                    # If ACME, try CESM repo as backup
+                                    if (get_model() == "acme" and svn_loc != SVN_LOCS["cesm"]):
+                                        success = download_if_in_repo(SVN_LOCS["cesm"], input_data_root, rel_path)
+                                        if (not success):
+                                            no_files_missing = False
+                                    else:
+                                        no_files_missing = False
+                            # if not download
+                            else:
+                                no_files_missing = False
+                        else:
+                            logging.info("  Already had input file: '%s'" % full_path)
 
                 else:
                     model = os.path.basename(data_list_file).split('.')[0]

{"golden_diff": "diff --git a/utils/python/CIME/check_input_data.py b/utils/python/CIME/check_input_data.py\n--- a/utils/python/CIME/check_input_data.py\n+++ b/utils/python/CIME/check_input_data.py\n@@ -148,32 +148,45 @@\n                     # expand xml variables\n                     full_path = case.get_resolved_value(full_path)\n                     rel_path  = full_path.replace(input_data_root, \"\")\n+                    model = os.path.basename(data_list_file).split('.')[0]\n \n-                    # There are some special values of rel_path that\n-                    # we need to ignore - some of the component models\n-                    # set things like 'NULL' or 'same_as_TS' -\n-                    # basically if rel_path does not contain '/' (a\n-                    # directory tree) you can assume it's a special\n-                    # value and ignore it (perhaps with a warning)\n-                    if (\"/\" in rel_path and not os.path.exists(full_path)):\n-                        model = os.path.basename(data_list_file).split('.')[0]\n-                        logging.warning(\"Model %s missing file %s = '%s'\" % (model,description,full_path))\n-\n-                        if (download):\n-                            success = download_if_in_repo(svn_loc, input_data_root, rel_path)\n-                            if (not success):\n-                                # If ACME, try CESM repo as backup\n-                                if (get_model() == \"acme\" and svn_loc != SVN_LOCS[\"cesm\"]):\n-                                    success = download_if_in_repo(SVN_LOCS[\"cesm\"], input_data_root, rel_path)\n-                                    if (not success):\n-                                        no_files_missing = False\n-                                else:\n-                                    no_files_missing = False\n-                        # if not download\n-                        else:\n+                    if (\"/\" in rel_path and rel_path == full_path):\n+                        # User pointing to a file outside of input_data_root, we cannot determine\n+                        # rel_path, and so cannot download the file. If it already exists, we can\n+                        # proceed\n+                        if not os.path.exists(full_path):\n+                            logging.warning(\"  Model %s missing file %s = '%s'\" % (model, description, full_path))\n+                            if download:\n+                                logging.warning(\"    Cannot download file since it lives outside of the input_data_root '%s'\" % input_data_root)\n                             no_files_missing = False\n+                        else:\n+                            logging.info(\"  Found input file: '%s'\" % full_path)\n+\n                     else:\n-                        logging.debug(\"Already had input file: '%s'\" % full_path)\n+                        # There are some special values of rel_path that\n+                        # we need to ignore - some of the component models\n+                        # set things like 'NULL' or 'same_as_TS' -\n+                        # basically if rel_path does not contain '/' (a\n+                        # directory tree) you can assume it's a special\n+                        # value and ignore it (perhaps with a warning)\n+                        if (\"/\" in rel_path and not os.path.exists(full_path)):\n+                            logging.warning(\"  Model %s missing file %s = '%s'\" % (model,description,full_path))\n+\n+                            if (download):\n+                                success = download_if_in_repo(svn_loc, input_data_root, rel_path)\n+                                if (not success):\n+                                    # If ACME, try CESM repo as backup\n+                                    if (get_model() == \"acme\" and svn_loc != SVN_LOCS[\"cesm\"]):\n+                                        success = download_if_in_repo(SVN_LOCS[\"cesm\"], input_data_root, rel_path)\n+                                        if (not success):\n+                                            no_files_missing = False\n+                                    else:\n+                                        no_files_missing = False\n+                            # if not download\n+                            else:\n+                                no_files_missing = False\n+                        else:\n+                            logging.info(\"  Already had input file: '%s'\" % full_path)\n \n                 else:\n                     model = os.path.basename(data_list_file).split('.')[0]\n", "issue": "check_input_data not supporting input files outside of input data repo\n@erichlf has asked for support for the following use case:\r\n* Create a partition file in his home directory\r\n* Point namelist files to use this partition file\r\n* check_input_data should find this file but it doesn't because it insists on looking in the input data repo\n", "before_files": [{"content": "\"\"\"\nAPI for checking input for testcase\n\"\"\"\n\nfrom CIME.XML.standard_module_setup import *\nfrom CIME.utils import get_model, SharedArea\n\nimport fnmatch, glob, shutil\n\nlogger = logging.getLogger(__name__)\n\n# Should probably be in XML somewhere\nSVN_LOCS = {\n    \"acme\" : \"https://acme-svn2.ornl.gov/acme-repo/acme/inputdata\",\n    \"cesm\" : \"https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata\"\n}\n\ndef find_files(rootdir, pattern):\n    \"\"\"\n    recursively find all files matching a pattern\n    \"\"\"\n    result = []\n    for root, _, files in os.walk(rootdir):\n        for filename in files:\n            if (fnmatch.fnmatch(filename, pattern)):\n                result.append(os.path.join(root, filename))\n\n    return result\n\ndef download_if_in_repo(svn_loc, input_data_root, rel_path):\n    \"\"\"\n    Return True if successfully downloaded\n    \"\"\"\n    rel_path = rel_path.strip('/')\n    full_url = os.path.join(svn_loc, rel_path)\n\n    full_path = os.path.join(input_data_root, rel_path)\n    logging.info(\"Trying to download file: '%s' to path '%s'\" % (full_url, full_path))\n    # Make sure local path exists, create if it does not\n    if(not os.path.exists(os.path.dirname(full_path))):\n        os.makedirs(os.path.dirname(full_path))\n\n    stat, out, err = run_cmd(\"svn --non-interactive --trust-server-cert ls %s\" % full_url)\n    if (stat != 0):\n        logging.warning(\"FAIL: SVN repo '%s' does not have file '%s'\\nReason:%s\\n%s\\n\" % (svn_loc, full_url, out, err))\n        return False\n    else:\n        # Use umask to make sure files are group read/writable. As long as parent directories\n        # have +s, then everything should work.\n        with SharedArea():\n            stat, output, errput = \\\n                run_cmd(\"svn --non-interactive --trust-server-cert export %s %s\" % (full_url, full_path))\n            if (stat != 0):\n                logging.warning(\"svn export failed with output: %s and errput %s\\n\" % (output, errput))\n                return False\n            else:\n                logging.info(\"SUCCESS\\n\")\n                return True\n\n###############################################################################\ndef check_all_input_data(case):\n###############################################################################\n\n    success = check_input_data(case=case, download=True)\n    expect(success, \"Failed to download input data\")\n\n    get_refcase  = case.get_value(\"GET_REFCASE\")\n    run_type     = case.get_value(\"RUN_TYPE\")\n    continue_run = case.get_value(\"CONTINUE_RUN\")\n\n    # We do not fully populate the inputdata directory on every\n    # machine and do not expect every user to download the 3TB+ of\n    # data in our inputdata repository. This code checks for the\n    # existence of inputdata in the local inputdata directory and\n    # attempts to download data from the server if it's needed and\n    # missing.\n    if get_refcase and run_type != \"startup\" and not continue_run:\n        din_loc_root = case.get_value(\"DIN_LOC_ROOT\")\n        run_refdate  = case.get_value(\"RUN_REFDATE\")\n        run_refcase  = case.get_value(\"RUN_REFCASE\")\n        run_refdir   = case.get_value(\"RUN_REFDIR\")\n        rundir       = case.get_value(\"RUNDIR\")\n\n        refdir = os.path.join(din_loc_root, run_refdir, run_refcase, run_refdate)\n        expect(os.path.isdir(refdir),\n\"\"\"\n*****************************************************************\nprestage ERROR: $refdir is not on local disk\nobtain this data from the svn input data repository\n> mkdir -p %s\n> cd %s\n> cd ..\n> svn export --force https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/%s\nor set GET_REFCASE to FALSE in env_run.xml\nand prestage the restart data to $RUNDIR manually\n*****************************************************************\"\"\" % (refdir, refdir, refdir))\n\n        logger.info(\" - Prestaging REFCASE (%s) to %s\" % (refdir, rundir))\n\n        # prestage the reference case's files.\n\n        if (not os.path.exists(rundir)):\n            logger.debug(\"Creating run directory: %s\"%rundir)\n            os.makedirs(rundir)\n\n        for rcfile in glob.iglob(os.path.join(refdir,\"*%s*\"%run_refcase)):\n            logger.debug(\"Staging file %s\"%rcfile)\n            rcbaseline = os.path.basename(rcfile)\n            if not os.path.exists(\"%s/%s\" % (rundir, rcbaseline)):\n                os.symlink(rcfile, \"%s/%s\" % ((rundir, rcbaseline)))\n\n        # copy the refcases' rpointer files to the run directory\n        for rpointerfile in  glob.iglob(os.path.join(\"%s\",\"*rpointer*\") % (refdir)):\n            logger.debug(\"Copy rpointer %s\"%rpointerfile)\n            shutil.copy(rpointerfile, rundir)\n\n\n        for cam2file in  glob.iglob(os.path.join(\"%s\",\"*.cam2.*\") % rundir):\n            camfile = cam2file.replace(\"cam2\", \"cam\")\n            os.symlink(cam2file, camfile)\n\ndef check_input_data(case, svn_loc=None, input_data_root=None, data_list_dir=\"Buildconf\", download=False):\n    \"\"\"\n    Return True if no files missing\n    \"\"\"\n    # Fill in defaults as needed\n    svn_loc = SVN_LOCS[get_model()] if svn_loc is None else svn_loc\n    input_data_root = case.get_value(\"DIN_LOC_ROOT\") if input_data_root is None else input_data_root\n\n    expect(os.path.isdir(input_data_root), \"Invalid input_data_root directory: '%s'\" % input_data_root)\n    expect(os.path.isdir(data_list_dir), \"Invalid data_list_dir directory: '%s'\" % data_list_dir)\n\n    data_list_files = find_files(data_list_dir, \"*.input_data_list\")\n    expect(data_list_files, \"No .input_data_list files found in dir '%s'\" % data_list_dir)\n\n    no_files_missing = True\n    for data_list_file in data_list_files:\n        logging.info(\"Loading input file: '%s'\" % data_list_file)\n        with open(data_list_file, \"r\") as fd:\n            lines = fd.readlines()\n\n        for line in lines:\n            line = line.strip()\n            if (line and not line.startswith(\"#\")):\n                tokens = line.split('=')\n                description, full_path = tokens[0].strip(), tokens[1].strip()\n                if(full_path):\n                    # expand xml variables\n                    full_path = case.get_resolved_value(full_path)\n                    rel_path  = full_path.replace(input_data_root, \"\")\n\n                    # There are some special values of rel_path that\n                    # we need to ignore - some of the component models\n                    # set things like 'NULL' or 'same_as_TS' -\n                    # basically if rel_path does not contain '/' (a\n                    # directory tree) you can assume it's a special\n                    # value and ignore it (perhaps with a warning)\n                    if (\"/\" in rel_path and not os.path.exists(full_path)):\n                        model = os.path.basename(data_list_file).split('.')[0]\n                        logging.warning(\"Model %s missing file %s = '%s'\" % (model,description,full_path))\n\n                        if (download):\n                            success = download_if_in_repo(svn_loc, input_data_root, rel_path)\n                            if (not success):\n                                # If ACME, try CESM repo as backup\n                                if (get_model() == \"acme\" and svn_loc != SVN_LOCS[\"cesm\"]):\n                                    success = download_if_in_repo(SVN_LOCS[\"cesm\"], input_data_root, rel_path)\n                                    if (not success):\n                                        no_files_missing = False\n                                else:\n                                    no_files_missing = False\n                        # if not download\n                        else:\n                            no_files_missing = False\n                    else:\n                        logging.debug(\"Already had input file: '%s'\" % full_path)\n\n                else:\n                    model = os.path.basename(data_list_file).split('.')[0]\n                    logging.warning(\"Model %s no file specified for %s\"%(model,description))\n\n    return no_files_missing\n", "path": "utils/python/CIME/check_input_data.py"}], "after_files": [{"content": "\"\"\"\nAPI for checking input for testcase\n\"\"\"\n\nfrom CIME.XML.standard_module_setup import *\nfrom CIME.utils import get_model, SharedArea\n\nimport fnmatch, glob, shutil\n\nlogger = logging.getLogger(__name__)\n\n# Should probably be in XML somewhere\nSVN_LOCS = {\n    \"acme\" : \"https://acme-svn2.ornl.gov/acme-repo/acme/inputdata\",\n    \"cesm\" : \"https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata\"\n}\n\ndef find_files(rootdir, pattern):\n    \"\"\"\n    recursively find all files matching a pattern\n    \"\"\"\n    result = []\n    for root, _, files in os.walk(rootdir):\n        for filename in files:\n            if (fnmatch.fnmatch(filename, pattern)):\n                result.append(os.path.join(root, filename))\n\n    return result\n\ndef download_if_in_repo(svn_loc, input_data_root, rel_path):\n    \"\"\"\n    Return True if successfully downloaded\n    \"\"\"\n    rel_path = rel_path.strip('/')\n    full_url = os.path.join(svn_loc, rel_path)\n\n    full_path = os.path.join(input_data_root, rel_path)\n    logging.info(\"Trying to download file: '%s' to path '%s'\" % (full_url, full_path))\n    # Make sure local path exists, create if it does not\n    if(not os.path.exists(os.path.dirname(full_path))):\n        os.makedirs(os.path.dirname(full_path))\n\n    stat, out, err = run_cmd(\"svn --non-interactive --trust-server-cert ls %s\" % full_url)\n    if (stat != 0):\n        logging.warning(\"FAIL: SVN repo '%s' does not have file '%s'\\nReason:%s\\n%s\\n\" % (svn_loc, full_url, out, err))\n        return False\n    else:\n        # Use umask to make sure files are group read/writable. As long as parent directories\n        # have +s, then everything should work.\n        with SharedArea():\n            stat, output, errput = \\\n                run_cmd(\"svn --non-interactive --trust-server-cert export %s %s\" % (full_url, full_path))\n            if (stat != 0):\n                logging.warning(\"svn export failed with output: %s and errput %s\\n\" % (output, errput))\n                return False\n            else:\n                logging.info(\"SUCCESS\\n\")\n                return True\n\n###############################################################################\ndef check_all_input_data(case):\n###############################################################################\n\n    success = check_input_data(case=case, download=True)\n    expect(success, \"Failed to download input data\")\n\n    get_refcase  = case.get_value(\"GET_REFCASE\")\n    run_type     = case.get_value(\"RUN_TYPE\")\n    continue_run = case.get_value(\"CONTINUE_RUN\")\n\n    # We do not fully populate the inputdata directory on every\n    # machine and do not expect every user to download the 3TB+ of\n    # data in our inputdata repository. This code checks for the\n    # existence of inputdata in the local inputdata directory and\n    # attempts to download data from the server if it's needed and\n    # missing.\n    if get_refcase and run_type != \"startup\" and not continue_run:\n        din_loc_root = case.get_value(\"DIN_LOC_ROOT\")\n        run_refdate  = case.get_value(\"RUN_REFDATE\")\n        run_refcase  = case.get_value(\"RUN_REFCASE\")\n        run_refdir   = case.get_value(\"RUN_REFDIR\")\n        rundir       = case.get_value(\"RUNDIR\")\n\n        refdir = os.path.join(din_loc_root, run_refdir, run_refcase, run_refdate)\n        expect(os.path.isdir(refdir),\n\"\"\"\n*****************************************************************\nprestage ERROR: $refdir is not on local disk\nobtain this data from the svn input data repository\n> mkdir -p %s\n> cd %s\n> cd ..\n> svn export --force https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/%s\nor set GET_REFCASE to FALSE in env_run.xml\nand prestage the restart data to $RUNDIR manually\n*****************************************************************\"\"\" % (refdir, refdir, refdir))\n\n        logger.info(\" - Prestaging REFCASE (%s) to %s\" % (refdir, rundir))\n\n        # prestage the reference case's files.\n\n        if (not os.path.exists(rundir)):\n            logger.debug(\"Creating run directory: %s\"%rundir)\n            os.makedirs(rundir)\n\n        for rcfile in glob.iglob(os.path.join(refdir,\"*%s*\"%run_refcase)):\n            logger.debug(\"Staging file %s\"%rcfile)\n            rcbaseline = os.path.basename(rcfile)\n            if not os.path.exists(\"%s/%s\" % (rundir, rcbaseline)):\n                os.symlink(rcfile, \"%s/%s\" % ((rundir, rcbaseline)))\n\n        # copy the refcases' rpointer files to the run directory\n        for rpointerfile in  glob.iglob(os.path.join(\"%s\",\"*rpointer*\") % (refdir)):\n            logger.debug(\"Copy rpointer %s\"%rpointerfile)\n            shutil.copy(rpointerfile, rundir)\n\n\n        for cam2file in  glob.iglob(os.path.join(\"%s\",\"*.cam2.*\") % rundir):\n            camfile = cam2file.replace(\"cam2\", \"cam\")\n            os.symlink(cam2file, camfile)\n\ndef check_input_data(case, svn_loc=None, input_data_root=None, data_list_dir=\"Buildconf\", download=False):\n    \"\"\"\n    Return True if no files missing\n    \"\"\"\n    # Fill in defaults as needed\n    svn_loc = SVN_LOCS[get_model()] if svn_loc is None else svn_loc\n    input_data_root = case.get_value(\"DIN_LOC_ROOT\") if input_data_root is None else input_data_root\n\n    expect(os.path.isdir(input_data_root), \"Invalid input_data_root directory: '%s'\" % input_data_root)\n    expect(os.path.isdir(data_list_dir), \"Invalid data_list_dir directory: '%s'\" % data_list_dir)\n\n    data_list_files = find_files(data_list_dir, \"*.input_data_list\")\n    expect(data_list_files, \"No .input_data_list files found in dir '%s'\" % data_list_dir)\n\n    no_files_missing = True\n    for data_list_file in data_list_files:\n        logging.info(\"Loading input file: '%s'\" % data_list_file)\n        with open(data_list_file, \"r\") as fd:\n            lines = fd.readlines()\n\n        for line in lines:\n            line = line.strip()\n            if (line and not line.startswith(\"#\")):\n                tokens = line.split('=')\n                description, full_path = tokens[0].strip(), tokens[1].strip()\n                if(full_path):\n                    # expand xml variables\n                    full_path = case.get_resolved_value(full_path)\n                    rel_path  = full_path.replace(input_data_root, \"\")\n                    model = os.path.basename(data_list_file).split('.')[0]\n\n                    if (\"/\" in rel_path and rel_path == full_path):\n                        # User pointing to a file outside of input_data_root, we cannot determine\n                        # rel_path, and so cannot download the file. If it already exists, we can\n                        # proceed\n                        if not os.path.exists(full_path):\n                            logging.warning(\"  Model %s missing file %s = '%s'\" % (model, description, full_path))\n                            if download:\n                                logging.warning(\"    Cannot download file since it lives outside of the input_data_root '%s'\" % input_data_root)\n                            no_files_missing = False\n                        else:\n                            logging.info(\"  Found input file: '%s'\" % full_path)\n\n                    else:\n                        # There are some special values of rel_path that\n                        # we need to ignore - some of the component models\n                        # set things like 'NULL' or 'same_as_TS' -\n                        # basically if rel_path does not contain '/' (a\n                        # directory tree) you can assume it's a special\n                        # value and ignore it (perhaps with a warning)\n                        if (\"/\" in rel_path and not os.path.exists(full_path)):\n                            logging.warning(\"  Model %s missing file %s = '%s'\" % (model,description,full_path))\n\n                            if (download):\n                                success = download_if_in_repo(svn_loc, input_data_root, rel_path)\n                                if (not success):\n                                    # If ACME, try CESM repo as backup\n                                    if (get_model() == \"acme\" and svn_loc != SVN_LOCS[\"cesm\"]):\n                                        success = download_if_in_repo(SVN_LOCS[\"cesm\"], input_data_root, rel_path)\n                                        if (not success):\n                                            no_files_missing = False\n                                    else:\n                                        no_files_missing = False\n                            # if not download\n                            else:\n                                no_files_missing = False\n                        else:\n                            logging.info(\"  Already had input file: '%s'\" % full_path)\n\n                else:\n                    model = os.path.basename(data_list_file).split('.')[0]\n                    logging.warning(\"Model %s no file specified for %s\"%(model,description))\n\n    return no_files_missing\n", "path": "utils/python/CIME/check_input_data.py"}]}