We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
PIO_NUMTASKS is getting reset
Setting env_run.xml's PIO_STRIDE to default -99 and PIO_NUMTASKS to 64 resets both PIO_STRIDE and PIO_NUMTASKS to different default values at run-time.

XML comments:
```
    <entry id="PIO_STRIDE">
      <desc>
     stride in compute comm of io tasks for each component, if this value is -99 it will 
     be computed based on PIO_NUMTASKS and number of compute tasks
    </desc>
```

but instead of 64, PIO_NUMTASKS is getting reset to 8 or 16 depending on the number of component's MPI tasks.

Note that setting PIO_STRIDE to a specific value and leaving PIO_NUMTASKS at default -99 works as expected: PIO_NUMTASKS is set to NTASKS / PIO_STRIDE.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/lib/CIME/case_run.py`
Content:
```
1 from CIME.XML.standard_module_setup import *
2 from CIME.case_submit               import submit
3 from CIME.utils                     import gzip_existing_file, new_lid, run_and_log_case_status
4 from CIME.check_lockedfiles         import check_lockedfiles
5 from CIME.get_timing                import get_timing
6 from CIME.provenance                import save_prerun_provenance, save_postrun_provenance
7 from CIME.preview_namelists         import create_namelists
8 from CIME.case_st_archive           import case_st_archive, restore_from_archive
9 
10 import shutil, time, sys, os, glob
11 
12 logger = logging.getLogger(__name__)
13 
14 ###############################################################################
15 def pre_run_check(case, lid, skip_pnl=False):
16 ###############################################################################
17 
18     # Pre run initialization code..
19     caseroot = case.get_value("CASEROOT")
20     din_loc_root = case.get_value("DIN_LOC_ROOT")
21     batchsubmit = case.get_value("BATCHSUBMIT")
22     mpilib = case.get_value("MPILIB")
23     rundir = case.get_value("RUNDIR")
24     build_complete = case.get_value("BUILD_COMPLETE")
25 
26     if case.get_value("TESTCASE") == "PFS":
27         env_mach_pes = os.path.join(caseroot,"env_mach_pes.xml")
28         shutil.copy(env_mach_pes,"{}.{}".format(env_mach_pes, lid))
29 
30     # check for locked files.
31     check_lockedfiles(case.get_value("CASEROOT"))
32     logger.debug("check_lockedfiles OK")
33 
34     # check that build is done
35     expect(build_complete,
36            "BUILD_COMPLETE is not true\nPlease rebuild the model interactively")
37     logger.debug("build complete is {} ".format(build_complete))
38 
39     # load the module environment...
40     case.load_env()
41 
42     # set environment variables
43     # This is a requirement for yellowstone only
44     if mpilib == "mpi-serial" and "MP_MPILIB" in os.environ:
45         del os.environ["MP_MPILIB"]
46     else:
47         os.environ["MPILIB"] = mpilib
48 
49     if batchsubmit is None or len(batchsubmit) == 0:
50         os.environ["LBQUERY"] = "FALSE"
51         os.environ["BATCHQUERY"] = "undefined"
52     elif batchsubmit == 'UNSET':
53         os.environ["LBQUERY"] = "FALSE"
54         os.environ["BATCHQUERY"] = "undefined"
55     else:
56         os.environ["LBQUERY"] = "TRUE"
57 
58     # create the timing directories, optionally cleaning them if needed.
59     if not os.path.isdir(rundir):
60         os.mkdir(rundir)
61 
62     if os.path.isdir(os.path.join(rundir, "timing")):
63         shutil.rmtree(os.path.join(rundir, "timing"))
64 
65     os.makedirs(os.path.join(rundir, "timing", "checkpoints"))
66 
67     # This needs to be done everytime the LID changes in order for log files to be set up correctly
68     # The following also needs to be called in case a user changes a user_nl_xxx file OR an env_run.xml
69     # variable while the job is in the queue
70     if not skip_pnl:
71         create_namelists(case)
72 
73     logger.info("-------------------------------------------------------------------------")
74     logger.info(" - Prestage required restarts into {}".format(rundir))
75     logger.info(" - Case input data directory (DIN_LOC_ROOT) is {} ".format(din_loc_root))
76     logger.info(" - Checking for required input datasets in DIN_LOC_ROOT")
77     logger.info("-------------------------------------------------------------------------")
78 
79 ###############################################################################
80 def _run_model_impl(case, lid, skip_pnl=False):
81 ###############################################################################
82 
83     pre_run_check(case, lid, skip_pnl=skip_pnl)
84 
85     model = case.get_value("MODEL")
86 
87     # Set OMP_NUM_THREADS
88     env_mach_pes = case.get_env("mach_pes")
89     comp_classes = case.get_values("COMP_CLASSES")
90     thread_count = env_mach_pes.get_max_thread_count(comp_classes)
91     os.environ["OMP_NUM_THREADS"] = str(thread_count)
92 
93     # Run the model
94     logger.info("{} MODEL EXECUTION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
95 
96     cmd = case.get_mpirun_cmd(job="case.run")
97     cmd = case.get_resolved_value(cmd)
98     logger.info("run command is {} ".format(cmd))
99 
100     rundir = case.get_value("RUNDIR")
101     loop = True
102 
103     while loop:
104         loop = False
105         stat = run_cmd(cmd, from_dir=rundir)[0]
106         model_logfile = os.path.join(rundir, model + ".log." + lid)
107         # Determine if failure was due to a failed node, if so, try to restart
108         if stat != 0:
109             node_fail_re = case.get_value("NODE_FAIL_REGEX")
110             if node_fail_re:
111                 node_fail_regex = re.compile(node_fail_re)
112                 model_logfile = os.path.join(rundir, model + ".log." + lid)
113                 if os.path.exists(model_logfile):
114                     num_fails = len(node_fail_regex.findall(open(model_logfile, 'r').read()))
115                     if num_fails > 0 and case.spare_nodes >= num_fails:
116                         # We failed due to node failure!
117                         logger.warning("Detected model run failed due to node failure, restarting")
118 
119                         # Archive the last consistent set of restart files and restore them
120                         case_st_archive(case, no_resubmit=True)
121                         restore_from_archive(case)
122 
123                         orig_cont = case.get_value("CONTINUE_RUN")
124                         if not orig_cont:
125                             case.set_value("CONTINUE_RUN", True)
126                             create_namelists(case)
127 
128                         lid = new_lid()
129                         loop = True
130 
131                         case.spare_nodes -= num_fails
132 
133             if not loop:
134                 # We failed and we're not restarting
135                 expect(False, "RUN FAIL: Command '{}' failed\nSee log file for details: {}".format(cmd, model_logfile))
136 
137     logger.info("{} MODEL EXECUTION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
138 
139     post_run_check(case, lid)
140 
141     return lid
142 
143 ###############################################################################
144 def run_model(case, lid, skip_pnl=False):
145 ###############################################################################
146     functor = lambda: _run_model_impl(case, lid, skip_pnl=skip_pnl)
147     return run_and_log_case_status(functor, "case.run", caseroot=case.get_value("CASEROOT"))
148 
149 ###############################################################################
150 def post_run_check(case, lid):
151 ###############################################################################
152 
153     rundir = case.get_value("RUNDIR")
154     model = case.get_value("MODEL")
155 
156     # find the last model.log and cpl.log
157     model_logfile = os.path.join(rundir, model + ".log." + lid)
158     cpl_logfile = os.path.join(rundir, "cpl" + ".log." + lid)
159 
160     if not os.path.isfile(model_logfile):
161         expect(False, "Model did not complete, no {} log file ".format(model_logfile))
162     elif not os.path.isfile(cpl_logfile):
163         expect(False, "Model did not complete, no cpl log file '{}'".format(cpl_logfile))
164     elif os.stat(model_logfile).st_size == 0:
165         expect(False, "Run FAILED")
166     else:
167         with open(cpl_logfile, 'r') as fd:
168             if 'SUCCESSFUL TERMINATION' not in fd.read():
169                 expect(False, "Model did not complete - see {} \n ".format(cpl_logfile))
170 
171 ###############################################################################
172 def save_logs(case, lid):
173 ###############################################################################
174     logdir = case.get_value("LOGDIR")
175     if logdir is not None and len(logdir) > 0:
176         if not os.path.isdir(logdir):
177             os.makedirs(logdir)
178 
179         caseroot = case.get_value("CASEROOT")
180         rundir = case.get_value("RUNDIR")
181         logfiles = glob.glob(os.path.join(rundir, "*.log.{}".format(lid)))
182         for logfile in logfiles:
183             if os.path.isfile(logfile):
184                 logfile_gz = gzip_existing_file(logfile)
185                 shutil.copy(logfile_gz,
186                             os.path.join(caseroot, logdir, os.path.basename(logfile_gz)))
187 
188 ###############################################################################
189 def resubmit_check(case):
190 ###############################################################################
191 
192     # check to see if we need to do resubmission from this particular job,
193     # Note that Mira requires special logic
194 
195     dout_s = case.get_value("DOUT_S")
196     logger.warn("dout_s {} ".format(dout_s))
197     mach = case.get_value("MACH")
198     logger.warn("mach {} ".format(mach))
199     testcase = case.get_value("TESTCASE")
200     resubmit_num = case.get_value("RESUBMIT")
201     logger.warn("resubmit_num {}".format(resubmit_num))
202     # If dout_s is True than short-term archiving handles the resubmit
203     # If dout_s is True and machine is mira submit the st_archive script
204     resubmit = False
205     if not dout_s and resubmit_num > 0:
206         resubmit = True
207     elif dout_s and mach == 'mira':
208         caseroot = case.get_value("CASEROOT")
209         cimeroot = case.get_value("CIMEROOT")
210         cmd = "ssh cooleylogin1 'cd {}; CIMEROOT={} ./case.submit {} --job case.st_archive'".format(caseroot, cimeroot, caseroot)
211         run_cmd(cmd, verbose=True)
212 
213     if resubmit:
214         if testcase is not None and testcase in ['ERR']:
215             job = "case.test"
216         else:
217             job = "case.run"
218         submit(case, job=job, resubmit=True)
219 
220 ###############################################################################
221 def do_external(script_name, caseroot, rundir, lid, prefix):
222 ###############################################################################
223     filename = "{}.external.log.{}".format(prefix, lid)
224     outfile = os.path.join(rundir, filename)
225     cmd = script_name + " 1> {} {} 2>&1".format(outfile, caseroot)
226     logger.info("running {}".format(script_name))
227     run_cmd_no_fail(cmd)
228 
229 ###############################################################################
230 def do_data_assimilation(da_script, caseroot, cycle, lid, rundir):
231 ###############################################################################
232     filename = "da.log.{}".format(lid)
233     outfile = os.path.join(rundir, filename)
234     cmd = da_script + " 1> {} {} {:d} 2>&1".format(outfile, caseroot, cycle)
235     logger.info("running {}".format(da_script))
236     run_cmd_no_fail(cmd)
237 
238 ###############################################################################
239 def case_run(case, skip_pnl=False):
240 ###############################################################################
241     # Set up the run, run the model, do the postrun steps
242     run_with_submit = case.get_value("RUN_WITH_SUBMIT")
243     expect(run_with_submit,
244            "You are not calling the run script via the submit script. "
245            "As a result, short-term archiving will not be called automatically."
246            "Please submit your run using the submit script like so:"
247            " ./case.submit")
248 
249     # Forces user to use case.submit if they re-submit
250     if case.get_value("TESTCASE") is None:
251         case.set_value("RUN_WITH_SUBMIT", False)
252 
253     prerun_script = case.get_value("PRERUN_SCRIPT")
254     postrun_script = case.get_value("POSTRUN_SCRIPT")
255 
256     data_assimilation = case.get_value("DATA_ASSIMILATION")
257     data_assimilation_cycles = case.get_value("DATA_ASSIMILATION_CYCLES")
258     data_assimilation_script = case.get_value("DATA_ASSIMILATION_SCRIPT")
259 
260     # set up the LID
261     lid = new_lid()
262 
263     save_prerun_provenance(case)
264 
265     for cycle in range(data_assimilation_cycles):
266         # After the first DA cycle, runs are restart runs
267         if cycle > 0:
268             case.set_value("CONTINUE_RUN", "TRUE")
269             lid = new_lid()
270 
271         if prerun_script:
272             do_external(prerun_script, case.get_value("CASEROOT"), case.get_value("RUNDIR"),
273                         lid, prefix="prerun")
274 
275         lid = run_model(case, lid, skip_pnl)
276         save_logs(case, lid)       # Copy log files back to caseroot
277         if case.get_value("CHECK_TIMING") or case.get_value("SAVE_TIMING"):
278             get_timing(case, lid)     # Run the getTiming script
279 
280         if data_assimilation:
281             do_data_assimilation(data_assimilation_script, case.get_value("CASEROOT"), cycle, lid,
282                                  case.get_value("RUNDIR"))
283 
284         if postrun_script:
285             do_external(postrun_script, case.get_value("CASEROOT"), case.get_value("RUNDIR"),
286                         lid, prefix="postrun")
287 
288         save_postrun_provenance(case)
289 
290     logger.warn("check for resubmit")
291     resubmit_check(case)
292 
293     return True
294
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/lib/CIME/case_run.py b/scripts/lib/CIME/case_run.py
--- a/scripts/lib/CIME/case_run.py
+++ b/scripts/lib/CIME/case_run.py
@@ -269,8 +269,10 @@
             lid = new_lid()
 
         if prerun_script:
+            case.flush()
             do_external(prerun_script, case.get_value("CASEROOT"), case.get_value("RUNDIR"),
                         lid, prefix="prerun")
+            case.read_xml()
 
         lid = run_model(case, lid, skip_pnl)
         save_logs(case, lid)       # Copy log files back to caseroot
@@ -278,12 +280,16 @@
             get_timing(case, lid)     # Run the getTiming script
 
         if data_assimilation:
+            case.flush()
             do_data_assimilation(data_assimilation_script, case.get_value("CASEROOT"), cycle, lid,
                                  case.get_value("RUNDIR"))
+            case.read_xml()
 
         if postrun_script:
+            case.flush()
             do_external(postrun_script, case.get_value("CASEROOT"), case.get_value("RUNDIR"),
                         lid, prefix="postrun")
+            case.read_xml()
 
         save_postrun_provenance(case)

{"golden_diff": "diff --git a/scripts/lib/CIME/case_run.py b/scripts/lib/CIME/case_run.py\n--- a/scripts/lib/CIME/case_run.py\n+++ b/scripts/lib/CIME/case_run.py\n@@ -269,8 +269,10 @@\n             lid = new_lid()\n \n         if prerun_script:\n+            case.flush()\n             do_external(prerun_script, case.get_value(\"CASEROOT\"), case.get_value(\"RUNDIR\"),\n                         lid, prefix=\"prerun\")\n+            case.read_xml()\n \n         lid = run_model(case, lid, skip_pnl)\n         save_logs(case, lid)       # Copy log files back to caseroot\n@@ -278,12 +280,16 @@\n             get_timing(case, lid)     # Run the getTiming script\n \n         if data_assimilation:\n+            case.flush()\n             do_data_assimilation(data_assimilation_script, case.get_value(\"CASEROOT\"), cycle, lid,\n                                  case.get_value(\"RUNDIR\"))\n+            case.read_xml()\n \n         if postrun_script:\n+            case.flush()\n             do_external(postrun_script, case.get_value(\"CASEROOT\"), case.get_value(\"RUNDIR\"),\n                         lid, prefix=\"postrun\")\n+            case.read_xml()\n \n         save_postrun_provenance(case)\n", "issue": "PIO_NUMTASKS is getting reset\nSetting env_run.xml's PIO_STRIDE to default -99 and PIO_NUMTASKS to 64 resets both PIO_STRIDE and PIO_NUMTASKS to different default values at run-time.\r\n\r\nXML comments:\r\n```\r\n    <entry id=\"PIO_STRIDE\">\r\n      <desc>\r\n     stride in compute comm of io tasks for each component, if this value is -99 it will \r\n     be computed based on PIO_NUMTASKS and number of compute tasks\r\n    </desc>\r\n```\r\n\r\nbut instead of 64, PIO_NUMTASKS is getting reset to 8 or 16 depending on the number of component's MPI tasks.\r\n\r\nNote that setting PIO_STRIDE to a specific value and leaving PIO_NUMTASKS at default -99 works as expected: PIO_NUMTASKS is set to NTASKS / PIO_STRIDE.\n", "before_files": [{"content": "from CIME.XML.standard_module_setup import *\nfrom CIME.case_submit               import submit\nfrom CIME.utils                     import gzip_existing_file, new_lid, run_and_log_case_status\nfrom CIME.check_lockedfiles         import check_lockedfiles\nfrom CIME.get_timing                import get_timing\nfrom CIME.provenance                import save_prerun_provenance, save_postrun_provenance\nfrom CIME.preview_namelists         import create_namelists\nfrom CIME.case_st_archive           import case_st_archive, restore_from_archive\n\nimport shutil, time, sys, os, glob\n\nlogger = logging.getLogger(__name__)\n\n###############################################################################\ndef pre_run_check(case, lid, skip_pnl=False):\n###############################################################################\n\n    # Pre run initialization code..\n    caseroot = case.get_value(\"CASEROOT\")\n    din_loc_root = case.get_value(\"DIN_LOC_ROOT\")\n    batchsubmit = case.get_value(\"BATCHSUBMIT\")\n    mpilib = case.get_value(\"MPILIB\")\n    rundir = case.get_value(\"RUNDIR\")\n    build_complete = case.get_value(\"BUILD_COMPLETE\")\n\n    if case.get_value(\"TESTCASE\") == \"PFS\":\n        env_mach_pes = os.path.join(caseroot,\"env_mach_pes.xml\")\n        shutil.copy(env_mach_pes,\"{}.{}\".format(env_mach_pes, lid))\n\n    # check for locked files.\n    check_lockedfiles(case.get_value(\"CASEROOT\"))\n    logger.debug(\"check_lockedfiles OK\")\n\n    # check that build is done\n    expect(build_complete,\n           \"BUILD_COMPLETE is not true\\nPlease rebuild the model interactively\")\n    logger.debug(\"build complete is {} \".format(build_complete))\n\n    # load the module environment...\n    case.load_env()\n\n    # set environment variables\n    # This is a requirement for yellowstone only\n    if mpilib == \"mpi-serial\" and \"MP_MPILIB\" in os.environ:\n        del os.environ[\"MP_MPILIB\"]\n    else:\n        os.environ[\"MPILIB\"] = mpilib\n\n    if batchsubmit is None or len(batchsubmit) == 0:\n        os.environ[\"LBQUERY\"] = \"FALSE\"\n        os.environ[\"BATCHQUERY\"] = \"undefined\"\n    elif batchsubmit == 'UNSET':\n        os.environ[\"LBQUERY\"] = \"FALSE\"\n        os.environ[\"BATCHQUERY\"] = \"undefined\"\n    else:\n        os.environ[\"LBQUERY\"] = \"TRUE\"\n\n    # create the timing directories, optionally cleaning them if needed.\n    if not os.path.isdir(rundir):\n        os.mkdir(rundir)\n\n    if os.path.isdir(os.path.join(rundir, \"timing\")):\n        shutil.rmtree(os.path.join(rundir, \"timing\"))\n\n    os.makedirs(os.path.join(rundir, \"timing\", \"checkpoints\"))\n\n    # This needs to be done everytime the LID changes in order for log files to be set up correctly\n    # The following also needs to be called in case a user changes a user_nl_xxx file OR an env_run.xml\n    # variable while the job is in the queue\n    if not skip_pnl:\n        create_namelists(case)\n\n    logger.info(\"-------------------------------------------------------------------------\")\n    logger.info(\" - Prestage required restarts into {}\".format(rundir))\n    logger.info(\" - Case input data directory (DIN_LOC_ROOT) is {} \".format(din_loc_root))\n    logger.info(\" - Checking for required input datasets in DIN_LOC_ROOT\")\n    logger.info(\"-------------------------------------------------------------------------\")\n\n###############################################################################\ndef _run_model_impl(case, lid, skip_pnl=False):\n###############################################################################\n\n    pre_run_check(case, lid, skip_pnl=skip_pnl)\n\n    model = case.get_value(\"MODEL\")\n\n    # Set OMP_NUM_THREADS\n    env_mach_pes = case.get_env(\"mach_pes\")\n    comp_classes = case.get_values(\"COMP_CLASSES\")\n    thread_count = env_mach_pes.get_max_thread_count(comp_classes)\n    os.environ[\"OMP_NUM_THREADS\"] = str(thread_count)\n\n    # Run the model\n    logger.info(\"{} MODEL EXECUTION BEGINS HERE\".format(time.strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n    cmd = case.get_mpirun_cmd(job=\"case.run\")\n    cmd = case.get_resolved_value(cmd)\n    logger.info(\"run command is {} \".format(cmd))\n\n    rundir = case.get_value(\"RUNDIR\")\n    loop = True\n\n    while loop:\n        loop = False\n        stat = run_cmd(cmd, from_dir=rundir)[0]\n        model_logfile = os.path.join(rundir, model + \".log.\" + lid)\n        # Determine if failure was due to a failed node, if so, try to restart\n        if stat != 0:\n            node_fail_re = case.get_value(\"NODE_FAIL_REGEX\")\n            if node_fail_re:\n                node_fail_regex = re.compile(node_fail_re)\n                model_logfile = os.path.join(rundir, model + \".log.\" + lid)\n                if os.path.exists(model_logfile):\n                    num_fails = len(node_fail_regex.findall(open(model_logfile, 'r').read()))\n                    if num_fails > 0 and case.spare_nodes >= num_fails:\n                        # We failed due to node failure!\n                        logger.warning(\"Detected model run failed due to node failure, restarting\")\n\n                        # Archive the last consistent set of restart files and restore them\n                        case_st_archive(case, no_resubmit=True)\n                        restore_from_archive(case)\n\n                        orig_cont = case.get_value(\"CONTINUE_RUN\")\n                        if not orig_cont:\n                            case.set_value(\"CONTINUE_RUN\", True)\n                            create_namelists(case)\n\n                        lid = new_lid()\n                        loop = True\n\n                        case.spare_nodes -= num_fails\n\n            if not loop:\n                # We failed and we're not restarting\n                expect(False, \"RUN FAIL: Command '{}' failed\\nSee log file for details: {}\".format(cmd, model_logfile))\n\n    logger.info(\"{} MODEL EXECUTION HAS FINISHED\".format(time.strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n    post_run_check(case, lid)\n\n    return lid\n\n###############################################################################\ndef run_model(case, lid, skip_pnl=False):\n###############################################################################\n    functor = lambda: _run_model_impl(case, lid, skip_pnl=skip_pnl)\n    return run_and_log_case_status(functor, \"case.run\", caseroot=case.get_value(\"CASEROOT\"))\n\n###############################################################################\ndef post_run_check(case, lid):\n###############################################################################\n\n    rundir = case.get_value(\"RUNDIR\")\n    model = case.get_value(\"MODEL\")\n\n    # find the last model.log and cpl.log\n    model_logfile = os.path.join(rundir, model + \".log.\" + lid)\n    cpl_logfile = os.path.join(rundir, \"cpl\" + \".log.\" + lid)\n\n    if not os.path.isfile(model_logfile):\n        expect(False, \"Model did not complete, no {} log file \".format(model_logfile))\n    elif not os.path.isfile(cpl_logfile):\n        expect(False, \"Model did not complete, no cpl log file '{}'\".format(cpl_logfile))\n    elif os.stat(model_logfile).st_size == 0:\n        expect(False, \"Run FAILED\")\n    else:\n        with open(cpl_logfile, 'r') as fd:\n            if 'SUCCESSFUL TERMINATION' not in fd.read():\n                expect(False, \"Model did not complete - see {} \\n \".format(cpl_logfile))\n\n###############################################################################\ndef save_logs(case, lid):\n###############################################################################\n    logdir = case.get_value(\"LOGDIR\")\n    if logdir is not None and len(logdir) > 0:\n        if not os.path.isdir(logdir):\n            os.makedirs(logdir)\n\n        caseroot = case.get_value(\"CASEROOT\")\n        rundir = case.get_value(\"RUNDIR\")\n        logfiles = glob.glob(os.path.join(rundir, \"*.log.{}\".format(lid)))\n        for logfile in logfiles:\n            if os.path.isfile(logfile):\n                logfile_gz = gzip_existing_file(logfile)\n                shutil.copy(logfile_gz,\n                            os.path.join(caseroot, logdir, os.path.basename(logfile_gz)))\n\n###############################################################################\ndef resubmit_check(case):\n###############################################################################\n\n    # check to see if we need to do resubmission from this particular job,\n    # Note that Mira requires special logic\n\n    dout_s = case.get_value(\"DOUT_S\")\n    logger.warn(\"dout_s {} \".format(dout_s))\n    mach = case.get_value(\"MACH\")\n    logger.warn(\"mach {} \".format(mach))\n    testcase = case.get_value(\"TESTCASE\")\n    resubmit_num = case.get_value(\"RESUBMIT\")\n    logger.warn(\"resubmit_num {}\".format(resubmit_num))\n    # If dout_s is True than short-term archiving handles the resubmit\n    # If dout_s is True and machine is mira submit the st_archive script\n    resubmit = False\n    if not dout_s and resubmit_num > 0:\n        resubmit = True\n    elif dout_s and mach == 'mira':\n        caseroot = case.get_value(\"CASEROOT\")\n        cimeroot = case.get_value(\"CIMEROOT\")\n        cmd = \"ssh cooleylogin1 'cd {}; CIMEROOT={} ./case.submit {} --job case.st_archive'\".format(caseroot, cimeroot, caseroot)\n        run_cmd(cmd, verbose=True)\n\n    if resubmit:\n        if testcase is not None and testcase in ['ERR']:\n            job = \"case.test\"\n        else:\n            job = \"case.run\"\n        submit(case, job=job, resubmit=True)\n\n###############################################################################\ndef do_external(script_name, caseroot, rundir, lid, prefix):\n###############################################################################\n    filename = \"{}.external.log.{}\".format(prefix, lid)\n    outfile = os.path.join(rundir, filename)\n    cmd = script_name + \" 1> {} {} 2>&1\".format(outfile, caseroot)\n    logger.info(\"running {}\".format(script_name))\n    run_cmd_no_fail(cmd)\n\n###############################################################################\ndef do_data_assimilation(da_script, caseroot, cycle, lid, rundir):\n###############################################################################\n    filename = \"da.log.{}\".format(lid)\n    outfile = os.path.join(rundir, filename)\n    cmd = da_script + \" 1> {} {} {:d} 2>&1\".format(outfile, caseroot, cycle)\n    logger.info(\"running {}\".format(da_script))\n    run_cmd_no_fail(cmd)\n\n###############################################################################\ndef case_run(case, skip_pnl=False):\n###############################################################################\n    # Set up the run, run the model, do the postrun steps\n    run_with_submit = case.get_value(\"RUN_WITH_SUBMIT\")\n    expect(run_with_submit,\n           \"You are not calling the run script via the submit script. \"\n           \"As a result, short-term archiving will not be called automatically.\"\n           \"Please submit your run using the submit script like so:\"\n           \" ./case.submit\")\n\n    # Forces user to use case.submit if they re-submit\n    if case.get_value(\"TESTCASE\") is None:\n        case.set_value(\"RUN_WITH_SUBMIT\", False)\n\n    prerun_script = case.get_value(\"PRERUN_SCRIPT\")\n    postrun_script = case.get_value(\"POSTRUN_SCRIPT\")\n\n    data_assimilation = case.get_value(\"DATA_ASSIMILATION\")\n    data_assimilation_cycles = case.get_value(\"DATA_ASSIMILATION_CYCLES\")\n    data_assimilation_script = case.get_value(\"DATA_ASSIMILATION_SCRIPT\")\n\n    # set up the LID\n    lid = new_lid()\n\n    save_prerun_provenance(case)\n\n    for cycle in range(data_assimilation_cycles):\n        # After the first DA cycle, runs are restart runs\n        if cycle > 0:\n            case.set_value(\"CONTINUE_RUN\", \"TRUE\")\n            lid = new_lid()\n\n        if prerun_script:\n            do_external(prerun_script, case.get_value(\"CASEROOT\"), case.get_value(\"RUNDIR\"),\n                        lid, prefix=\"prerun\")\n\n        lid = run_model(case, lid, skip_pnl)\n        save_logs(case, lid)       # Copy log files back to caseroot\n        if case.get_value(\"CHECK_TIMING\") or case.get_value(\"SAVE_TIMING\"):\n            get_timing(case, lid)     # Run the getTiming script\n\n        if data_assimilation:\n            do_data_assimilation(data_assimilation_script, case.get_value(\"CASEROOT\"), cycle, lid,\n                                 case.get_value(\"RUNDIR\"))\n\n        if postrun_script:\n            do_external(postrun_script, case.get_value(\"CASEROOT\"), case.get_value(\"RUNDIR\"),\n                        lid, prefix=\"postrun\")\n\n        save_postrun_provenance(case)\n\n    logger.warn(\"check for resubmit\")\n    resubmit_check(case)\n\n    return True\n", "path": "scripts/lib/CIME/case_run.py"}], "after_files": [{"content": "from CIME.XML.standard_module_setup import *\nfrom CIME.case_submit               import submit\nfrom CIME.utils                     import gzip_existing_file, new_lid, run_and_log_case_status\nfrom CIME.check_lockedfiles         import check_lockedfiles\nfrom CIME.get_timing                import get_timing\nfrom CIME.provenance                import save_prerun_provenance, save_postrun_provenance\nfrom CIME.preview_namelists         import create_namelists\nfrom CIME.case_st_archive           import case_st_archive, restore_from_archive\n\nimport shutil, time, sys, os, glob\n\nlogger = logging.getLogger(__name__)\n\n###############################################################################\ndef pre_run_check(case, lid, skip_pnl=False):\n###############################################################################\n\n    # Pre run initialization code..\n    caseroot = case.get_value(\"CASEROOT\")\n    din_loc_root = case.get_value(\"DIN_LOC_ROOT\")\n    batchsubmit = case.get_value(\"BATCHSUBMIT\")\n    mpilib = case.get_value(\"MPILIB\")\n    rundir = case.get_value(\"RUNDIR\")\n    build_complete = case.get_value(\"BUILD_COMPLETE\")\n\n    if case.get_value(\"TESTCASE\") == \"PFS\":\n        env_mach_pes = os.path.join(caseroot,\"env_mach_pes.xml\")\n        shutil.copy(env_mach_pes,\"{}.{}\".format(env_mach_pes, lid))\n\n    # check for locked files.\n    check_lockedfiles(case.get_value(\"CASEROOT\"))\n    logger.debug(\"check_lockedfiles OK\")\n\n    # check that build is done\n    expect(build_complete,\n           \"BUILD_COMPLETE is not true\\nPlease rebuild the model interactively\")\n    logger.debug(\"build complete is {} \".format(build_complete))\n\n    # load the module environment...\n    case.load_env()\n\n    # set environment variables\n    # This is a requirement for yellowstone only\n    if mpilib == \"mpi-serial\" and \"MP_MPILIB\" in os.environ:\n        del os.environ[\"MP_MPILIB\"]\n    else:\n        os.environ[\"MPILIB\"] = mpilib\n\n    if batchsubmit is None or len(batchsubmit) == 0:\n        os.environ[\"LBQUERY\"] = \"FALSE\"\n        os.environ[\"BATCHQUERY\"] = \"undefined\"\n    elif batchsubmit == 'UNSET':\n        os.environ[\"LBQUERY\"] = \"FALSE\"\n        os.environ[\"BATCHQUERY\"] = \"undefined\"\n    else:\n        os.environ[\"LBQUERY\"] = \"TRUE\"\n\n    # create the timing directories, optionally cleaning them if needed.\n    if not os.path.isdir(rundir):\n        os.mkdir(rundir)\n\n    if os.path.isdir(os.path.join(rundir, \"timing\")):\n        shutil.rmtree(os.path.join(rundir, \"timing\"))\n\n    os.makedirs(os.path.join(rundir, \"timing\", \"checkpoints\"))\n\n    # This needs to be done everytime the LID changes in order for log files to be set up correctly\n    # The following also needs to be called in case a user changes a user_nl_xxx file OR an env_run.xml\n    # variable while the job is in the queue\n    if not skip_pnl:\n        create_namelists(case)\n\n    logger.info(\"-------------------------------------------------------------------------\")\n    logger.info(\" - Prestage required restarts into {}\".format(rundir))\n    logger.info(\" - Case input data directory (DIN_LOC_ROOT) is {} \".format(din_loc_root))\n    logger.info(\" - Checking for required input datasets in DIN_LOC_ROOT\")\n    logger.info(\"-------------------------------------------------------------------------\")\n\n###############################################################################\ndef _run_model_impl(case, lid, skip_pnl=False):\n###############################################################################\n\n    pre_run_check(case, lid, skip_pnl=skip_pnl)\n\n    model = case.get_value(\"MODEL\")\n\n    # Set OMP_NUM_THREADS\n    env_mach_pes = case.get_env(\"mach_pes\")\n    comp_classes = case.get_values(\"COMP_CLASSES\")\n    thread_count = env_mach_pes.get_max_thread_count(comp_classes)\n    os.environ[\"OMP_NUM_THREADS\"] = str(thread_count)\n\n    # Run the model\n    logger.info(\"{} MODEL EXECUTION BEGINS HERE\".format(time.strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n    cmd = case.get_mpirun_cmd(job=\"case.run\")\n    cmd = case.get_resolved_value(cmd)\n    logger.info(\"run command is {} \".format(cmd))\n\n    rundir = case.get_value(\"RUNDIR\")\n    loop = True\n\n    while loop:\n        loop = False\n        stat = run_cmd(cmd, from_dir=rundir)[0]\n        model_logfile = os.path.join(rundir, model + \".log.\" + lid)\n        # Determine if failure was due to a failed node, if so, try to restart\n        if stat != 0:\n            node_fail_re = case.get_value(\"NODE_FAIL_REGEX\")\n            if node_fail_re:\n                node_fail_regex = re.compile(node_fail_re)\n                model_logfile = os.path.join(rundir, model + \".log.\" + lid)\n                if os.path.exists(model_logfile):\n                    num_fails = len(node_fail_regex.findall(open(model_logfile, 'r').read()))\n                    if num_fails > 0 and case.spare_nodes >= num_fails:\n                        # We failed due to node failure!\n                        logger.warning(\"Detected model run failed due to node failure, restarting\")\n\n                        # Archive the last consistent set of restart files and restore them\n                        case_st_archive(case, no_resubmit=True)\n                        restore_from_archive(case)\n\n                        orig_cont = case.get_value(\"CONTINUE_RUN\")\n                        if not orig_cont:\n                            case.set_value(\"CONTINUE_RUN\", True)\n                            create_namelists(case)\n\n                        lid = new_lid()\n                        loop = True\n\n                        case.spare_nodes -= num_fails\n\n            if not loop:\n                # We failed and we're not restarting\n                expect(False, \"RUN FAIL: Command '{}' failed\\nSee log file for details: {}\".format(cmd, model_logfile))\n\n    logger.info(\"{} MODEL EXECUTION HAS FINISHED\".format(time.strftime(\"%Y-%m-%d %H:%M:%S\")))\n\n    post_run_check(case, lid)\n\n    return lid\n\n###############################################################################\ndef run_model(case, lid, skip_pnl=False):\n###############################################################################\n    functor = lambda: _run_model_impl(case, lid, skip_pnl=skip_pnl)\n    return run_and_log_case_status(functor, \"case.run\", caseroot=case.get_value(\"CASEROOT\"))\n\n###############################################################################\ndef post_run_check(case, lid):\n###############################################################################\n\n    rundir = case.get_value(\"RUNDIR\")\n    model = case.get_value(\"MODEL\")\n\n    # find the last model.log and cpl.log\n    model_logfile = os.path.join(rundir, model + \".log.\" + lid)\n    cpl_logfile = os.path.join(rundir, \"cpl\" + \".log.\" + lid)\n\n    if not os.path.isfile(model_logfile):\n        expect(False, \"Model did not complete, no {} log file \".format(model_logfile))\n    elif not os.path.isfile(cpl_logfile):\n        expect(False, \"Model did not complete, no cpl log file '{}'\".format(cpl_logfile))\n    elif os.stat(model_logfile).st_size == 0:\n        expect(False, \"Run FAILED\")\n    else:\n        with open(cpl_logfile, 'r') as fd:\n            if 'SUCCESSFUL TERMINATION' not in fd.read():\n                expect(False, \"Model did not complete - see {} \\n \".format(cpl_logfile))\n\n###############################################################################\ndef save_logs(case, lid):\n###############################################################################\n    logdir = case.get_value(\"LOGDIR\")\n    if logdir is not None and len(logdir) > 0:\n        if not os.path.isdir(logdir):\n            os.makedirs(logdir)\n\n        caseroot = case.get_value(\"CASEROOT\")\n        rundir = case.get_value(\"RUNDIR\")\n        logfiles = glob.glob(os.path.join(rundir, \"*.log.{}\".format(lid)))\n        for logfile in logfiles:\n            if os.path.isfile(logfile):\n                logfile_gz = gzip_existing_file(logfile)\n                shutil.copy(logfile_gz,\n                            os.path.join(caseroot, logdir, os.path.basename(logfile_gz)))\n\n###############################################################################\ndef resubmit_check(case):\n###############################################################################\n\n    # check to see if we need to do resubmission from this particular job,\n    # Note that Mira requires special logic\n\n    dout_s = case.get_value(\"DOUT_S\")\n    logger.warn(\"dout_s {} \".format(dout_s))\n    mach = case.get_value(\"MACH\")\n    logger.warn(\"mach {} \".format(mach))\n    testcase = case.get_value(\"TESTCASE\")\n    resubmit_num = case.get_value(\"RESUBMIT\")\n    logger.warn(\"resubmit_num {}\".format(resubmit_num))\n    # If dout_s is True than short-term archiving handles the resubmit\n    # If dout_s is True and machine is mira submit the st_archive script\n    resubmit = False\n    if not dout_s and resubmit_num > 0:\n        resubmit = True\n    elif dout_s and mach == 'mira':\n        caseroot = case.get_value(\"CASEROOT\")\n        cimeroot = case.get_value(\"CIMEROOT\")\n        cmd = \"ssh cooleylogin1 'cd {}; CIMEROOT={} ./case.submit {} --job case.st_archive'\".format(caseroot, cimeroot, caseroot)\n        run_cmd(cmd, verbose=True)\n\n    if resubmit:\n        if testcase is not None and testcase in ['ERR']:\n            job = \"case.test\"\n        else:\n            job = \"case.run\"\n        submit(case, job=job, resubmit=True)\n\n###############################################################################\ndef do_external(script_name, caseroot, rundir, lid, prefix):\n###############################################################################\n    filename = \"{}.external.log.{}\".format(prefix, lid)\n    outfile = os.path.join(rundir, filename)\n    cmd = script_name + \" 1> {} {} 2>&1\".format(outfile, caseroot)\n    logger.info(\"running {}\".format(script_name))\n    run_cmd_no_fail(cmd)\n\n###############################################################################\ndef do_data_assimilation(da_script, caseroot, cycle, lid, rundir):\n###############################################################################\n    filename = \"da.log.{}\".format(lid)\n    outfile = os.path.join(rundir, filename)\n    cmd = da_script + \" 1> {} {} {:d} 2>&1\".format(outfile, caseroot, cycle)\n    logger.info(\"running {}\".format(da_script))\n    run_cmd_no_fail(cmd)\n\n###############################################################################\ndef case_run(case, skip_pnl=False):\n###############################################################################\n    # Set up the run, run the model, do the postrun steps\n    run_with_submit = case.get_value(\"RUN_WITH_SUBMIT\")\n    expect(run_with_submit,\n           \"You are not calling the run script via the submit script. \"\n           \"As a result, short-term archiving will not be called automatically.\"\n           \"Please submit your run using the submit script like so:\"\n           \" ./case.submit\")\n\n    # Forces user to use case.submit if they re-submit\n    if case.get_value(\"TESTCASE\") is None:\n        case.set_value(\"RUN_WITH_SUBMIT\", False)\n\n    prerun_script = case.get_value(\"PRERUN_SCRIPT\")\n    postrun_script = case.get_value(\"POSTRUN_SCRIPT\")\n\n    data_assimilation = case.get_value(\"DATA_ASSIMILATION\")\n    data_assimilation_cycles = case.get_value(\"DATA_ASSIMILATION_CYCLES\")\n    data_assimilation_script = case.get_value(\"DATA_ASSIMILATION_SCRIPT\")\n\n    # set up the LID\n    lid = new_lid()\n\n    save_prerun_provenance(case)\n\n    for cycle in range(data_assimilation_cycles):\n        # After the first DA cycle, runs are restart runs\n        if cycle > 0:\n            case.set_value(\"CONTINUE_RUN\", \"TRUE\")\n            lid = new_lid()\n\n        if prerun_script:\n            case.flush()\n            do_external(prerun_script, case.get_value(\"CASEROOT\"), case.get_value(\"RUNDIR\"),\n                        lid, prefix=\"prerun\")\n            case.read_xml()\n\n        lid = run_model(case, lid, skip_pnl)\n        save_logs(case, lid)       # Copy log files back to caseroot\n        if case.get_value(\"CHECK_TIMING\") or case.get_value(\"SAVE_TIMING\"):\n            get_timing(case, lid)     # Run the getTiming script\n\n        if data_assimilation:\n            case.flush()\n            do_data_assimilation(data_assimilation_script, case.get_value(\"CASEROOT\"), cycle, lid,\n                                 case.get_value(\"RUNDIR\"))\n            case.read_xml()\n\n        if postrun_script:\n            case.flush()\n            do_external(postrun_script, case.get_value(\"CASEROOT\"), case.get_value(\"RUNDIR\"),\n                        lid, prefix=\"postrun\")\n            case.read_xml()\n\n        save_postrun_provenance(case)\n\n    logger.warn(\"check for resubmit\")\n    resubmit_check(case)\n\n    return True\n", "path": "scripts/lib/CIME/case_run.py"}]}