12288631034

Committed 12 Dec 2024 02:24AM UTC coverage: 58.979% (+0.5%) from 58.509%

Build # 12288631034

Build Type

Pull #539

github

Committed by

web-flow

Commit Message

Merge 9135f89bd into da047828f

Pull Request Pull Request #539: Check CICE4 restart file dates

Run Details

57 of 62 new or added lines in 3 files covered. (91.94%)

32 existing lines in 2 files now uncovered.

2910 of 4934 relevant lines covered (58.98%)

1.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.01

/payu/models/cice.py

"""
The payu interface for the CICE model
-------------------------------------------------------------------------------
Contact: Marshall Ward <marshall.ward@anu.edu.au>
-------------------------------------------------------------------------------
Distributed as part of Payu, Copyright 2011 Marshall Ward
Licensed under the Apache License, Version 2.0
http://www.apache.org/licenses/LICENSE-2.0
"""

# Python 3 preparation
from __future__ import print_function

# Standard Library
import errno
import os
import sys
import shutil
import datetime
import struct
import re
import tarfile

# Extensions
import f90nml

# Local
import payu.calendar as cal
from payu.fsops import make_symlink
from payu.models.model import Model
from payu.namcouple import Namcouple


class Cice(Model):

    def __init__(self, expt, name, config):
        super(Cice, self).__init__(expt, name, config)

        self.model_type = 'cice'
        self.default_exec = 'cice'

        # Default repo details
        self.repo_url = 'https://github.com/CWSL/cice4.git'
        self.repo_tag = 'access'

        self.config_files = ['cice_in.nml']
        self.optional_config_files = ['input_ice.nml']

        self.ice_nml_fname = 'cice_in.nml'

        self.history_nml_fname = 'ice_history.nml'  # only used by payu

        self.set_timestep = self.set_local_timestep

        self.copy_inputs = False

        # regex patterns for matching log files. When empty, no logs compressed
        self.logs_to_compress = [r"iceout[0-9]{3}",
                                 r"debug\.root\.[0-9]{2}",
                                 r"ice_diag\.d",
                                 r"ice_diag_out"]
        self.log_tar_name = "logfiles.tar.gz"

    def set_model_pathnames(self):
        super(Cice, self).set_model_pathnames()

        self.build_exec_path = os.path.join(self.codebase_path,
                                            'build_access-om_360x300_6p')

        ice_nml_path = os.path.join(self.control_path, self.ice_nml_fname)
        self.ice_in = f90nml.read(ice_nml_path)

        # Assume local paths are relative to the work path
        setup_nml = self.ice_in['setup_nml']

        res_path = os.path.normpath(setup_nml['restart_dir'])
        input_dir = setup_nml.get('input_dir', None)

        if input_dir is None:
            # Default to reading and writing inputs/restarts in-place
            input_path = res_path
            init_path = res_path
        else:
            input_path = os.path.normpath(input_dir)
            init_path = input_path

        # Determine if there is a work input path from the path to the
        # grid.nc file. Older cice versions don't have a defined INPUT
        # directory, but it is implied by this path
        grid_nml = self.ice_in['grid_nml']
        path, _ = os.path.split(grid_nml['grid_file'])
        if path and not path == os.path.curdir:
            assert not os.path.isabs(path)
            path = os.path.normpath(path)
            # Get input_dir from grid_file path unless otherwise specified
            if input_dir is None:
                input_path = path
            else:
                if path != input_path:
                    print('payu: error: Grid file path in {nmlfile} '
                          '({path}) does not match input path '
                          '({inputpath})'.format(
                            nmlfile=self.ice_nml_fname,
                            path=path,
                            inputpath=input_path))
                    sys.exit(1)

        # Check for consistency in input paths due to cice having the same
        # information in multiple locations
        path, _ = os.path.split(self.ice_in['grid_nml'].get('kmt_file'))
        path = os.path.normpath(path)
        if path != input_path:
            print('payu: error: '
                  'kmt file path in {nmlfile} ({path}) does not match '
                  'input path ({inputpath})'.format(
                    nmlfile=self.ice_nml_fname,
                    path=path,
                    inputpath=input_path))
            sys.exit(1)

        if not os.path.isabs(input_path):
            input_path = os.path.join(self.work_path, input_path)
        if not os.path.isabs(init_path):
            init_path = os.path.join(self.work_path, init_path)
        self.work_input_path = input_path
        self.work_init_path = init_path

        if not os.path.isabs(res_path):
            res_path = os.path.join(self.work_path, res_path)
        self.work_restart_path = res_path

        work_out_path = os.path.normpath(setup_nml['history_dir'])

        if not os.path.isabs(work_out_path):
            work_out_path = os.path.join(self.work_path, work_out_path)
        self.work_output_path = work_out_path

        self.split_paths = (self.work_init_path != self.work_restart_path)

        if self.split_paths:
            self.copy_inputs = False
            self.copy_restarts = False

    def set_model_output_paths(self):
        super(Cice, self).set_model_output_paths()

        res_dir = self.ice_in['setup_nml']['restart_dir']

        # Use the local initialization restarts if present
        # TODO: Check for multiple res_paths across input paths?
        if self.expt.counter == 0:
            for input_path in self.input_paths:
                if os.path.isabs(res_dir):
                    init_res_path = res_dir
                else:
                    init_res_path = os.path.join(input_path, res_dir)
                if os.path.isdir(init_res_path):
                    self.prior_restart_path = init_res_path

    def get_ptr_restart_dir(self):
        return os.path.relpath(self.work_init_path, self.work_path)

    def setup(self):
        super(Cice, self).setup()

        # If there is a seperate ice_history.nml,
        # update the cice namelist with its contents
        history_nml_fpath = os.path.join(self.control_path,
                                         self.history_nml_fname)
        if os.path.isfile(history_nml_fpath):
            history_nml = f90nml.read(history_nml_fpath)
            self.ice_in.patch(history_nml)

        setup_nml = self.ice_in['setup_nml']
        self._calc_runtime()

        if self.prior_restart_path:
            self._make_restart_ptr()

            # Update input namelist
            setup_nml['runtype'] = 'continue'
            setup_nml['restart'] = True

        else:
            # Locate and link any restart files (if required)
            if not setup_nml['ice_ic'] in ('none', 'default'):
                self.link_restart(setup_nml['ice_ic'])

            if setup_nml['restart']:
                self.link_restart(setup_nml['pointer_file'])

        # Write any changes to the work directory copy of the cice
        # namelist
        nml_path = os.path.join(self.work_path, self.ice_nml_fname)
        self.ice_in.write(nml_path, force=True)

    def _calc_runtime(self):
        """
        Calculate 1: the previous number of timesteps simulated, and 2:
        the number of timesteps to simulate in the next run.
        Modifies the working self.ice_in namelist in place

        Note 1: This method is overridden in the cice5 driver.

        Note 2: For ESM1.5, the actual model start date and run time are
        controlled via the separate input_ice.nml namelist, with relevant
        calculations in the access driver.
        """
        setup_nml = self.ice_in['setup_nml']

        init_date = datetime.date(year=setup_nml['year_init'], month=1, day=1)
        if setup_nml['days_per_year'] == 365:
            caltype = cal.NOLEAP
        else:
            caltype = cal.GREGORIAN

        if self.prior_restart_path:

            prior_nml_path = os.path.join(self.prior_restart_path,
                                          self.ice_nml_fname)

            # With later versions this file exists in the prior restart path,
            # but this was not always the case, so check, and if not there use
            # prior output path
            if not os.path.exists(prior_nml_path) and self.prior_output_path:
                prior_nml_path = os.path.join(self.prior_output_path,
                                              self.ice_nml_fname)

            # If we cannot find a prior namelist, then we cannot determine
            # the start time and must abort the run.
            if not os.path.exists(prior_nml_path):
                print('payu: error: Cannot find prior namelist {nml}'.format(
                    nml=self.ice_nml_fname))
                sys.exit(errno.ENOENT)

            prior_setup_nml = f90nml.read(prior_nml_path)['setup_nml']

            # The total time in seconds since the beginning of the experiment
            prior_runtime = prior_setup_nml['istep0'] + prior_setup_nml['npt']
            prior_runtime_seconds = prior_runtime * prior_setup_nml['dt']

        else:
            # If no prior restart directory exists, set the prior runtime to 0
            prior_runtime_seconds = 0

        # Calculate runtime for this run.
        if self.expt.runtime:
            run_start_date = cal.date_plus_seconds(init_date,
                                                   prior_runtime_seconds,
                                                   caltype)
            run_runtime = cal.runtime_from_date(
                run_start_date,
                self.expt.runtime['years'],
                self.expt.runtime['months'],
                self.expt.runtime['days'],
                self.expt.runtime.get('seconds', 0),
                caltype
            )
        else:
            run_runtime = setup_nml['npt']*setup_nml['dt']

        # Add the prior runtime and new runtime to the working copy of the
        # CICE namelist.
        setup_nml['npt'] = run_runtime / setup_nml['dt']
        assert (prior_runtime_seconds % setup_nml['dt'] == 0)
        setup_nml['istep0'] = int(prior_runtime_seconds / setup_nml['dt'])

    def set_local_timestep(self, t_step):
        dt = self.ice_in['setup_nml']['dt']
        npt = self.ice_in['setup_nml']['npt']

        self.ice_in['setup_nml']['dt'] = t_step
        self.ice_in['setup_nml']['npt'] = (int(dt) * int(npt)) // int(t_step)

        ice_in_path = os.path.join(self.work_path, self.ice_nml_fname)
        self.ice_in.write(ice_in_path, force=True)

    def set_access_timestep(self, t_step):
        # TODO: Figure out some way to move this to the ACCESS driver
        # Re-read ice timestep and move this over there
        self.set_local_timestep(t_step)

        input_ice_path = os.path.join(self.work_path, 'input_ice.nml')
        input_ice = f90nml.read(input_ice_path)

        input_ice['coupling_nml']['dt_cice'] = t_step

        input_ice.write(input_ice_path, force=True)

    def set_oasis_timestep(self, t_step):
        # TODO: Move over to access driver
        for model in self.expt.models:
            if model.model_type == 'oasis':
                namcpl_path = os.path.join(model.work_path, 'namcouple')
                namcpl = Namcouple(namcpl_path, 'access')
                namcpl.set_ice_timestep(str(t_step))
                namcpl.write()

    def archive(self, **kwargs):
        super(Cice, self).archive()

        os.rename(self.work_restart_path, self.restart_path)

        if not self.split_paths:
            res_ptr_path = os.path.join(self.restart_path, 'ice.restart_file')
            with open(res_ptr_path) as f:
                res_name = os.path.basename(f.read()).strip()

            assert os.path.exists(os.path.join(self.restart_path, res_name))

            # Delete the old restart file (keep the one in ice.restart_file)
            for f in self.get_prior_restart_files():
                if f.startswith('iced.'):
                    if f == res_name:
                        continue
                    os.remove(os.path.join(self.restart_path, f))
        else:
            shutil.rmtree(self.work_input_path)

        archive_config = self.expt.config.get('archive', {})
        compressing_logs = archive_config.get('compress_logs', True)
        if compressing_logs:
            self.compress_log_files()

    def get_log_files(self):
        """
        Find model log files in the work directory based on regex patterns
        in self.logs_to_compress.

        Returns
        -------
        log_files: list of paths to model log files.
        """
        log_files = []
        for filename in os.listdir(self.work_path):
            if re.match("|".join(self.logs_to_compress), filename):
                log_files.append(os.path.join(self.work_path, filename))
        return log_files

    def compress_log_files(self):
        """
        Compress model log files into tarball.
        """
        log_files = self.get_log_files()
        with tarfile.open(name=os.path.join(self.work_path, self.log_tar_name),
                          mode="w:gz") as tar:
            for file in log_files:
                tar.add(file, arcname=os.path.basename(file))

        # Delete files after tarball is written
        for file in log_files:
            os.remove(file)

    def collate(self):
        pass

    def link_restart(self, fpath):

        input_work_path = os.path.join(self.work_path, fpath)

        # Exit if the restart file already exists
        if os.path.isfile(input_work_path):
            return

        input_path = None
        for i_path in self.input_paths:
            test_path = os.path.join(i_path, fpath)
            if os.path.isfile(test_path):
                input_path = test_path
                break
        if input_path is None:
            raise RuntimeError(
                f"Cannot find previous restart file, expected {fpath} to exist"
            )

        make_symlink(input_path, input_work_path)

    def _make_restart_ptr(self):
        """
        CICE4 restart pointers are created in the access driver, where
        the correct run start dates are available.
        """
        pass

    def overwrite_restart_ptr(self,
                              run_start_date,
                              previous_runtime,
                              calendar_file):
        """
        Generate restart pointer file 'ice.restart_file' pointing to
        'iced.YYYYMMDD' with the correct start date.=
        Additionally check that the `iced.YYYYMNDD` restart file's header
        has the correct previous runtime.
        Typically called from the access driver, which provides the
        the correct date and runtime.

        Parameters
        ----------
        run_start_date: datetime.date
            Start date of the new simulation
        previous_runtime:  int
            Seconds between experiment initialisation date and start date
        calendar_file:  str
            Calendar restart file used to calculate timing information
        """
        # Expected iced restart file name
        iced_restart_file = self.find_matching_iced(self.prior_restart_path,
                                                    run_start_date)

        res_ptr_path = os.path.join(self.work_init_path,
                                    'ice.restart_file')
        if os.path.islink(res_ptr_path):
            # If we've linked in a previous pointer it should be deleted
            os.remove(res_ptr_path)

        iced_path = os.path.join(self.prior_restart_path,
                                 iced_restart_file)

        # Check binary restart has correct time
        self._cice4_check_date_consistency(iced_path,
                                           previous_runtime,
                                           calendar_file)

        with open(res_ptr_path, 'w') as res_ptr:
            res_dir = self.get_ptr_restart_dir()
            res_ptr.write(os.path.join(res_dir, iced_restart_file))

    def _cice4_check_date_consistency(self,
                                      iced_path,
                                      previous_runtime,
                                      calendar_file):
        """
        Check that the previous runtime in iced restart file header
        matches the runtime calculated from the calendar restart file.

        Parameters
        ----------
        iced_path: str or Path
            Path to iced restart file
        previous_runtime:  int
            Seconds between experiment initialisation date and start date
        calendar_file:  str or Path
            Calendar restart file used to calculate timing information
        """
        _, _, cice_iced_runtime, _ = read_binary_iced_header(iced_path)
        if previous_runtime != cice_iced_runtime:
            msg = (f"Previous runtime from calendar file "
                   f"{calendar_file}: {previous_runtime} "
                   "does not match previous runtime in restart"
                   f"file {iced_path}: {cice_iced_runtime}.")
            raise RuntimeError(msg)

    def find_matching_iced(self, dir_path, date):
        """
        Check a directory for an iced.YYYYMMDD restart file matching a
        specified date. Typically called from access.py driver which
        provides the correct end date.
        Raises an error if the expected file is not found.

        Parameters
        ----------
        dir_path: str or Path
            Path to directory containing iced restart files
        date: datetime.date
            Date for matching iced file names

        Returns
        -------
        iced_file_name: str
            Name of iced restart file found in dir_path matching
            the specified date
        """
        # Expected iced restart file name
        date_int = cal.date_to_int(date)
        iced_restart_file = f"iced.{date_int:08d}"

        dir_files = [f for f in os.listdir(dir_path)
                     if os.path.isfile(os.path.join(dir_path, f))]

        if iced_restart_file not in dir_files:
            msg = (f"CICE restart file not found in {dir_path}. Expected "
                   f"{iced_restart_file} to exist. Is 'dumpfreq' set "
                   f"in {self.ice_nml_fname} consistently with the run-length?"
                   )
            raise FileNotFoundError(msg)

        return iced_restart_file


CICE4_RESTART_HEADER_SIZE = 24
CICE4_RESTART_HEADER_FORMAT = '>iidd'


def read_binary_iced_header(iced_path):
    """
    Read header information from a CICE4 binary restart file.
    """
    with open(iced_path, 'rb') as iced_file:
        header = iced_file.read(CICE4_RESTART_HEADER_SIZE)
        bint, istep0, time, time_forc = struct.unpack(
                                            CICE4_RESTART_HEADER_FORMAT,
                                            header)

    return (bint, istep0, time, time_forc)

1	"""
2	The payu interface for the CICE model
3	-------------------------------------------------------------------------------
4	Contact: Marshall Ward <marshall.ward@anu.edu.au>
5	-------------------------------------------------------------------------------
6	Distributed as part of Payu, Copyright 2011 Marshall Ward
7	Licensed under the Apache License, Version 2.0
8	http://www.apache.org/licenses/LICENSE-2.0
9	"""
10
11	# Python 3 preparation
12	from __future__ import print_function	3✔
13
14	# Standard Library
15	import errno	3✔
16	import os	3✔
17	import sys	3✔
18	import shutil	3✔
19	import datetime	3✔
20	import struct	3✔
21	import re	3✔
22	import tarfile	3✔
23
24	# Extensions
25	import f90nml	3✔
26
27	# Local
28	import payu.calendar as cal	3✔
29	from payu.fsops import make_symlink	3✔
30	from payu.models.model import Model	3✔
31	from payu.namcouple import Namcouple	3✔
32
33
34	class Cice(Model):	3✔
35
36	def __init__(self, expt, name, config):	3✔
37	super(Cice, self).__init__(expt, name, config)	3✔
38
39	self.model_type = 'cice'	3✔
40	self.default_exec = 'cice'	3✔
41
42	# Default repo details
43	self.repo_url = 'https://github.com/CWSL/cice4.git'	3✔
44	self.repo_tag = 'access'	3✔
45
46	self.config_files = ['cice_in.nml']	3✔
47	self.optional_config_files = ['input_ice.nml']	3✔
48
49	self.ice_nml_fname = 'cice_in.nml'	3✔
50
51	self.history_nml_fname = 'ice_history.nml' # only used by payu	3✔
52
53	self.set_timestep = self.set_local_timestep	3✔
54
55	self.copy_inputs = False	3✔
56
57	# regex patterns for matching log files. When empty, no logs compressed
58	self.logs_to_compress = [r"iceout[0-9]{3}",	3✔
59	r"debug\.root\.[0-9]{2}",
60	r"ice_diag\.d",
61	r"ice_diag_out"]
62	self.log_tar_name = "logfiles.tar.gz"	3✔
63
64	def set_model_pathnames(self):	3✔
65	super(Cice, self).set_model_pathnames()	3✔
66
67	self.build_exec_path = os.path.join(self.codebase_path,	3✔
68	'build_access-om_360x300_6p')
69
70	ice_nml_path = os.path.join(self.control_path, self.ice_nml_fname)	3✔
71	self.ice_in = f90nml.read(ice_nml_path)	3✔
72
73	# Assume local paths are relative to the work path
74	setup_nml = self.ice_in['setup_nml']	3✔
75
76	res_path = os.path.normpath(setup_nml['restart_dir'])	3✔
77	input_dir = setup_nml.get('input_dir', None)	3✔
78
79	if input_dir is None:	3✔
80	# Default to reading and writing inputs/restarts in-place
81	input_path = res_path	3✔
82	init_path = res_path	3✔
83	else:
84	input_path = os.path.normpath(input_dir)	×
85	init_path = input_path	×
86
87	# Determine if there is a work input path from the path to the
88	# grid.nc file. Older cice versions don't have a defined INPUT
89	# directory, but it is implied by this path
90	grid_nml = self.ice_in['grid_nml']	3✔
91	path, _ = os.path.split(grid_nml['grid_file'])	3✔
92	if path and not path == os.path.curdir:	3✔
93	assert not os.path.isabs(path)	3✔
94	path = os.path.normpath(path)	3✔
95	# Get input_dir from grid_file path unless otherwise specified
96	if input_dir is None:	3✔
97	input_path = path	3✔
98	else:
99	if path != input_path:	×
100	print('payu: error: Grid file path in {nmlfile} '	×
101	'({path}) does not match input path '
102	'({inputpath})'.format(
103	nmlfile=self.ice_nml_fname,
104	path=path,
105	inputpath=input_path))
106	sys.exit(1)	×
107
108	# Check for consistency in input paths due to cice having the same
109	# information in multiple locations
110	path, _ = os.path.split(self.ice_in['grid_nml'].get('kmt_file'))	3✔
111	path = os.path.normpath(path)	3✔
112	if path != input_path:	3✔
113	print('payu: error: '	×
114	'kmt file path in {nmlfile} ({path}) does not match '
115	'input path ({inputpath})'.format(
116	nmlfile=self.ice_nml_fname,
117	path=path,
118	inputpath=input_path))
119	sys.exit(1)	×
120
121	if not os.path.isabs(input_path):	3✔
122	input_path = os.path.join(self.work_path, input_path)	3✔
123	if not os.path.isabs(init_path):	3✔
124	init_path = os.path.join(self.work_path, init_path)	3✔
125	self.work_input_path = input_path	3✔
126	self.work_init_path = init_path	3✔
127
128	if not os.path.isabs(res_path):	3✔
129	res_path = os.path.join(self.work_path, res_path)	3✔
130	self.work_restart_path = res_path	3✔
131
132	work_out_path = os.path.normpath(setup_nml['history_dir'])	3✔
133
134	if not os.path.isabs(work_out_path):	3✔
135	work_out_path = os.path.join(self.work_path, work_out_path)	3✔
136	self.work_output_path = work_out_path	3✔
137
138	self.split_paths = (self.work_init_path != self.work_restart_path)	3✔
139
140	if self.split_paths:	3✔
141	self.copy_inputs = False	×
142	self.copy_restarts = False	×
143
144	def set_model_output_paths(self):	3✔
145	super(Cice, self).set_model_output_paths()	3✔
146
147	res_dir = self.ice_in['setup_nml']['restart_dir']	3✔
148
149	# Use the local initialization restarts if present
150	# TODO: Check for multiple res_paths across input paths?
151	if self.expt.counter == 0:	3✔
152	for input_path in self.input_paths:	3✔
153	if os.path.isabs(res_dir):	×
154	init_res_path = res_dir	×
155	else:
156	init_res_path = os.path.join(input_path, res_dir)	×
157	if os.path.isdir(init_res_path):	×
158	self.prior_restart_path = init_res_path	×
159
160	def get_ptr_restart_dir(self):	3✔
161	return os.path.relpath(self.work_init_path, self.work_path)	3✔
162
163	def setup(self):	3✔
164	super(Cice, self).setup()	3✔
165
166	# If there is a seperate ice_history.nml,
167	# update the cice namelist with its contents
168	history_nml_fpath = os.path.join(self.control_path,	3✔
169	self.history_nml_fname)
170	if os.path.isfile(history_nml_fpath):	3✔
171	history_nml = f90nml.read(history_nml_fpath)	3✔
172	self.ice_in.patch(history_nml)	3✔
173
174	setup_nml = self.ice_in['setup_nml']	3✔
175	self._calc_runtime()	3✔
176
177	if self.prior_restart_path:	3✔
178	self._make_restart_ptr()	3✔
179
180	# Update input namelist
181	setup_nml['runtype'] = 'continue'	3✔
182	setup_nml['restart'] = True	3✔
183
184	else:
185	# Locate and link any restart files (if required)
186	if not setup_nml['ice_ic'] in ('none', 'default'):	3✔
UNCOV 187	self.link_restart(setup_nml['ice_ic'])	×
188
189	if setup_nml['restart']:	3✔
190	self.link_restart(setup_nml['pointer_file'])	3✔
191
192	# Write any changes to the work directory copy of the cice
193	# namelist
194	nml_path = os.path.join(self.work_path, self.ice_nml_fname)	3✔
195	self.ice_in.write(nml_path, force=True)	3✔
196
197	def _calc_runtime(self):	3✔
198	"""
199	Calculate 1: the previous number of timesteps simulated, and 2:
200	the number of timesteps to simulate in the next run.
201	Modifies the working self.ice_in namelist in place
202
203	Note 1: This method is overridden in the cice5 driver.
204
205	Note 2: For ESM1.5, the actual model start date and run time are
206	controlled via the separate input_ice.nml namelist, with relevant
207	calculations in the access driver.
208	"""
209	setup_nml = self.ice_in['setup_nml']	3✔
210
211	init_date = datetime.date(year=setup_nml['year_init'], month=1, day=1)	3✔
212	if setup_nml['days_per_year'] == 365:	3✔
213	caltype = cal.NOLEAP	3✔
214	else:
UNCOV 215	caltype = cal.GREGORIAN	×
216
217	if self.prior_restart_path:	3✔
218
219	prior_nml_path = os.path.join(self.prior_restart_path,	3✔
220	self.ice_nml_fname)
221
222	# With later versions this file exists in the prior restart path,
223	# but this was not always the case, so check, and if not there use
224	# prior output path
225	if not os.path.exists(prior_nml_path) and self.prior_output_path:	3✔
UNCOV 226	prior_nml_path = os.path.join(self.prior_output_path,	×
227	self.ice_nml_fname)
228
229	# If we cannot find a prior namelist, then we cannot determine
230	# the start time and must abort the run.
231	if not os.path.exists(prior_nml_path):	3✔
UNCOV 232	print('payu: error: Cannot find prior namelist {nml}'.format(	×
233	nml=self.ice_nml_fname))
UNCOV 234	sys.exit(errno.ENOENT)	×
235
236	prior_setup_nml = f90nml.read(prior_nml_path)['setup_nml']	3✔
237
238	# The total time in seconds since the beginning of the experiment
239	prior_runtime = prior_setup_nml['istep0'] + prior_setup_nml['npt']	3✔
240	prior_runtime_seconds = prior_runtime * prior_setup_nml['dt']	3✔
241
242	else:
243	# If no prior restart directory exists, set the prior runtime to 0
244	prior_runtime_seconds = 0	3✔
245
246	# Calculate runtime for this run.
247	if self.expt.runtime:	3✔
248	run_start_date = cal.date_plus_seconds(init_date,	3✔
249	prior_runtime_seconds,
250	caltype)
251	run_runtime = cal.runtime_from_date(	3✔
252	run_start_date,
253	self.expt.runtime['years'],
254	self.expt.runtime['months'],
255	self.expt.runtime['days'],
256	self.expt.runtime.get('seconds', 0),
257	caltype
258	)
259	else:
260	run_runtime = setup_nml['npt']*setup_nml['dt']	3✔
261
262	# Add the prior runtime and new runtime to the working copy of the
263	# CICE namelist.
264	setup_nml['npt'] = run_runtime / setup_nml['dt']	3✔
265	assert (prior_runtime_seconds % setup_nml['dt'] == 0)	3✔
266	setup_nml['istep0'] = int(prior_runtime_seconds / setup_nml['dt'])	3✔
267
268	def set_local_timestep(self, t_step):	3✔
UNCOV 269	dt = self.ice_in['setup_nml']['dt']	×
270	npt = self.ice_in['setup_nml']['npt']	×
271
UNCOV 272	self.ice_in['setup_nml']['dt'] = t_step	×
273	self.ice_in['setup_nml']['npt'] = (int(dt) * int(npt)) // int(t_step)	×
274
UNCOV 275	ice_in_path = os.path.join(self.work_path, self.ice_nml_fname)	×
276	self.ice_in.write(ice_in_path, force=True)	×
277
278	def set_access_timestep(self, t_step):	3✔
279	# TODO: Figure out some way to move this to the ACCESS driver
280	# Re-read ice timestep and move this over there
UNCOV 281	self.set_local_timestep(t_step)	×
282
UNCOV 283	input_ice_path = os.path.join(self.work_path, 'input_ice.nml')	×
284	input_ice = f90nml.read(input_ice_path)	×
285
UNCOV 286	input_ice['coupling_nml']['dt_cice'] = t_step	×
287
UNCOV 288	input_ice.write(input_ice_path, force=True)	×
289
290	def set_oasis_timestep(self, t_step):	3✔
291	# TODO: Move over to access driver
UNCOV 292	for model in self.expt.models:	×
293	if model.model_type == 'oasis':	×
294	namcpl_path = os.path.join(model.work_path, 'namcouple')	×
295	namcpl = Namcouple(namcpl_path, 'access')	×
296	namcpl.set_ice_timestep(str(t_step))	×
297	namcpl.write()	×
298
299	def archive(self, **kwargs):	3✔
UNCOV 300	super(Cice, self).archive()	×
301
UNCOV 302	os.rename(self.work_restart_path, self.restart_path)	×
303
UNCOV 304	if not self.split_paths:	×
305	res_ptr_path = os.path.join(self.restart_path, 'ice.restart_file')	×
306	with open(res_ptr_path) as f:	×
307	res_name = os.path.basename(f.read()).strip()	×
308
UNCOV 309	assert os.path.exists(os.path.join(self.restart_path, res_name))	×
310
311	# Delete the old restart file (keep the one in ice.restart_file)
UNCOV 312	for f in self.get_prior_restart_files():	×
313	if f.startswith('iced.'):	×
314	if f == res_name:	×
315	continue	×
316	os.remove(os.path.join(self.restart_path, f))	×
317	else:
UNCOV 318	shutil.rmtree(self.work_input_path)	×
319
UNCOV 320	archive_config = self.expt.config.get('archive', {})	×
321	compressing_logs = archive_config.get('compress_logs', True)	×
322	if compressing_logs:	×
323	self.compress_log_files()	×
324
325	def get_log_files(self):	3✔
326	"""
327	Find model log files in the work directory based on regex patterns
328	in self.logs_to_compress.
329
330	Returns
331	-------
332	log_files: list of paths to model log files.
333	"""
334	log_files = []	3✔
335	for filename in os.listdir(self.work_path):	3✔
336	if re.match("\|".join(self.logs_to_compress), filename):	3✔
337	log_files.append(os.path.join(self.work_path, filename))	3✔
338	return log_files	3✔
339
340	def compress_log_files(self):	3✔
341	"""
342	Compress model log files into tarball.
343	"""
344	log_files = self.get_log_files()	3✔
345	with tarfile.open(name=os.path.join(self.work_path, self.log_tar_name),	3✔
346	mode="w:gz") as tar:
347	for file in log_files:	3✔
348	tar.add(file, arcname=os.path.basename(file))	3✔
349
350	# Delete files after tarball is written
351	for file in log_files:	3✔
352	os.remove(file)	3✔
353
354	def collate(self):	3✔
UNCOV 355	pass	×
356
357	def link_restart(self, fpath):	3✔
358
359	input_work_path = os.path.join(self.work_path, fpath)	3✔
360
361	# Exit if the restart file already exists
362	if os.path.isfile(input_work_path):	3✔
UNCOV 363	return	×
364
365	input_path = None	3✔
366	for i_path in self.input_paths:	3✔
UNCOV 367	test_path = os.path.join(i_path, fpath)	×
368	if os.path.isfile(test_path):	×
369	input_path = test_path	×
370	break	×
371	if input_path is None:	3✔
372	raise RuntimeError(	3✔
373	f"Cannot find previous restart file, expected {fpath} to exist"
374	)
375
UNCOV 376	make_symlink(input_path, input_work_path)	×
377
378	def _make_restart_ptr(self):	3✔
379	"""
380	CICE4 restart pointers are created in the access driver, where
381	the correct run start dates are available.
382	"""
383	pass	3✔
384
385	def overwrite_restart_ptr(self,	3✔
386	run_start_date,
387	previous_runtime,
388	calendar_file):
389	"""
390	Generate restart pointer file 'ice.restart_file' pointing to
391	'iced.YYYYMMDD' with the correct start date.=
392	Additionally check that the `iced.YYYYMNDD` restart file's header
393	has the correct previous runtime.
394	Typically called from the access driver, which provides the
395	the correct date and runtime.
396
397	Parameters
398	----------
399	run_start_date: datetime.date
400	Start date of the new simulation
401	previous_runtime: int
402	Seconds between experiment initialisation date and start date
403	calendar_file: str
404	Calendar restart file used to calculate timing information
405	"""
406	# Expected iced restart file name
407	iced_restart_file = self.find_matching_iced(self.prior_restart_path,	3✔
408	run_start_date)
409
410	res_ptr_path = os.path.join(self.work_init_path,	3✔
411	'ice.restart_file')
412	if os.path.islink(res_ptr_path):	3✔
413	# If we've linked in a previous pointer it should be deleted
NEW 414	os.remove(res_ptr_path)	×
415
416	iced_path = os.path.join(self.prior_restart_path,	3✔
417	iced_restart_file)
418
419	# Check binary restart has correct time
420	self._cice4_check_date_consistency(iced_path,	3✔
421	previous_runtime,
422	calendar_file)
423
424	with open(res_ptr_path, 'w') as res_ptr:	3✔
425	res_dir = self.get_ptr_restart_dir()	3✔
426	res_ptr.write(os.path.join(res_dir, iced_restart_file))	3✔
427
428	def _cice4_check_date_consistency(self,	3✔
429	iced_path,
430	previous_runtime,
431	calendar_file):
432	"""
433	Check that the previous runtime in iced restart file header
434	matches the runtime calculated from the calendar restart file.
435
436	Parameters
437	----------
438	iced_path: str or Path
439	Path to iced restart file
440	previous_runtime: int
441	Seconds between experiment initialisation date and start date
442	calendar_file: str or Path
443	Calendar restart file used to calculate timing information
444	"""
445	_, _, cice_iced_runtime, _ = read_binary_iced_header(iced_path)	3✔
446	if previous_runtime != cice_iced_runtime:	3✔
447	msg = (f"Previous runtime from calendar file "	3✔
448	f"{calendar_file}: {previous_runtime} "
449	"does not match previous runtime in restart"
450	f"file {iced_path}: {cice_iced_runtime}.")
451	raise RuntimeError(msg)	3✔
452
453	def find_matching_iced(self, dir_path, date):	3✔
454	"""
455	Check a directory for an iced.YYYYMMDD restart file matching a
456	specified date. Typically called from access.py driver which
457	provides the correct end date.
458	Raises an error if the expected file is not found.
459
460	Parameters
461	----------
462	dir_path: str or Path
463	Path to directory containing iced restart files
464	date: datetime.date
465	Date for matching iced file names
466
467	Returns
468	-------
469	iced_file_name: str
470	Name of iced restart file found in dir_path matching
471	the specified date
472	"""
473	# Expected iced restart file name
474	date_int = cal.date_to_int(date)	3✔
475	iced_restart_file = f"iced.{date_int:08d}"	3✔
476
477	dir_files = [f for f in os.listdir(dir_path)	3✔
478	if os.path.isfile(os.path.join(dir_path, f))]
479
480	if iced_restart_file not in dir_files:	3✔
481	msg = (f"CICE restart file not found in {dir_path}. Expected "	3✔
482	f"{iced_restart_file} to exist. Is 'dumpfreq' set "
483	f"in {self.ice_nml_fname} consistently with the run-length?"
484	)
485	raise FileNotFoundError(msg)	3✔
486
487	return iced_restart_file	3✔
488
489
490	CICE4_RESTART_HEADER_SIZE = 24	3✔
491	CICE4_RESTART_HEADER_FORMAT = '>iidd'	3✔
492
493
494	def read_binary_iced_header(iced_path):	3✔
495	"""
496	Read header information from a CICE4 binary restart file.
497	"""
498	with open(iced_path, 'rb') as iced_file:	3✔
499	header = iced_file.read(CICE4_RESTART_HEADER_SIZE)	3✔
500	bint, istep0, time, time_forc = struct.unpack(	3✔
501	CICE4_RESTART_HEADER_FORMAT,
502	header)
503
504	return (bint, istep0, time, time_forc)	3✔

payu-org / payu / 12288631034

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous