6489602581

Committed 12 Oct 2023 12:25AM UTC coverage: 45.573% (+2.8%) from 42.772%

Build # 6489602581

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #363 from jo-basevi/358-date-based-frequency

Add support for date-based restart frequency

Run Details

111 of 147 new or added lines in 10 files covered. (75.51%)

2 existing lines in 1 file now uncovered.

1580 of 3467 relevant lines covered (45.57%)

1.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.68

/payu/models/model.py

"""Generic model interface, primarily to be inherited by other models.

:copyright: Copyright 2011 Marshall Ward, see AUTHORS for details
:license: Apache License, Version 2.0, see LICENSE for details
"""
import errno
import os
import shutil
import shlex
import sys
import subprocess as sp

from payu import envmod
from payu.fsops import mkdir_p, required_libs


class Model(object):
    """Abstract model class."""

    def __init__(self, expt, model_name, model_config):
        """Create the model interface."""
        # Inherit experiment configuration
        self.expt = expt
        self.name = model_name
        self.config = model_config
        self.top_level_model = False

        # Model details
        self.model_type = None
        self.default_exec = None
        self.input_basepath = None
        self.modules = []
        self.config_files = []
        self.optional_config_files = []

        # Path names
        self.work_input_path = None
        self.work_restart_path = None
        self.work_init_path = None
        # A string to add before the exe name, useful for e.g. gdb, gprof,
        # valgrind
        self.exec_prefix = None
        self.exec_path = None
        self.exec_name = None
        self.codebase_path = None
        self.work_path_local = None
        self.work_input_path_local = None
        self.work_restart_path_local = None
        self.work_init_path_local = None
        self.exec_path_local = None

        self.build_exec_path = None
        self.build_path = None

        self.required_libs = None

        # Control flags
        self.copy_restarts = False
        self.copy_inputs = False

        # Codebase details
        self.repo_url = None
        self.repo_tag = None
        self.build_command = None

    def set_model_pathnames(self):
        """Define the paths associated with this model."""
        self.control_path = self.expt.control_path
        self.input_basepath = self.expt.lab.input_basepath
        self.work_path = self.expt.work_path
        self.codebase_path = self.expt.lab.codebase_path

        if len(self.expt.models) > 1:

            self.control_path = os.path.join(self.control_path, self.name)
            self.work_path = os.path.join(self.work_path, self.name)
            self.codebase_path = os.path.join(self.codebase_path, self.name)

        # NOTE: Individual models may override the work subdirectories
        self.work_input_path = self.work_path
        self.work_restart_path = self.work_path
        self.work_output_path = self.work_path
        self.work_init_path = self.work_path

        self.exec_prefix = self.config.get('exe_prefix', '')
        self.exec_name = self.config.get('exe', self.default_exec)
        if self.exec_name:
            # By default os.path.join will not prepend the lab bin_path
            # to an absolute path
            self.exec_path = os.path.join(self.expt.lab.bin_path,
                                          self.exec_name)
        else:
            self.exec_path = None

        if self.exec_path:
            # Make exec_name consistent for models with fully qualified path.
            # In all cases it will just be the name of the executable without a
            # path
            self.exec_name = os.path.basename(self.exec_path)

    def set_local_pathnames(self):

        # This is the path relative to the control directory, required for
        # manifests and must be called after set_model_pathnames to ensure it
        # captures changes made in model subclasses which override
        # set_model_pathnames

        # XXX: If path is relative to control_path, why not use it?
        self.work_path_local = os.path.normpath(
            os.path.join(
                'work',
                os.path.relpath(self.work_path, self.expt.work_path)
            )
        )
        self.work_input_path_local = os.path.normpath(
            os.path.join(
                'work',
                os.path.relpath(self.work_input_path, self.expt.work_path)
            )
        )
        self.work_restart_path_local = os.path.normpath(
            os.path.join(
                'work',
                os.path.relpath(self.work_restart_path, self.expt.work_path)
            )
        )
        self.work_init_path_local = os.path.normpath(
            os.path.join(
                'work',
                os.path.relpath(self.work_init_path, self.expt.work_path)
            )
        )
        if self.exec_path:
            # Local path in work directory
            self.exec_path_local = os.path.join(
                self.work_path_local,
                os.path.basename(self.exec_path)
            )

    def set_input_paths(self):
        if len(self.expt.models) == 1:
            input_dirs = self.expt.config.get('input')
        else:
            input_dirs = self.config.get('input')

        if input_dirs is None:
            input_dirs = []
        elif isinstance(input_dirs, str):
            input_dirs = [input_dirs]

        self.input_paths = []
        for input_dir in input_dirs:

            # First test for absolute path
            if os.path.exists(input_dir):
                self.input_paths.append(input_dir)
            else:
                # Test for path relative to /${lab_path}/input/${model_name}
                assert self.input_basepath
                rel_path = os.path.join(self.input_basepath, input_dir)
                if os.path.exists(rel_path):
                    self.input_paths.append(rel_path)
                else:
                    sys.exit('payu: error: Input directory {0} not found; '
                             'aborting.'.format(rel_path))

    def set_model_output_paths(self):

        self.output_path = self.expt.output_path
        self.restart_path = self.expt.restart_path

        self.prior_output_path = self.expt.prior_output_path
        self.prior_restart_path = self.expt.prior_restart_path

        if len(self.expt.models) > 1:

            # If '-d' option specified for collate don't want to change the
            # output path, but respect the absolute path specified
            if os.environ.get('PAYU_DIR_PATH') is None:
                self.output_path = os.path.join(self.output_path, self.name)

            self.restart_path = os.path.join(self.restart_path, self.name)

            if self.prior_output_path:
                self.prior_output_path = os.path.join(self.prior_output_path,
                                                      self.name)

            if self.prior_restart_path:
                self.prior_restart_path = os.path.join(self.prior_restart_path,
                                                       self.name)

    def get_prior_restart_files(self):

        try:
            respath = self.prior_restart_path
            return [f for f in os.listdir(respath)
                    if os.path.isfile(os.path.join(respath, f))]
        except Exception as e:
            print("No prior restart files found: {error}".format(error=str(e)))
            return []

    def setup(self):

        print("Setting up {model}".format(model=self.name))
        # Create experiment directory structure
        mkdir_p(self.work_init_path)
        mkdir_p(self.work_input_path)
        mkdir_p(self.work_restart_path)
        mkdir_p(self.work_output_path)

        # Copy configuration files from control path
        for f_name in self.config_files:
            f_path = os.path.join(self.control_path, f_name)
            shutil.copy(f_path, self.work_path)

        for f_name in self.optional_config_files:
            f_path = os.path.join(self.control_path, f_name)
            try:
                shutil.copy(f_path, self.work_path)
            except IOError as exc:
                if exc.errno == errno.ENOENT:
                    pass
                else:
                    raise

        # Add restart files from prior run to restart manifest
        if (not self.expt.manifest.have_manifest['restart'] and
                self.prior_restart_path):
            restart_files = self.get_prior_restart_files()
            for f_name in restart_files:
                f_orig = os.path.join(self.prior_restart_path, f_name)
                f_link = os.path.join(self.work_init_path_local, f_name)
                self.expt.manifest.add_filepath(
                    'restart',
                    f_link,
                    f_orig,
                    self.copy_restarts
                )

        # Add input files to manifest if we don't already have a populated
        # input manifest, or we specify scaninputs is True (default)
        if (not self.expt.manifest.have_manifest['input'] or
                self.expt.manifest.scaninputs):
            for input_path in self.input_paths:
                if os.path.isfile(input_path):
                    # Build a mock walk iterator for a single file
                    fwalk = iter([(
                        os.path.dirname(input_path),
                        [],
                        [os.path.basename(input_path)]
                    )])
                    # Overwrite the input_path as a directory
                    input_path = os.path.dirname(input_path)
                else:
                    fwalk = os.walk(input_path)

                for path, dirs, files in fwalk:
                    workrelpath = os.path.relpath(path, input_path)
                    subdir = os.path.normpath(
                        os.path.join(self.work_input_path_local,
                                     workrelpath)
                    )

                    if not os.path.exists(subdir):
                        os.mkdir(subdir)

                    for f_name in files:
                        f_orig = os.path.join(path, f_name)
                        f_link = os.path.join(
                            self.work_input_path_local,
                            workrelpath,
                            f_name
                        )
                        # Do not use input file if already linked
                        # as a restart file
                        if not os.path.exists(f_link):
                            self.expt.manifest.add_filepath(
                                'input',
                                f_link,
                                f_orig,
                                self.copy_inputs
                            )

        # Make symlink to executable in work directory
        if self.exec_path:
            # Check whether executable path exists
            if not os.path.isfile(self.exec_path):
                raise FileNotFoundError(
                    f'Executable for {self.name} model '
                    f'not found on path: {self.exec_path}')

            # Check whether executable has executable permission
            if not os.access(self.exec_path, os.X_OK):
                raise PermissionError(
                    f'Executable for {self.name} model '
                    f'is not executable: {self.exec_path}')

            # If have exe manifest this implies exe reproduce is True. Do not
            # want to overwrite exe manifest in this case
            if not self.expt.manifest.have_manifest['exe']:
                # Add to exe manifest (this is always done so any change in exe
                # path will be picked up)
                self.expt.manifest.add_filepath(
                    'exe',
                    self.exec_path_local,
                    self.exec_path
                )

            # Populate information about required dynamically loaded libraries
            self.required_libs = required_libs(self.exec_path)

        timestep = self.config.get('timestep')
        if timestep:
            self.set_timestep(timestep)

    def set_timestep(self, timestep):
        """Set the model timestep."""
        raise NotImplementedError

    def archive(self):
        """Store model output to laboratory archive."""

        # Traverse the model directory deleting symlinks, zero length files
        # and empty directories
        for path, dirs, files in os.walk(self.work_path, topdown=False):
            for f_name in files:
                f_path = os.path.join(path, f_name)
                if os.path.islink(f_path) or os.path.getsize(f_path) == 0:
                    os.remove(f_path)
            if len(os.listdir(path)) == 0:
                os.rmdir(path)

    def collate(self):
        """Collate any tiled output into a single file."""
        raise NotImplementedError

    def build_model(self):

        if not self.repo_url:
            return

        # Check to see if executable already exists.
        if self.exec_path and os.path.isfile(self.exec_path):
            print('payu: warning: {0} will be overwritten.'
                  ''.format(self.exec_path))

        # First step is always to go to the codebase.
        curdir = os.getcwd()

        # Do the build. First check whether there is a build command in the
        # config. If not check for the model default, otherwise just run make.

        try:
            build_path = self.config['build']['path_to_build_command']
        except KeyError:
            if self.build_path:
                build_path = self.build_path
            else:
                build_path = './'

        os.chdir(os.path.join(self.codebase_path, build_path))

        try:
            cmd = self.config['build']['command']
        except KeyError:
            if self.build_command:
                cmd = self.build_command
            else:
                cmd = 'make'

        print('Running command {0}'.format(cmd))
        sp.check_call(shlex.split(cmd))

        try:
            build_exec_path = os.path.join(self.codebase_path,
                                           self.config['build']['exec_path'])
        except KeyError:
            if self.build_exec_path:
                build_exec_path = self.build_exec_path
            else:
                build_exec_path = self.codebase_path

        # Copy new executable to bin dir
        if self.exec_path:
            # Create the bin path if it doesn't exist
            mkdir_p(self.expt.lab.bin_path)

            build_exec_path = os.path.join(build_exec_path, self.exec_name)
            shutil.copy(build_exec_path, self.exec_path)

        os.chdir(curdir)

    def get_codebase(self):

        if not self.repo_url:
            return

        # Disable the user's .gitconfig file
        os.environ['GIT_CONFIG_NOGLOBAL'] = 'yes'

        build_config = self.config.get('build', {})
        self.repo_url = build_config.get('repository', self.repo_url)
        self.repo_tag = build_config.get('tag', self.repo_tag)

        git_path = os.path.join(self.codebase_path, '.git')
        if not os.path.exists(git_path):
            cmd = 'git clone {0} {1}'.format(self.repo_url, self.codebase_path)
            sp.check_call(shlex.split(cmd))

        curdir = os.getcwd()
        os.chdir(self.codebase_path)
        sp.check_call(shlex.split('git checkout {0}'.format(self.repo_tag)))
        sp.check_call(shlex.split('git pull'))
        os.chdir(curdir)

    def profile(self):
        # TODO: Replace with call to "profile" drivers

        if self.expt.config.get('hpctoolkit', False) and self.exec_name:

            envmod.module('load', 'hpctoolkit')

            # Create the code structure file
            hpcstruct_fname = '{0}.hpcstruct'.format(self.exec_name)
            hpcstruct_path = os.path.join(self.expt.lab.bin_path,
                                          hpcstruct_fname)

            # TODO: Validate struct file
            if not os.path.isfile(hpcstruct_path):
                cmd = 'hpcstruct -o {0} {1}'.format(hpcstruct_path,
                                                    self.exec_path)
                sp.check_call(shlex.split(cmd))

            # Parse the profile output
            hpctk_header = 'hpctoolkit-{0}-measurements'.format(self.exec_name)
            hpctk_measure_dir = [os.path.join(self.output_path, f)
                                 for f in os.listdir(self.output_path)
                                 if f.startswith(hpctk_header)][0]

            hpctk_db_dir = hpctk_measure_dir.replace('measurements',
                                                     'database')

            # TODO: This needs to be model-specifc
            src_path = os.path.join(self.codebase_path, 'src')

            cmd = 'hpcprof-mpi -S {0} -I {1} -o {2} {3}'.format(
                hpcstruct_path, src_path, hpctk_db_dir, hpctk_measure_dir)
            sp.check_call(shlex.split(cmd))

        if self.expt.config.get('scalasca', False):

            envmod.module('use', '/home/900/mpc900/my_modules')
            envmod.module('load', 'scalasca')

            scorep_path = [os.path.join(self.output_path, f)
                           for f in os.listdir(self.output_path)
                           if f.startswith('scorep')][0]
            cmd = 'scalasca -examine -s {0}'.format(scorep_path)
            sp.check_call(shlex.split(cmd))

        if self.expt.config.get('scorep', False):

            envmod.module('load', 'scorep')

            scorep_path = [os.path.join(self.output_path, f)
                           for f in os.listdir(self.output_path)
                           if f.startswith('scorep')][0]
            cube_path = [os.path.join(scorep_path, f)
                         for f in os.listdir(scorep_path)
                         if f.endswith('.cubex')][0]
            cmd = 'scorep-score {0}'.format(cube_path)
            sp.check_call(shlex.split(cmd))

    def get_restart_datetime(self, restart_path):
        """Given a restart path, parse the restart files and return a cftime
        datetime (currently used for date-based restart pruning)"""
        raise NotImplementedError

1	"""Generic model interface, primarily to be inherited by other models.
2
3	:copyright: Copyright 2011 Marshall Ward, see AUTHORS for details
4	:license: Apache License, Version 2.0, see LICENSE for details
5	"""
6	import errno	3✔
7	import os	3✔
8	import shutil	3✔
9	import shlex	3✔
10	import sys	3✔
11	import subprocess as sp	3✔
12
13	from payu import envmod	3✔
14	from payu.fsops import mkdir_p, required_libs	3✔
15
16
17	class Model(object):	3✔
18	"""Abstract model class."""
19
20	def __init__(self, expt, model_name, model_config):	3✔
21	"""Create the model interface."""
22	# Inherit experiment configuration
23	self.expt = expt	3✔
24	self.name = model_name	3✔
25	self.config = model_config	3✔
26	self.top_level_model = False	3✔
27
28	# Model details
29	self.model_type = None	3✔
30	self.default_exec = None	3✔
31	self.input_basepath = None	3✔
32	self.modules = []	3✔
33	self.config_files = []	3✔
34	self.optional_config_files = []	3✔
35
36	# Path names
37	self.work_input_path = None	3✔
38	self.work_restart_path = None	3✔
39	self.work_init_path = None	3✔
40	# A string to add before the exe name, useful for e.g. gdb, gprof,
41	# valgrind
42	self.exec_prefix = None	3✔
43	self.exec_path = None	3✔
44	self.exec_name = None	3✔
45	self.codebase_path = None	3✔
46	self.work_path_local = None	3✔
47	self.work_input_path_local = None	3✔
48	self.work_restart_path_local = None	3✔
49	self.work_init_path_local = None	3✔
50	self.exec_path_local = None	3✔
51
52	self.build_exec_path = None	3✔
53	self.build_path = None	3✔
54
55	self.required_libs = None	3✔
56
57	# Control flags
58	self.copy_restarts = False	3✔
59	self.copy_inputs = False	3✔
60
61	# Codebase details
62	self.repo_url = None	3✔
63	self.repo_tag = None	3✔
64	self.build_command = None	3✔
65
66	def set_model_pathnames(self):	3✔
67	"""Define the paths associated with this model."""
68	self.control_path = self.expt.control_path	3✔
69	self.input_basepath = self.expt.lab.input_basepath	3✔
70	self.work_path = self.expt.work_path	3✔
71	self.codebase_path = self.expt.lab.codebase_path	3✔
72
73	if len(self.expt.models) > 1:	3✔
74
75	self.control_path = os.path.join(self.control_path, self.name)	3✔
76	self.work_path = os.path.join(self.work_path, self.name)	3✔
77	self.codebase_path = os.path.join(self.codebase_path, self.name)	3✔
78
79	# NOTE: Individual models may override the work subdirectories
80	self.work_input_path = self.work_path	3✔
81	self.work_restart_path = self.work_path	3✔
82	self.work_output_path = self.work_path	3✔
83	self.work_init_path = self.work_path	3✔
84
85	self.exec_prefix = self.config.get('exe_prefix', '')	3✔
86	self.exec_name = self.config.get('exe', self.default_exec)	3✔
87	if self.exec_name:	3✔
88	# By default os.path.join will not prepend the lab bin_path
89	# to an absolute path
90	self.exec_path = os.path.join(self.expt.lab.bin_path,	3✔
91	self.exec_name)
92	else:
93	self.exec_path = None	3✔
94
95	if self.exec_path:	3✔
96	# Make exec_name consistent for models with fully qualified path.
97	# In all cases it will just be the name of the executable without a
98	# path
99	self.exec_name = os.path.basename(self.exec_path)	3✔
100
101	def set_local_pathnames(self):	3✔
102
103	# This is the path relative to the control directory, required for
104	# manifests and must be called after set_model_pathnames to ensure it
105	# captures changes made in model subclasses which override
106	# set_model_pathnames
107
108	# XXX: If path is relative to control_path, why not use it?
109	self.work_path_local = os.path.normpath(	3✔
110	os.path.join(
111	'work',
112	os.path.relpath(self.work_path, self.expt.work_path)
113	)
114	)
115	self.work_input_path_local = os.path.normpath(	3✔
116	os.path.join(
117	'work',
118	os.path.relpath(self.work_input_path, self.expt.work_path)
119	)
120	)
121	self.work_restart_path_local = os.path.normpath(	3✔
122	os.path.join(
123	'work',
124	os.path.relpath(self.work_restart_path, self.expt.work_path)
125	)
126	)
127	self.work_init_path_local = os.path.normpath(	3✔
128	os.path.join(
129	'work',
130	os.path.relpath(self.work_init_path, self.expt.work_path)
131	)
132	)
133	if self.exec_path:	3✔
134	# Local path in work directory
135	self.exec_path_local = os.path.join(	3✔
136	self.work_path_local,
137	os.path.basename(self.exec_path)
138	)
139
140	def set_input_paths(self):	3✔
141	if len(self.expt.models) == 1:	3✔
142	input_dirs = self.expt.config.get('input')	3✔
143	else:
144	input_dirs = self.config.get('input')	3✔
145
146	if input_dirs is None:	3✔
147	input_dirs = []	3✔
148	elif isinstance(input_dirs, str):	3✔
149	input_dirs = [input_dirs]	3✔
150
151	self.input_paths = []	3✔
152	for input_dir in input_dirs:	3✔
153
154	# First test for absolute path
155	if os.path.exists(input_dir):	3✔
156	self.input_paths.append(input_dir)	×
157	else:
158	# Test for path relative to /${lab_path}/input/${model_name}
159	assert self.input_basepath	3✔
160	rel_path = os.path.join(self.input_basepath, input_dir)	3✔
161	if os.path.exists(rel_path):	3✔
162	self.input_paths.append(rel_path)	3✔
163	else:
164	sys.exit('payu: error: Input directory {0} not found; '	×
165	'aborting.'.format(rel_path))
166
167	def set_model_output_paths(self):	3✔
168
169	self.output_path = self.expt.output_path	3✔
170	self.restart_path = self.expt.restart_path	3✔
171
172	self.prior_output_path = self.expt.prior_output_path	3✔
173	self.prior_restart_path = self.expt.prior_restart_path	3✔
174
175	if len(self.expt.models) > 1:	3✔
176
177	# If '-d' option specified for collate don't want to change the
178	# output path, but respect the absolute path specified
179	if os.environ.get('PAYU_DIR_PATH') is None:	3✔
180	self.output_path = os.path.join(self.output_path, self.name)	3✔
181
182	self.restart_path = os.path.join(self.restart_path, self.name)	3✔
183
184	if self.prior_output_path:	3✔
185	self.prior_output_path = os.path.join(self.prior_output_path,	×
186	self.name)
187
188	if self.prior_restart_path:	3✔
189	self.prior_restart_path = os.path.join(self.prior_restart_path,	3✔
190	self.name)
191
192	def get_prior_restart_files(self):	3✔
193
194	try:	3✔
195	respath = self.prior_restart_path	3✔
196	return [f for f in os.listdir(respath)	3✔
197	if os.path.isfile(os.path.join(respath, f))]
198	except Exception as e:	×
199	print("No prior restart files found: {error}".format(error=str(e)))	×
200	return []	×
201
202	def setup(self):	3✔
203
204	print("Setting up {model}".format(model=self.name))	3✔
205	# Create experiment directory structure
206	mkdir_p(self.work_init_path)	3✔
207	mkdir_p(self.work_input_path)	3✔
208	mkdir_p(self.work_restart_path)	3✔
209	mkdir_p(self.work_output_path)	3✔
210
211	# Copy configuration files from control path
212	for f_name in self.config_files:	3✔
213	f_path = os.path.join(self.control_path, f_name)	3✔
214	shutil.copy(f_path, self.work_path)	3✔
215
216	for f_name in self.optional_config_files:	3✔
217	f_path = os.path.join(self.control_path, f_name)	×
218	try:	×
219	shutil.copy(f_path, self.work_path)	×
220	except IOError as exc:	×
221	if exc.errno == errno.ENOENT:	×
222	pass	×
223	else:
224	raise	×
225
226	# Add restart files from prior run to restart manifest
227	if (not self.expt.manifest.have_manifest['restart'] and	3✔
228	self.prior_restart_path):
229	restart_files = self.get_prior_restart_files()	3✔
230	for f_name in restart_files:	3✔
231	f_orig = os.path.join(self.prior_restart_path, f_name)	3✔
232	f_link = os.path.join(self.work_init_path_local, f_name)	3✔
233	self.expt.manifest.add_filepath(	3✔
234	'restart',
235	f_link,
236	f_orig,
237	self.copy_restarts
238	)
239
240	# Add input files to manifest if we don't already have a populated
241	# input manifest, or we specify scaninputs is True (default)
242	if (not self.expt.manifest.have_manifest['input'] or	3✔
243	self.expt.manifest.scaninputs):
244	for input_path in self.input_paths:	3✔
245	if os.path.isfile(input_path):	3✔
246	# Build a mock walk iterator for a single file
247	fwalk = iter([(	×
248	os.path.dirname(input_path),
249	[],
250	[os.path.basename(input_path)]
251	)])
252	# Overwrite the input_path as a directory
253	input_path = os.path.dirname(input_path)	×
254	else:
255	fwalk = os.walk(input_path)	3✔
256
257	for path, dirs, files in fwalk:	3✔
258	workrelpath = os.path.relpath(path, input_path)	3✔
259	subdir = os.path.normpath(	3✔
260	os.path.join(self.work_input_path_local,
261	workrelpath)
262	)
263
264	if not os.path.exists(subdir):	3✔
265	os.mkdir(subdir)	×
266
267	for f_name in files:	3✔
268	f_orig = os.path.join(path, f_name)	3✔
269	f_link = os.path.join(	3✔
270	self.work_input_path_local,
271	workrelpath,
272	f_name
273	)
274	# Do not use input file if already linked
275	# as a restart file
276	if not os.path.exists(f_link):	3✔
277	self.expt.manifest.add_filepath(	3✔
278	'input',
279	f_link,
280	f_orig,
281	self.copy_inputs
282	)
283
284	# Make symlink to executable in work directory
285	if self.exec_path:	3✔
286	# Check whether executable path exists
287	if not os.path.isfile(self.exec_path):	3✔
288	raise FileNotFoundError(	×
289	f'Executable for {self.name} model '
290	f'not found on path: {self.exec_path}')
291
292	# Check whether executable has executable permission
293	if not os.access(self.exec_path, os.X_OK):	3✔
294	raise PermissionError(	×
295	f'Executable for {self.name} model '
296	f'is not executable: {self.exec_path}')
297
298	# If have exe manifest this implies exe reproduce is True. Do not
299	# want to overwrite exe manifest in this case
300	if not self.expt.manifest.have_manifest['exe']:	3✔
301	# Add to exe manifest (this is always done so any change in exe
302	# path will be picked up)
303	self.expt.manifest.add_filepath(	3✔
304	'exe',
305	self.exec_path_local,
306	self.exec_path
307	)
308
309	# Populate information about required dynamically loaded libraries
310	self.required_libs = required_libs(self.exec_path)	3✔
311
312	timestep = self.config.get('timestep')	3✔
313	if timestep:	3✔
314	self.set_timestep(timestep)	×
315
316	def set_timestep(self, timestep):	3✔
317	"""Set the model timestep."""
318	raise NotImplementedError	×
319
320	def archive(self):	3✔
321	"""Store model output to laboratory archive."""
322
323	# Traverse the model directory deleting symlinks, zero length files
324	# and empty directories
325	for path, dirs, files in os.walk(self.work_path, topdown=False):	×
326	for f_name in files:	×
327	f_path = os.path.join(path, f_name)	×
328	if os.path.islink(f_path) or os.path.getsize(f_path) == 0:	×
329	os.remove(f_path)	×
330	if len(os.listdir(path)) == 0:	×
331	os.rmdir(path)	×
332
333	def collate(self):	3✔
334	"""Collate any tiled output into a single file."""
335	raise NotImplementedError	×
336
337	def build_model(self):	3✔
338
339	if not self.repo_url:	×
340	return	×
341
342	# Check to see if executable already exists.
343	if self.exec_path and os.path.isfile(self.exec_path):	×
344	print('payu: warning: {0} will be overwritten.'	×
345	''.format(self.exec_path))
346
347	# First step is always to go to the codebase.
348	curdir = os.getcwd()	×
349
350	# Do the build. First check whether there is a build command in the
351	# config. If not check for the model default, otherwise just run make.
352
353	try:	×
354	build_path = self.config['build']['path_to_build_command']	×
355	except KeyError:	×
356	if self.build_path:	×
357	build_path = self.build_path	×
358	else:
359	build_path = './'	×
360
361	os.chdir(os.path.join(self.codebase_path, build_path))	×
362
363	try:	×
364	cmd = self.config['build']['command']	×
365	except KeyError:	×
366	if self.build_command:	×
367	cmd = self.build_command	×
368	else:
369	cmd = 'make'	×
370
371	print('Running command {0}'.format(cmd))	×
372	sp.check_call(shlex.split(cmd))	×
373
374	try:	×
375	build_exec_path = os.path.join(self.codebase_path,	×
376	self.config['build']['exec_path'])
377	except KeyError:	×
378	if self.build_exec_path:	×
379	build_exec_path = self.build_exec_path	×
380	else:
381	build_exec_path = self.codebase_path	×
382
383	# Copy new executable to bin dir
384	if self.exec_path:	×
385	# Create the bin path if it doesn't exist
386	mkdir_p(self.expt.lab.bin_path)	×
387
388	build_exec_path = os.path.join(build_exec_path, self.exec_name)	×
389	shutil.copy(build_exec_path, self.exec_path)	×
390
391	os.chdir(curdir)	×
392
393	def get_codebase(self):	3✔
394
395	if not self.repo_url:	×
396	return	×
397
398	# Disable the user's .gitconfig file
399	os.environ['GIT_CONFIG_NOGLOBAL'] = 'yes'	×
400
401	build_config = self.config.get('build', {})	×
402	self.repo_url = build_config.get('repository', self.repo_url)	×
403	self.repo_tag = build_config.get('tag', self.repo_tag)	×
404
405	git_path = os.path.join(self.codebase_path, '.git')	×
406	if not os.path.exists(git_path):	×
407	cmd = 'git clone {0} {1}'.format(self.repo_url, self.codebase_path)	×
408	sp.check_call(shlex.split(cmd))	×
409
410	curdir = os.getcwd()	×
411	os.chdir(self.codebase_path)	×
412	sp.check_call(shlex.split('git checkout {0}'.format(self.repo_tag)))	×
413	sp.check_call(shlex.split('git pull'))	×
414	os.chdir(curdir)	×
415
416	def profile(self):	3✔
417	# TODO: Replace with call to "profile" drivers
418
419	if self.expt.config.get('hpctoolkit', False) and self.exec_name:	×
420
421	envmod.module('load', 'hpctoolkit')	×
422
423	# Create the code structure file
424	hpcstruct_fname = '{0}.hpcstruct'.format(self.exec_name)	×
425	hpcstruct_path = os.path.join(self.expt.lab.bin_path,	×
426	hpcstruct_fname)
427
428	# TODO: Validate struct file
429	if not os.path.isfile(hpcstruct_path):	×
430	cmd = 'hpcstruct -o {0} {1}'.format(hpcstruct_path,	×
431	self.exec_path)
432	sp.check_call(shlex.split(cmd))	×
433
434	# Parse the profile output
435	hpctk_header = 'hpctoolkit-{0}-measurements'.format(self.exec_name)	×
436	hpctk_measure_dir = [os.path.join(self.output_path, f)	×
437	for f in os.listdir(self.output_path)
438	if f.startswith(hpctk_header)][0]
439
440	hpctk_db_dir = hpctk_measure_dir.replace('measurements',	×
441	'database')
442
443	# TODO: This needs to be model-specifc
444	src_path = os.path.join(self.codebase_path, 'src')	×
445
446	cmd = 'hpcprof-mpi -S {0} -I {1} -o {2} {3}'.format(	×
447	hpcstruct_path, src_path, hpctk_db_dir, hpctk_measure_dir)
448	sp.check_call(shlex.split(cmd))	×
449
450	if self.expt.config.get('scalasca', False):	×
451
452	envmod.module('use', '/home/900/mpc900/my_modules')	×
453	envmod.module('load', 'scalasca')	×
454
455	scorep_path = [os.path.join(self.output_path, f)	×
456	for f in os.listdir(self.output_path)
457	if f.startswith('scorep')][0]
458	cmd = 'scalasca -examine -s {0}'.format(scorep_path)	×
459	sp.check_call(shlex.split(cmd))	×
460
461	if self.expt.config.get('scorep', False):	×
462
463	envmod.module('load', 'scorep')	×
464
465	scorep_path = [os.path.join(self.output_path, f)	×
466	for f in os.listdir(self.output_path)
467	if f.startswith('scorep')][0]
468	cube_path = [os.path.join(scorep_path, f)	×
469	for f in os.listdir(scorep_path)
470	if f.endswith('.cubex')][0]
471	cmd = 'scorep-score {0}'.format(cube_path)	×
472	sp.check_call(shlex.split(cmd))	×
473
474	def get_restart_datetime(self, restart_path):	3✔
475	"""Given a restart path, parse the restart files and return a cftime
476	datetime (currently used for date-based restart pruning)"""
NEW 477	raise NotImplementedError	×

payu-org / payu / 6489602581

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous