6489602581

Committed 12 Oct 2023 12:25AM UTC coverage: 45.573% (+2.8%) from 42.772%

Build # 6489602581

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #363 from jo-basevi/358-date-based-frequency

Add support for date-based restart frequency

Run Details

111 of 147 new or added lines in 10 files covered. (75.51%)

2 existing lines in 1 file now uncovered.

1580 of 3467 relevant lines covered (45.57%)

1.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

36.9

/payu/cli.py

"""payu.cli
   ========

   Command line interface tools

   :copyright: Copyright 2011 Marshall Ward, see AUTHORS for details.
   :license: Apache License, Version 2.0, see LICENSE for details
"""

import argparse
from distutils import sysconfig
import importlib
import os
import pkgutil
import shlex
import subprocess
import sys

import payu
import payu.envmod as envmod
from payu.fsops import is_conda
from payu.models import index as supported_models
from payu.schedulers import index as scheduler_index
import payu.subcommands


# Default configuration
DEFAULT_CONFIG = 'config.yaml'


def parse():
    """Parse the command line inputs and execute the subcommand."""

    parser = generate_parser()

    # Display help if no arguments are provided
    if len(sys.argv) == 1:
        parser.print_help()
    else:
        args = vars(parser.parse_args())
        run_cmd = args.pop('run_cmd')
        run_cmd(**args)


def generate_parser():
    """Parse the command line inputs generate and return parser."""

    # Build the list of subcommand modules
    modnames = [mod for (_, mod, _)
                in pkgutil.iter_modules(payu.subcommands.__path__,
                                        prefix=payu.subcommands.__name__ + '.')
                if mod.endswith('_cmd')]

    subcmds = [importlib.import_module(mod) for mod in modnames]

    # Construct the subcommand parser
    parser = argparse.ArgumentParser()
    parser.add_argument('--version', action='version',
                        version='payu {0}'.format(payu.__version__))

    subparsers = parser.add_subparsers()

    for cmd in subcmds:
        cmd_parser = subparsers.add_parser(cmd.title, **cmd.parameters)
        cmd_parser.set_defaults(run_cmd=cmd.runcmd)

        for arg in cmd.arguments:
            cmd_parser.add_argument(*arg['flags'], **arg['parameters'])

    return parser


def get_model_type(model_type, config):
    """Determine and validate the active model type."""

    # If no model type is given, then check the config file
    if not model_type:
        model_type = config.get('model')

    # If there is still no model type, try the parent directory
    if not model_type:
        model_type = os.path.basename(os.path.abspath(os.pardir))
        print('payu: warning: Assuming model is {0} based on parent directory '
              'name.'.format(model_type))

    if model_type not in supported_models:
        print('payu: error: Unknown model {0}'.format(model_type))
        sys.exit(-1)


def set_env_vars(init_run=None, n_runs=None, lab_path=None, dir_path=None,
                 reproduce=False, force=False, force_prune_restarts=False):
    """Construct the environment variables used by payu for resubmissions."""
    payu_env_vars = {}

    if not is_conda():
        # Setup Python dynamic library link
        lib_paths = sysconfig.get_config_vars('LIBDIR')
        payu_env_vars['LD_LIBRARY_PATH'] = ':'.join(lib_paths)

    if 'PYTHONPATH' in os.environ:
        payu_env_vars['PYTHONPATH'] = os.environ['PYTHONPATH']

    # Set (or import) the path to the PAYU scripts (PAYU_PATH)
    # NOTE: We may be able to use sys.path[0] here.
    payu_binpath = os.environ.get('PAYU_PATH')

    if not payu_binpath or not os.path.isdir(payu_binpath):
        payu_binpath = os.path.dirname(sys.argv[0])

    payu_env_vars['PAYU_PATH'] = payu_binpath

    # Set the run counters
    if init_run:
        init_run = int(init_run)
        assert init_run >= 0
        payu_env_vars['PAYU_CURRENT_RUN'] = init_run

    if n_runs:
        n_runs = int(n_runs)
        assert n_runs > 0
        payu_env_vars['PAYU_N_RUNS'] = n_runs

    # Import explicit project paths
    if lab_path:
        payu_env_vars['PAYU_LAB_PATH'] = os.path.normpath(lab_path)

    if dir_path:
        payu_env_vars['PAYU_DIR_PATH'] = os.path.normpath(dir_path)

    if reproduce:
        payu_env_vars['PAYU_REPRODUCE'] = reproduce

    if force:
        payu_env_vars['PAYU_FORCE'] = force

    if force_prune_restarts:
        payu_env_vars['PAYU_FORCE_PRUNE_RESTARTS'] = force_prune_restarts

    # Pass through important module related environment variables
    module_env_vars = ['MODULESHOME', 'MODULES_CMD', 'MODULEPATH', 'MODULEV']
    for var in module_env_vars:
        if var in os.environ:
            payu_env_vars[var] = os.environ[var]

    return payu_env_vars


def submit_job(script, config, vars=None):
    """Submit a userscript the scheduler."""

    # TODO: Temporary stub to replicate the old approach
    sched_name = config.get('scheduler', 'pbs')
    sched_type = scheduler_index[sched_name]
    sched = sched_type()
    cmd = sched.submit(script, config, vars)
    print(cmd)

    subprocess.check_call(shlex.split(cmd))

1	"""payu.cli
2	========
3
4	Command line interface tools
5
6	:copyright: Copyright 2011 Marshall Ward, see AUTHORS for details.
7	:license: Apache License, Version 2.0, see LICENSE for details
8	"""
9
10	import argparse	3✔
11	from distutils import sysconfig	3✔
12	import importlib	3✔
13	import os	3✔
14	import pkgutil	3✔
15	import shlex	3✔
16	import subprocess	3✔
17	import sys	3✔
18
19	import payu	3✔
20	import payu.envmod as envmod	3✔
21	from payu.fsops import is_conda	3✔
22	from payu.models import index as supported_models	3✔
23	from payu.schedulers import index as scheduler_index	3✔
24	import payu.subcommands	3✔
25
26
27	# Default configuration
28	DEFAULT_CONFIG = 'config.yaml'	3✔
29
30
31	def parse():	3✔
32	"""Parse the command line inputs and execute the subcommand."""
33
34	parser = generate_parser()	×
35
36	# Display help if no arguments are provided
37	if len(sys.argv) == 1:	×
38	parser.print_help()	×
39	else:
40	args = vars(parser.parse_args())	×
41	run_cmd = args.pop('run_cmd')	×
42	run_cmd(**args)	×
43
44
45	def generate_parser():	3✔
46	"""Parse the command line inputs generate and return parser."""
47
48	# Build the list of subcommand modules
49	modnames = [mod for (_, mod, _)	3✔
50	in pkgutil.iter_modules(payu.subcommands.__path__,
51	prefix=payu.subcommands.__name__ + '.')
52	if mod.endswith('_cmd')]
53
54	subcmds = [importlib.import_module(mod) for mod in modnames]	3✔
55
56	# Construct the subcommand parser
57	parser = argparse.ArgumentParser()	3✔
58	parser.add_argument('--version', action='version',	3✔
59	version='payu {0}'.format(payu.__version__))
60
61	subparsers = parser.add_subparsers()	3✔
62
63	for cmd in subcmds:	3✔
64	cmd_parser = subparsers.add_parser(cmd.title, **cmd.parameters)	3✔
65	cmd_parser.set_defaults(run_cmd=cmd.runcmd)	3✔
66
67	for arg in cmd.arguments:	3✔
68	cmd_parser.add_argument(arg['flags'], *arg['parameters'])	3✔
69
70	return parser	3✔
71
72
73	def get_model_type(model_type, config):	3✔
74	"""Determine and validate the active model type."""
75
76	# If no model type is given, then check the config file
77	if not model_type:	×
78	model_type = config.get('model')	×
79
80	# If there is still no model type, try the parent directory
81	if not model_type:	×
82	model_type = os.path.basename(os.path.abspath(os.pardir))	×
83	print('payu: warning: Assuming model is {0} based on parent directory '	×
84	'name.'.format(model_type))
85
86	if model_type not in supported_models:	×
87	print('payu: error: Unknown model {0}'.format(model_type))	×
88	sys.exit(-1)	×
89
90
91	def set_env_vars(init_run=None, n_runs=None, lab_path=None, dir_path=None,	3✔
92	reproduce=False, force=False, force_prune_restarts=False):
93	"""Construct the environment variables used by payu for resubmissions."""
94	payu_env_vars = {}	×
95
96	if not is_conda():	×
97	# Setup Python dynamic library link
98	lib_paths = sysconfig.get_config_vars('LIBDIR')	×
99	payu_env_vars['LD_LIBRARY_PATH'] = ':'.join(lib_paths)	×
100
101	if 'PYTHONPATH' in os.environ:	×
102	payu_env_vars['PYTHONPATH'] = os.environ['PYTHONPATH']	×
103
104	# Set (or import) the path to the PAYU scripts (PAYU_PATH)
105	# NOTE: We may be able to use sys.path[0] here.
106	payu_binpath = os.environ.get('PAYU_PATH')	×
107
108	if not payu_binpath or not os.path.isdir(payu_binpath):	×
109	payu_binpath = os.path.dirname(sys.argv[0])	×
110
111	payu_env_vars['PAYU_PATH'] = payu_binpath	×
112
113	# Set the run counters
114	if init_run:	×
115	init_run = int(init_run)	×
116	assert init_run >= 0	×
117	payu_env_vars['PAYU_CURRENT_RUN'] = init_run	×
118
119	if n_runs:	×
120	n_runs = int(n_runs)	×
121	assert n_runs > 0	×
122	payu_env_vars['PAYU_N_RUNS'] = n_runs	×
123
124	# Import explicit project paths
125	if lab_path:	×
126	payu_env_vars['PAYU_LAB_PATH'] = os.path.normpath(lab_path)	×
127
128	if dir_path:	×
129	payu_env_vars['PAYU_DIR_PATH'] = os.path.normpath(dir_path)	×
130
131	if reproduce:	×
132	payu_env_vars['PAYU_REPRODUCE'] = reproduce	×
133
134	if force:	×
135	payu_env_vars['PAYU_FORCE'] = force	×
136
NEW 137	if force_prune_restarts:	×
NEW 138	payu_env_vars['PAYU_FORCE_PRUNE_RESTARTS'] = force_prune_restarts	×
139
140	# Pass through important module related environment variables
141	module_env_vars = ['MODULESHOME', 'MODULES_CMD', 'MODULEPATH', 'MODULEV']	×
142	for var in module_env_vars:	×
143	if var in os.environ:	×
144	payu_env_vars[var] = os.environ[var]	×
145
146	return payu_env_vars	×
147
148
149	def submit_job(script, config, vars=None):	3✔
150	"""Submit a userscript the scheduler."""
151
152	# TODO: Temporary stub to replicate the old approach
153	sched_name = config.get('scheduler', 'pbs')	×
154	sched_type = scheduler_index[sched_name]	×
155	sched = sched_type()	×
156	cmd = sched.submit(script, config, vars)	×
157	print(cmd)	×
158
159	subprocess.check_call(shlex.split(cmd))	×

payu-org / payu / 6489602581

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous