3988894829

Build Type

github-actions

Committed by GitHub

Commit Message

Merge 89db93e8c into da1060a80

Pull Request Pull Request #190: Replace setup.py plugins with stand-alone scripts

Run Details

199 of 199 new or added lines in 7 files covered. (100.0%)

2010 of 2644 relevant lines covered (76.02%)

0.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.83

/py/desiutil/modules.py

# Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-
"""
================
desiutil.modules
================

This package contains code for processing and installing `Module files`_.

.. _`Module files`: http://modules.sourceforge.net
"""
import os
import re
import sys
from argparse import ArgumentParser
from shutil import which
from stat import S_IRUSR, S_IRGRP
try:
    from ConfigParser import SafeConfigParser
except ImportError:
    from configparser import ConfigParser as SafeConfigParser
from pkg_resources import resource_filename
from . import __version__ as desiutilVersion
from .io import unlock_file
from .log import log


def init_modules(moduleshome=None, method=False, command=False):
    """Set up the Modules infrastructure.

    Parameters
    ----------
    moduleshome : :class:`str`, optional
        The path containing the Modules init code.  If not provided,
        :envvar:`MODULESHOME` will be used.
    method : :class:`bool`, optional
        If ``True`` the function returned will be suitable for converting
        into an instance method.
    command : :class:`bool`, optional
        If ``True``, return the command used to call Modules, rather than
        a function.

    Returns
    -------
    callable
        A function that wraps the ``module()`` function, and deals with
        setting :data:`sys.path`.  Returns ``None`` if no Modules infrastructure
        could be found.
    """
    if moduleshome is None:
        try:
            moduleshome = os.environ['MODULESHOME']
        except KeyError:
            return None
    if not os.path.isdir(moduleshome):
        return None
    if 'MODULEPATH' not in os.environ:
        os.environ['MODULEPATH'] = ''
        dot_modulespath = os.path.join(moduleshome, 'init', '.modulespath')
        if os.path.exists(dot_modulespath):
            path = list()
            with open(dot_modulespath, 'r') as f:
                for line in f.readlines():
                    line = re.sub("#.*$", '', line.strip())
                    if line != '':
                        path.append(line)
            os.environ['MODULEPATH'] = ':'.join(path)
        modulerc = os.path.join(moduleshome, 'init', 'modulerc')
        if os.path.exists(modulerc):
            path = list()
            with open(modulerc, 'r') as f:
                for line in f.readlines():
                    line = re.sub("#.*$", '', line.strip())
                    if line != '' and line.startswith('module use'):
                        p = os.path.expanduser(line.replace('module use ', '').strip())
                        path.append(p)
            os.environ['MODULEPATH'] = ':'.join(path)
    if 'LOADEDMODULES' not in os.environ:
        os.environ['LOADEDMODULES'] = ''
    if os.path.exists(os.path.join(moduleshome, 'modulecmd.tcl')):
        #
        # TCL version!
        #
        if 'TCLSH' in os.environ:
            tclsh = os.environ['TCLSH']
        else:
            tclsh = which('tclsh')
        if tclsh is None:
            raise ValueError("TCL Modules detected, but no tclsh excecutable found.")
        modulecmd = [tclsh, os.path.join(moduleshome, 'modulecmd.tcl'), 'python']
    elif os.path.exists(os.path.join(moduleshome, 'libexec', 'lmod')):
        #
        # Lmod version!
        #
        modulecmd = [os.path.join(moduleshome, 'libexec', 'lmod'), 'python']
    else:
        #
        # This should work on all NERSC systems, assuming the user's environment
        # is not totally screwed up.
        #
        tmpcmd = which('modulecmd')
        if tmpcmd is None:
            raise ValueError("Modules environment detected, but no 'modulecmd' excecutable found.")
        modulecmd = [tmpcmd, 'python']
    if 'MODULE_VERSION' in os.environ:
        os.environ['MODULE_VERSION_STACK'] = os.environ['MODULE_VERSION']
    if command:
        return modulecmd

    def desiutil_module(command, *arguments):
        """Call the Modules command.

        Parameters
        ----------
        command : :class:`str`
            Command passed to the base module command.
        arguments : :class:`list`
            Arguments passed to the module command.

        Returns
        -------
        None

        Notes
        -----
        The base module function does not update :data:`sys.path` to
        reflect any additional directories added to
        :envvar:`PYTHONPATH`.  The wrapper function takes care
        of that (and uses set theory!).

        This module also avoids potential Python 3 conflicts.
        """
        import os
        import subprocess
        from sys import path
        try:
            old_python_path = set(os.environ['PYTHONPATH'].split(':'))
        except KeyError:
            old_python_path = set()
        cmd = modulecmd + [command] + list(arguments)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        out, err = p.communicate()
        status = p.returncode
        # exec out in globals(), locals()
        exec(out, globals(), locals())
        try:
            new_python_path = set(os.environ['PYTHONPATH'].split(':'))
        except KeyError:
            new_python_path = set()
        add_path = new_python_path - old_python_path
        for p in add_path:
            path.insert(int(path[0] == ''), p)
        return

    if method:
        def desiutil_module_method(self, command, *arguments):
            return desiutil_module(command, *arguments)
        desiutil_module_method.__doc__ = desiutil_module.__doc__
        return desiutil_module_method
    return desiutil_module


def configure_module(product, version, product_root, working_dir=None, dev=False):
    """Decide what needs to go in the Module file.

    Parameters
    ----------
    product : :class:`str`
        Name of the product.
    version : :class:`str`
        Version of the product.
    product_root : :class:`str`
        Directory that contains the installed code.
    working_dir : :class:`str`, optional
        The directory to examine.  If not set, the current working directory
        will be used.
    dev : :class:`bool`, optional
        If ``True``, interpret the directory as a 'development' install,
        *e.g.* a trunk or branch install.

    Returns
    -------
    :class:`dict`
        A dictionary containing the module configuration parameters.
    """
    if working_dir is None:
        working_dir = os.getcwd()
    module_keywords = {
        'name': product,
        'version': version,
        'product_root': product_root,
        'needs_bin': '# ',
        'needs_python': '# ',
        'needs_trunk_py': '# ',
        'trunk_py_dir': '/py',
        'needs_ld_lib': '# ',
        'needs_idl': '# ',
        'pyversion': "python{0:d}.{1:d}".format(*sys.version_info)
        }
    if os.path.isdir(os.path.join(working_dir, 'bin')):
        module_keywords['needs_bin'] = ''
    if os.path.isdir(os.path.join(working_dir, 'lib')):
        module_keywords['needs_ld_lib'] = ''
    if os.path.isdir(os.path.join(working_dir, 'pro')):
        module_keywords['needs_idl'] = ''
    if (os.path.exists(os.path.join(working_dir, 'setup.py')) and
        (os.path.isdir(os.path.join(working_dir, product)) or
         os.path.isdir(os.path.join(working_dir, product.lower())))):
        if dev:
            module_keywords['needs_trunk_py'] = ''
            module_keywords['trunk_py_dir'] = ''
        else:
            module_keywords['needs_python'] = ''
    if os.path.isdir(os.path.join(working_dir, 'py')):
        if dev:
            module_keywords['needs_trunk_py'] = ''
        else:
            module_keywords['needs_python'] = ''
    if os.path.isdir(os.path.join(working_dir, 'python')):
        if dev:
            module_keywords['needs_trunk_py'] = ''
            module_keywords['trunk_py_dir'] = '/python'
        else:
            module_keywords['needs_python'] = ''
    if os.path.exists(os.path.join(working_dir, 'setup.cfg')):
        conf = SafeConfigParser()
        conf.read([os.path.join(working_dir, 'setup.cfg')])
        if conf.has_section('entry_points') or conf.has_section('options.entry_points'):
            module_keywords['needs_bin'] = ''
    return module_keywords


def process_module(module_file, module_keywords, module_dir):
    """Process a Module file.

    Parameters
    ----------
    module_file : :class:`str`
        A template Module file to process.
    module_keywords : :class:`dict`
        The parameters to use for Module file processing.
    module_dir : :class:`str`
        The directory where the Module file should be installed.

    Returns
    -------
    :class:`str`
        The text of the processed Module file.
    """
    if not os.path.isdir(os.path.join(module_dir, module_keywords['name'])):
        os.makedirs(os.path.join(module_dir, module_keywords['name']))
    install_module_file = os.path.join(module_dir, module_keywords['name'],
                                       module_keywords['version'])
    with open(module_file) as m:
        mod = m.read().format(**module_keywords)
    _write_module_data(install_module_file, mod)
    return mod


def default_module(module_keywords, module_dir):
    """Install or update a .version file to set the default Module.

    Parameters
    ----------
    module_keywords : :class:`dict`
        The parameters to use for Module file processing.
    module_dir : :class:`str`
        The directory where the Module file should be installed.

    Returns
    -------
    :class:`str`
        The text of the processed .version file.
    """
    dot_template = '#%Module1.0\nset ModulesVersion "{version}"\n'
    install_version_file = os.path.join(module_dir, module_keywords['name'],
                                        '.version')
    dot_version = dot_template.format(**module_keywords)
    _write_module_data(install_version_file, dot_version)
    return dot_version


def _write_module_data(filename, data):
    """Write and permission-lock Module file data.  This is intended
    to consolidate some duplicated code.
    """
    with unlock_file(filename, 'w') as f:
        f.write(data)
    os.chmod(filename, S_IRUSR | S_IRGRP)
    return


def main():
    """Entry-point for command-line scripts.

    Returns
    -------
    :class:`int`
        An integer suitable for passing to :func:`sys.exit`.
    """
    parser = ArgumentParser(description="Install a Module file for a DESI software product.",
                            prog=os.path.basename(sys.argv[0]))
    parser.add_argument('-d', '--default', dest='default', action='store_true', help='Mark this Module as default.')
    parser.add_argument('-m', '--modules', dest='modules', help='Set the Module install directory.')
    parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + desiutilVersion)
    parser.add_argument('product', help='Name of product.')
    parser.add_argument('product_version', help='Version of product.')
    options = parser.parse_args()

    if options.modules is None:
        try:
            self.modules = os.path.join('/global/common/software/desi',
                                        os.environ['NERSC_HOST'],
                                        'desiconda',
                                        'current',
                                        'modulefiles')
        except KeyError:
            try:
                options.modules = os.path.join(os.environ['DESI_PRODUCT_ROOT'],
                                               'modulefiles')
            except KeyError:
                log.error("Could not determine a Module install directory!")
                return 1

    dev = ('dev' in options.product_version or
           'main' in options.product_version or
           'master' in options.product_version or
           'branches' in options.product_version or
           'trunk' in options.product_version)
    working_dir = os.path.abspath('.')
    module_keywords = configure_module(options.product, options.product_version, working_dir, dev=dev)
    module_file = os.path.join(working_dir, 'etc', '{0}.module'.format(options.product))

    if not os.path.exists(module_file):
        log.warning("Could not find Module file: %s; using default.", module_file)
        module_file = resource_filename('desiutil', 'data/desiutil.module')

    process_module(module_file, module_keywords, options.modules)

    if options.default:
        default_module(module_keywords, options.modules)

    return 0

1	# Licensed under a 3-clause BSD style license - see LICENSE.rst
2	# -- coding: utf-8 --
3	"""	1✔
4	================
5	desiutil.modules
6	================
7
8	This package contains code for processing and installing `Module files`_.
9
10	.. _`Module files`: http://modules.sourceforge.net
11	"""
12	import os	1✔
13	import re	1✔
14	import sys	1✔
15	from argparse import ArgumentParser	1✔
16	from shutil import which	1✔
17	from stat import S_IRUSR, S_IRGRP	1✔
18	try:	1✔
19	from ConfigParser import SafeConfigParser	1✔
20	except ImportError:
21	from configparser import ConfigParser as SafeConfigParser
22	from pkg_resources import resource_filename	1✔
23	from . import __version__ as desiutilVersion	1✔
24	from .io import unlock_file	1✔
25	from .log import log	1✔
26
27
28	def init_modules(moduleshome=None, method=False, command=False):	1✔
29	"""Set up the Modules infrastructure.
30
31	Parameters
32	----------
33	moduleshome : :class:`str`, optional
34	The path containing the Modules init code. If not provided,
35	:envvar:`MODULESHOME` will be used.
36	method : :class:`bool`, optional
37	If ``True`` the function returned will be suitable for converting
38	into an instance method.
39	command : :class:`bool`, optional
40	If ``True``, return the command used to call Modules, rather than
41	a function.
42
43	Returns
44	-------
45	callable
46	A function that wraps the ``module()`` function, and deals with
47	setting :data:`sys.path`. Returns ``None`` if no Modules infrastructure
48	could be found.
49	"""
50	if moduleshome is None:	1✔
51	try:	1✔
52	moduleshome = os.environ['MODULESHOME']	1✔
53	except KeyError:	1✔
54	return None	1✔
55	if not os.path.isdir(moduleshome):	1✔
56	return None	1✔
57	if 'MODULEPATH' not in os.environ:	1✔
58	os.environ['MODULEPATH'] = ''	1✔
59	dot_modulespath = os.path.join(moduleshome, 'init', '.modulespath')	1✔
60	if os.path.exists(dot_modulespath):	1✔
61	path = list()	1✔
62	with open(dot_modulespath, 'r') as f:	1✔
63	for line in f.readlines():	1✔
64	line = re.sub("#.*$", '', line.strip())	1✔
65	if line != '':	1✔
66	path.append(line)	1✔
67	os.environ['MODULEPATH'] = ':'.join(path)	1✔
68	modulerc = os.path.join(moduleshome, 'init', 'modulerc')	1✔
69	if os.path.exists(modulerc):	1✔
70	path = list()	1✔
71	with open(modulerc, 'r') as f:	1✔
72	for line in f.readlines():	1✔
73	line = re.sub("#.*$", '', line.strip())	1✔
74	if line != '' and line.startswith('module use'):	1✔
75	p = os.path.expanduser(line.replace('module use ', '').strip())	1✔
76	path.append(p)	1✔
77	os.environ['MODULEPATH'] = ':'.join(path)	1✔
78	if 'LOADEDMODULES' not in os.environ:	1✔
79	os.environ['LOADEDMODULES'] = ''	1✔
80	if os.path.exists(os.path.join(moduleshome, 'modulecmd.tcl')):	1✔
81	#
82	# TCL version!
83	#
84	if 'TCLSH' in os.environ:	1✔
85	tclsh = os.environ['TCLSH']	1✔
86	else:
87	tclsh = which('tclsh')	1✔
88	if tclsh is None:	1✔
89	raise ValueError("TCL Modules detected, but no tclsh excecutable found.")	×
90	modulecmd = [tclsh, os.path.join(moduleshome, 'modulecmd.tcl'), 'python']	1✔
91	elif os.path.exists(os.path.join(moduleshome, 'libexec', 'lmod')):	1✔
92	#
93	# Lmod version!
94	#
95	modulecmd = [os.path.join(moduleshome, 'libexec', 'lmod'), 'python']	×
96	else:
97	#
98	# This should work on all NERSC systems, assuming the user's environment
99	# is not totally screwed up.
100	#
101	tmpcmd = which('modulecmd')	1✔
102	if tmpcmd is None:	1✔
103	raise ValueError("Modules environment detected, but no 'modulecmd' excecutable found.")	×
104	modulecmd = [tmpcmd, 'python']	1✔
105	if 'MODULE_VERSION' in os.environ:	1✔
106	os.environ['MODULE_VERSION_STACK'] = os.environ['MODULE_VERSION']	1✔
107	if command:	1✔
108	return modulecmd	1✔
109
110	def desiutil_module(command, *arguments):	1✔
111	"""Call the Modules command.
112
113	Parameters
114	----------
115	command : :class:`str`
116	Command passed to the base module command.
117	arguments : :class:`list`
118	Arguments passed to the module command.
119
120	Returns
121	-------
122	None
123
124	Notes
125	-----
126	The base module function does not update :data:`sys.path` to
127	reflect any additional directories added to
128	:envvar:`PYTHONPATH`. The wrapper function takes care
129	of that (and uses set theory!).
130
131	This module also avoids potential Python 3 conflicts.
132	"""
133	import os	×
134	import subprocess	×
135	from sys import path	×
136	try:	×
137	old_python_path = set(os.environ['PYTHONPATH'].split(':'))	×
138	except KeyError:	×
139	old_python_path = set()	×
140	cmd = modulecmd + [command] + list(arguments)	×
141	p = subprocess.Popen(cmd, stdout=subprocess.PIPE,	×
142	stderr=subprocess.PIPE)
143	out, err = p.communicate()	×
144	status = p.returncode	×
145	# exec out in globals(), locals()
146	exec(out, globals(), locals())	×
147	try:	×
148	new_python_path = set(os.environ['PYTHONPATH'].split(':'))	×
149	except KeyError:	×
150	new_python_path = set()	×
151	add_path = new_python_path - old_python_path	×
152	for p in add_path:	×
153	path.insert(int(path[0] == ''), p)	×
154	return	×
155
156	if method:	1✔
157	def desiutil_module_method(self, command, *arguments):	1✔
158	return desiutil_module(command, *arguments)	×
159	desiutil_module_method.__doc__ = desiutil_module.__doc__	1✔
160	return desiutil_module_method	1✔
161	return desiutil_module	1✔
162
163
164	def configure_module(product, version, product_root, working_dir=None, dev=False):	1✔
165	"""Decide what needs to go in the Module file.
166
167	Parameters
168	----------
169	product : :class:`str`
170	Name of the product.
171	version : :class:`str`
172	Version of the product.
173	product_root : :class:`str`
174	Directory that contains the installed code.
175	working_dir : :class:`str`, optional
176	The directory to examine. If not set, the current working directory
177	will be used.
178	dev : :class:`bool`, optional
179	If ``True``, interpret the directory as a 'development' install,
180	e.g. a trunk or branch install.
181
182	Returns
183	-------
184	:class:`dict`
185	A dictionary containing the module configuration parameters.
186	"""
187	if working_dir is None:	1✔
188	working_dir = os.getcwd()	×
189	module_keywords = {	1✔
190	'name': product,
191	'version': version,
192	'product_root': product_root,
193	'needs_bin': '# ',
194	'needs_python': '# ',
195	'needs_trunk_py': '# ',
196	'trunk_py_dir': '/py',
197	'needs_ld_lib': '# ',
198	'needs_idl': '# ',
199	'pyversion': "python{0:d}.{1:d}".format(*sys.version_info)
200	}
201	if os.path.isdir(os.path.join(working_dir, 'bin')):	1✔
202	module_keywords['needs_bin'] = ''	1✔
203	if os.path.isdir(os.path.join(working_dir, 'lib')):	1✔
204	module_keywords['needs_ld_lib'] = ''	1✔
205	if os.path.isdir(os.path.join(working_dir, 'pro')):	1✔
206	module_keywords['needs_idl'] = ''	1✔
207	if (os.path.exists(os.path.join(working_dir, 'setup.py')) and	1✔
208	(os.path.isdir(os.path.join(working_dir, product)) or
209	os.path.isdir(os.path.join(working_dir, product.lower())))):
210	if dev:	1✔
211	module_keywords['needs_trunk_py'] = ''	1✔
212	module_keywords['trunk_py_dir'] = ''	1✔
213	else:
214	module_keywords['needs_python'] = ''	1✔
215	if os.path.isdir(os.path.join(working_dir, 'py')):	1✔
216	if dev:	1✔
217	module_keywords['needs_trunk_py'] = ''	1✔
218	else:
219	module_keywords['needs_python'] = ''	1✔
220	if os.path.isdir(os.path.join(working_dir, 'python')):	1✔
221	if dev:	×
222	module_keywords['needs_trunk_py'] = ''	×
223	module_keywords['trunk_py_dir'] = '/python'	×
224	else:
225	module_keywords['needs_python'] = ''	×
226	if os.path.exists(os.path.join(working_dir, 'setup.cfg')):	1✔
227	conf = SafeConfigParser()	1✔
228	conf.read([os.path.join(working_dir, 'setup.cfg')])	1✔
229	if conf.has_section('entry_points') or conf.has_section('options.entry_points'):	1✔
230	module_keywords['needs_bin'] = ''	1✔
231	return module_keywords	1✔
232
233
234	def process_module(module_file, module_keywords, module_dir):	1✔
235	"""Process a Module file.
236
237	Parameters
238	----------
239	module_file : :class:`str`
240	A template Module file to process.
241	module_keywords : :class:`dict`
242	The parameters to use for Module file processing.
243	module_dir : :class:`str`
244	The directory where the Module file should be installed.
245
246	Returns
247	-------
248	:class:`str`
249	The text of the processed Module file.
250	"""
251	if not os.path.isdir(os.path.join(module_dir, module_keywords['name'])):	1✔
252	os.makedirs(os.path.join(module_dir, module_keywords['name']))	1✔
253	install_module_file = os.path.join(module_dir, module_keywords['name'],	1✔
254	module_keywords['version'])
255	with open(module_file) as m:	1✔
256	mod = m.read().format(**module_keywords)	1✔
257	_write_module_data(install_module_file, mod)	1✔
258	return mod	1✔
259
260
261	def default_module(module_keywords, module_dir):	1✔
262	"""Install or update a .version file to set the default Module.
263
264	Parameters
265	----------
266	module_keywords : :class:`dict`
267	The parameters to use for Module file processing.
268	module_dir : :class:`str`
269	The directory where the Module file should be installed.
270
271	Returns
272	-------
273	:class:`str`
274	The text of the processed .version file.
275	"""
276	dot_template = '#%Module1.0\nset ModulesVersion "{version}"\n'	1✔
277	install_version_file = os.path.join(module_dir, module_keywords['name'],	1✔
278	'.version')
279	dot_version = dot_template.format(**module_keywords)	1✔
280	_write_module_data(install_version_file, dot_version)	1✔
281	return dot_version	1✔
282
283
284	def _write_module_data(filename, data):	1✔
285	"""Write and permission-lock Module file data. This is intended
286	to consolidate some duplicated code.
287	"""
288	with unlock_file(filename, 'w') as f:	1✔
289	f.write(data)	1✔
290	os.chmod(filename, S_IRUSR \| S_IRGRP)	1✔
291	return	1✔
292
293
294	def main():
295	"""Entry-point for command-line scripts.
296
297	Returns
298	-------
299	:class:`int`
300	An integer suitable for passing to :func:`sys.exit`.
301	"""
302	parser = ArgumentParser(description="Install a Module file for a DESI software product.",
303	prog=os.path.basename(sys.argv[0]))
304	parser.add_argument('-d', '--default', dest='default', action='store_true', help='Mark this Module as default.')
305	parser.add_argument('-m', '--modules', dest='modules', help='Set the Module install directory.')
306	parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + desiutilVersion)
307	parser.add_argument('product', help='Name of product.')
308	parser.add_argument('product_version', help='Version of product.')
309	options = parser.parse_args()
310
311	if options.modules is None:
312	try:
313	self.modules = os.path.join('/global/common/software/desi',
314	os.environ['NERSC_HOST'],
315	'desiconda',
316	'current',
317	'modulefiles')
318	except KeyError:
319	try:
320	options.modules = os.path.join(os.environ['DESI_PRODUCT_ROOT'],
321	'modulefiles')
322	except KeyError:
323	log.error("Could not determine a Module install directory!")
324	return 1
325
326	dev = ('dev' in options.product_version or
327	'main' in options.product_version or
328	'master' in options.product_version or
329	'branches' in options.product_version or
330	'trunk' in options.product_version)
331	working_dir = os.path.abspath('.')
332	module_keywords = configure_module(options.product, options.product_version, working_dir, dev=dev)
333	module_file = os.path.join(working_dir, 'etc', '{0}.module'.format(options.product))
334
335	if not os.path.exists(module_file):
336	log.warning("Could not find Module file: %s; using default.", module_file)
337	module_file = resource_filename('desiutil', 'data/desiutil.module')
338
339	process_module(module_file, module_keywords, options.modules)
340
341	if options.default:
342	default_module(module_keywords, options.modules)
343
344	return 0

desihub / desiutil / 3988894829

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous