6496659630

Committed 12 Oct 2023 01:57PM UTC coverage: 32.333% (+0.003%) from 32.33%

Build # 6496659630

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #109 from 4dn-dcic/upd_py3_11

Updates to use python versions 3.8 - 3.11

Run Details

1174 of 3631 relevant lines covered (32.33%)

1.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/tasks.py

# -*- coding: utf-8 -*-
import os
import sys
import webbrowser
from invoke.exceptions import UnexpectedExit
from contextlib import contextmanager
from invoke import task, run
import cProfile
import pstats
import importlib


docs_dir = 'docs'
build_dir = os.path.join(docs_dir, '_build')
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
PROJECT_NAME = 'scripts'


@contextmanager
def setenv(**kwargs):
    # Backup
    prev = {}
    for k, v in kwargs.items():
        if k in os.environ:
            prev[k] = os.environ[k]
        os.environ[k] = v

    yield

    # Restore
    for k in kwargs.keys():
        if k in prev:
            os.environ[k] = prev[k]
        else:
            del os.environ[k]


@task(aliases=['notebooks'])
def notebook(ctx):
    """
    Start IPython notebook server.
    """
    with setenv(PYTHONPATH='{root}/{prj}:{root}:{root}/tests'.format(root=ROOT_DIR, prj=PROJECT_NAME),
                JUPYTER_CONFIG_DIR='{root}/notebooks'.format(root=ROOT_DIR)):

        os.chdir('notebooks')

        # Need pty=True to let Ctrl-C kill the notebook server. Shrugs.
        try:
            run('jupyter nbextension enable --py widgetsnbextension')
            run('jupyter notebook --ip=*', pty=True)
        except KeyboardInterrupt:
            pass
        print("If notebook does not open on your chorme automagically, try adding this to your bash_profie")
        print("export BROWSER=/Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome")
        print("*for MacOS and Chrome only")


@task()
def share_notebook(ctx, branch='master'):
    # get git repo called origin
    print("sharing from github repo, NOT local")
    repo = run('git remote get-url origin').stdout.strip('\n')
    if not repo.startswith("http"):
        # ssh like git@github.com:org/name
        repo = repo.split(':')[-1]

    org, name = repo.split('/')[-2:]

    # https://mybinder.org/v2/gh/hms-dbmi/pystarter.git/master
    url = "https://mybinder.org/v2/gh/%s/%s/%s?filepath=notebooks" % (org, name, branch)
    print("Notebook can be accessed from here %s" % url)
    webbrowser.open_new_tab(url)


@task
def loc(ctx):
    """
    Count lines-of-code.
    """
    excludes = ['/test/', 'docs', 'htmlcov', 'README.md', 'README.rst', '.eggs']

    run('find . -iname "*py" | grep -v {} | xargs wc -l | sort -n'.format(
        ' '.join('-e ' + e for e in excludes)))


@task(aliases=['tests'])
def test(ctx, watch=False, last_failing=False, no_flake=False, k=''):
    """Run the tests.
    Note: --watch requires pytest-xdist to be installed.
    """
    import pytest
    if not no_flake:
        flake(ctx)
    args = []
    if k:
        args.append('-k %s' % k)
    if watch:
        args.append('-f')
    if last_failing:
        args.append('--lf')
    retcode = pytest.main(args)
    if retcode != 0:
        print("test failed exiting")
        sys.exit(retcode)


@task
def flake(ctx):
    '''static linter to ensure code passes standards'''
    """Run flake8 on codebase."""
    run('flake8 --ignore E501,E722,W503 .', echo=True)
    print("flake8 passed!!!")


@task
def clean(ctx):
    '''remove temporary build stuff'''
    run("rm -rf build")
    run("rm -rf dist")
    run("rm -rf *.egg-info")
    clean_docs(ctx)
    print("Cleaned up.")


@task
def deploy(ctx, version=None, local=False):
    '''clean, run tests, version, tag and push tag to git for publishing to pypi through travis'''
    print("preparing for deploy...")
    print("first lets clean everything up.")
    clean(ctx)
    print("now lets make sure the tests pass")
    test(ctx)
    print("next get version information")
    version = update_version(ctx, version)
    print("then tag the release in git")
    git_tag(ctx, version, "new production release %s" % (version))
    print("Build is now triggered for production deployment of %s "
          "check travis for build status" % (version))
    if local:
        publish(ctx)
    print("Also follow these additional instructions here")


@task
def update_version(ctx, version=None):
    '''update version of your library to be used in pypi'''
    import importlib
    vmod = importlib.import_module('%s._version' % PROJECT_NAME)
    print("Current version is ", vmod.__version__)
    if version is None:
        msg = "What version would you like to set for new release (please use x.x.x / semantic versioning): "
        if sys.version_info < (3, 0):
            version = raw_input(msg)  # noqa: F821
        else:
            version = input(msg)

    # read the versions file
    lines = []
    with open(PROJECT_NAME + "/_version.py") as readfile:
        lines = readfile.readlines()

    if lines:
        with open(PROJECT_NAME + "/_version.py", 'w') as writefile:
            lines[-1] = '__version__ = "%s"\n' % (version.strip())
            writefile.writelines(lines)

    run("git add %s/_version.py" % PROJECT_NAME)
    run("git commit -m 'version bump'")
    print("version updated to", version)
    return version


@task
def git_tag(ctx, tag_name, msg):
    """create a tag and push to github.  Will trigger pypi publish if travis is setup"""
    run('git tag -a %s -m "%s"' % (tag_name, msg))
    run('git push --tags')
    run('git push')


@task
def clean_docs(ctx):
    """clean all generated docs"""
    run("rm -rf %s" % build_dir, echo=True)


@task
def browse_docs(ctx):
    """show the generated docs in a webbrowser"""
    path = os.path.join(build_dir, 'index.html')
    webbrowser.open_new_tab(path)


@task
def docs(ctx, clean=False, browse=False, watch=False):
    """Build the docs."""
    if clean:
        clean_docs(ctx)
    run("sphinx-build %s %s" % (docs_dir, build_dir), echo=True)
    if browse:
        browse_docs(ctx)
    if watch:
        watch_docs(ctx)


@task
def watch_docs(ctx):
    """Run build the docs when a file changes."""
    try:
        import sphinx_autobuild  # noqa
    except ImportError:
        print('ERROR: watch task requires the sphinx_autobuild package.')
        print('Install it with:')
        print('    pip install sphinx-autobuild')
        sys.exit(1)
    run('sphinx-autobuild {0} {1} --watch {2}'.format(
        docs_dir, build_dir, '4DNWranglerTools'), echo=True, pty=True)


@task
def browse_cov(ctx, norun=False):
    '''View test coverage results in browser'''
    if not norun:
        try:
            test(ctx)
        except UnexpectedExit:
            pass
    webbrowser.open_new_tab('htmlcov/index.html')


@task
def publish(ctx, test=False):
    """Publish to the cheeseshop."""
    clean(ctx)
    if test:
        run('python setup.py register -r test sdist bdist_wheel', echo=True)
        run('twine upload dist/* -r test', echo=True)
    else:
        run('python setup.py register sdist bdist_wheel', echo=True)
        run('twine upload dist/*', echo=True)


@task
def profile(ctx, module, method, filename=None):
    '''
    run modeule.method through the profile and dump results to filename
    '''

    mod = importlib.import_module(module)
    fn = getattr(mod, method)
    pr = cProfile.Profile()
    pr.enable()
    res = fn()
    pr.disable()
    if res:
        print(res)
    pr.create_stats()
    ps = pstats.Stats(pr).sort_stats('time')
    ps.print_stats()
    ps.print_callers()
    ps.print_callees()
    if filename is not None:
        ps.dump_stats(filename)

1	# -- coding: utf-8 --
2	import os	×
3	import sys	×
4	import webbrowser	×
5	from invoke.exceptions import UnexpectedExit	×
6	from contextlib import contextmanager	×
7	from invoke import task, run	×
8	import cProfile	×
9	import pstats	×
10	import importlib	×
11
12
13	docs_dir = 'docs'	×
14	build_dir = os.path.join(docs_dir, '_build')	×
15	ROOT_DIR = os.path.abspath(os.path.dirname(__file__))	×
16	PROJECT_NAME = 'scripts'	×
17
18
19	@contextmanager	×
20	def setenv(**kwargs):	×
21	# Backup
22	prev = {}	×
23	for k, v in kwargs.items():	×
24	if k in os.environ:	×
25	prev[k] = os.environ[k]	×
26	os.environ[k] = v	×
27
28	yield	×
29
30	# Restore
31	for k in kwargs.keys():	×
32	if k in prev:	×
33	os.environ[k] = prev[k]	×
34	else:
35	del os.environ[k]	×
36
37
38	@task(aliases=['notebooks'])	×
39	def notebook(ctx):	×
40	"""
41	Start IPython notebook server.
42	"""
43	with setenv(PYTHONPATH='{root}/{prj}:{root}:{root}/tests'.format(root=ROOT_DIR, prj=PROJECT_NAME),	×
44	JUPYTER_CONFIG_DIR='{root}/notebooks'.format(root=ROOT_DIR)):
45
46	os.chdir('notebooks')	×
47
48	# Need pty=True to let Ctrl-C kill the notebook server. Shrugs.
49	try:	×
50	run('jupyter nbextension enable --py widgetsnbextension')	×
51	run('jupyter notebook --ip=*', pty=True)	×
52	except KeyboardInterrupt:	×
53	pass	×
54	print("If notebook does not open on your chorme automagically, try adding this to your bash_profie")	×
55	print("export BROWSER=/Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome")	×
56	print("*for MacOS and Chrome only")	×
57
58
59	@task()	×
60	def share_notebook(ctx, branch='master'):	×
61	# get git repo called origin
62	print("sharing from github repo, NOT local")	×
63	repo = run('git remote get-url origin').stdout.strip('\n')	×
64	if not repo.startswith("http"):	×
65	# ssh like git@github.com:org/name
66	repo = repo.split(':')[-1]	×
67
68	org, name = repo.split('/')[-2:]	×
69
70	# https://mybinder.org/v2/gh/hms-dbmi/pystarter.git/master
71	url = "https://mybinder.org/v2/gh/%s/%s/%s?filepath=notebooks" % (org, name, branch)	×
72	print("Notebook can be accessed from here %s" % url)	×
73	webbrowser.open_new_tab(url)	×
74
75
76	@task	×
77	def loc(ctx):	×
78	"""
79	Count lines-of-code.
80	"""
81	excludes = ['/test/', 'docs', 'htmlcov', 'README.md', 'README.rst', '.eggs']	×
82
83	run('find . -iname "*py" \| grep -v {} \| xargs wc -l \| sort -n'.format(	×
84	' '.join('-e ' + e for e in excludes)))
85
86
87	@task(aliases=['tests'])	×
88	def test(ctx, watch=False, last_failing=False, no_flake=False, k=''):	×
89	"""Run the tests.
90	Note: --watch requires pytest-xdist to be installed.
91	"""
92	import pytest	×
93	if not no_flake:	×
94	flake(ctx)	×
95	args = []	×
96	if k:	×
97	args.append('-k %s' % k)	×
98	if watch:	×
99	args.append('-f')	×
100	if last_failing:	×
101	args.append('--lf')	×
102	retcode = pytest.main(args)	×
103	if retcode != 0:	×
104	print("test failed exiting")	×
105	sys.exit(retcode)	×
106
107
108	@task	×
109	def flake(ctx):	×
110	'''static linter to ensure code passes standards'''
111	"""Run flake8 on codebase."""
112	run('flake8 --ignore E501,E722,W503 .', echo=True)	×
113	print("flake8 passed!!!")	×
114
115
116	@task	×
117	def clean(ctx):	×
118	'''remove temporary build stuff'''
119	run("rm -rf build")	×
120	run("rm -rf dist")	×
121	run("rm -rf *.egg-info")	×
122	clean_docs(ctx)	×
123	print("Cleaned up.")	×
124
125
126	@task	×
127	def deploy(ctx, version=None, local=False):	×
128	'''clean, run tests, version, tag and push tag to git for publishing to pypi through travis'''
129	print("preparing for deploy...")	×
130	print("first lets clean everything up.")	×
131	clean(ctx)	×
132	print("now lets make sure the tests pass")	×
133	test(ctx)	×
134	print("next get version information")	×
135	version = update_version(ctx, version)	×
136	print("then tag the release in git")	×
137	git_tag(ctx, version, "new production release %s" % (version))	×
138	print("Build is now triggered for production deployment of %s "	×
139	"check travis for build status" % (version))
140	if local:	×
141	publish(ctx)	×
142	print("Also follow these additional instructions here")	×
143
144
145	@task	×
146	def update_version(ctx, version=None):	×
147	'''update version of your library to be used in pypi'''
148	import importlib	×
149	vmod = importlib.import_module('%s._version' % PROJECT_NAME)	×
150	print("Current version is ", vmod.__version__)	×
151	if version is None:	×
152	msg = "What version would you like to set for new release (please use x.x.x / semantic versioning): "	×
153	if sys.version_info < (3, 0):	×
154	version = raw_input(msg) # noqa: F821	×
155	else:
156	version = input(msg)	×
157
158	# read the versions file
159	lines = []	×
160	with open(PROJECT_NAME + "/_version.py") as readfile:	×
161	lines = readfile.readlines()	×
162
163	if lines:	×
164	with open(PROJECT_NAME + "/_version.py", 'w') as writefile:	×
165	lines[-1] = '__version__ = "%s"\n' % (version.strip())	×
166	writefile.writelines(lines)	×
167
168	run("git add %s/_version.py" % PROJECT_NAME)	×
169	run("git commit -m 'version bump'")	×
170	print("version updated to", version)	×
171	return version	×
172
173
174	@task	×
175	def git_tag(ctx, tag_name, msg):	×
176	"""create a tag and push to github. Will trigger pypi publish if travis is setup"""
177	run('git tag -a %s -m "%s"' % (tag_name, msg))	×
178	run('git push --tags')	×
179	run('git push')	×
180
181
182	@task	×
183	def clean_docs(ctx):	×
184	"""clean all generated docs"""
185	run("rm -rf %s" % build_dir, echo=True)	×
186
187
188	@task	×
189	def browse_docs(ctx):	×
190	"""show the generated docs in a webbrowser"""
191	path = os.path.join(build_dir, 'index.html')	×
192	webbrowser.open_new_tab(path)	×
193
194
195	@task	×
196	def docs(ctx, clean=False, browse=False, watch=False):	×
197	"""Build the docs."""
198	if clean:	×
199	clean_docs(ctx)	×
200	run("sphinx-build %s %s" % (docs_dir, build_dir), echo=True)	×
201	if browse:	×
202	browse_docs(ctx)	×
203	if watch:	×
204	watch_docs(ctx)	×
205
206
207	@task	×
208	def watch_docs(ctx):	×
209	"""Run build the docs when a file changes."""
210	try:	×
211	import sphinx_autobuild # noqa	×
212	except ImportError:	×
213	print('ERROR: watch task requires the sphinx_autobuild package.')	×
214	print('Install it with:')	×
215	print(' pip install sphinx-autobuild')	×
216	sys.exit(1)	×
217	run('sphinx-autobuild {0} {1} --watch {2}'.format(	×
218	docs_dir, build_dir, '4DNWranglerTools'), echo=True, pty=True)
219
220
221	@task	×
222	def browse_cov(ctx, norun=False):	×
223	'''View test coverage results in browser'''
224	if not norun:	×
225	try:	×
226	test(ctx)	×
227	except UnexpectedExit:	×
228	pass	×
229	webbrowser.open_new_tab('htmlcov/index.html')	×
230
231
232	@task	×
233	def publish(ctx, test=False):	×
234	"""Publish to the cheeseshop."""
235	clean(ctx)	×
236	if test:	×
237	run('python setup.py register -r test sdist bdist_wheel', echo=True)	×
238	run('twine upload dist/* -r test', echo=True)	×
239	else:
240	run('python setup.py register sdist bdist_wheel', echo=True)	×
241	run('twine upload dist/*', echo=True)	×
242
243
244	@task	×
245	def profile(ctx, module, method, filename=None):	×
246	'''
247	run modeule.method through the profile and dump results to filename
248	'''
249
250	mod = importlib.import_module(module)	×
251	fn = getattr(mod, method)	×
252	pr = cProfile.Profile()	×
253	pr.enable()	×
254	res = fn()	×
255	pr.disable()	×
256	if res:	×
257	print(res)	×
258	pr.create_stats()	×
259	ps = pstats.Stats(pr).sort_stats('time')	×
260	ps.print_stats()	×
261	ps.print_callers()	×
262	ps.print_callees()	×
263	if filename is not None:	×
264	ps.dump_stats(filename)	×

4dn-dcic / dcicwrangling / 6496659630

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous