reanahub / reana-commons / 67

pending completion

Build # 67

Build Type

Pull #23

travis-ci

Committed by

web-flow

Commit Message

utils: hexdigest to calculate inputs hash

* Changes digest to hexdigest which I accidentally removed from
  d1f01fb6b while rebasing.

Pull Request Pull Request #23: utils: hexdigest to calculate inputs hash

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

0 of 187 relevant lines covered (0.0%)

0.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/reana_commons/utils.py

# -*- coding: utf-8 -*-
#
# This file is part of REANA.
# Copyright (C) 2018 CERN.
#
# REANA is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# REANA is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
# Suite 330, Boston, MA 02111-1307, USA.
#
# In applying this license, CERN does not waive the privileges and immunities
# granted to it by virtue of its status as an Intergovernmental Organization or
# submit itself to any jurisdiction.
"""REANA commons utils."""

import hashlib
import json
import os

import fs

import click


def click_table_printer(headers, _filter, data):
    """Generate space separated output for click commands."""
    _filter = [h.lower() for h in _filter] + [h.upper() for h in _filter]
    headers = [h for h in headers if not _filter or h in _filter]
    # Maximum header width
    header_widths = [len(h) for h in headers]

    for row in data:
        for idx in range(len(headers)):
            # If a row contains an element which is wider update maximum width
            if header_widths[idx] < len(str(row[idx])):
                header_widths[idx] = len(str(row[idx]))
    # Prepare the format string with the maximum widths
    formatted_output_parts = ['{{:<{0}}}'.format(hw)
                              for hw in header_widths]
    formatted_output = '   '.join(formatted_output_parts)
    # Print the table with the headers capitalized
    click.echo(formatted_output.format(*[h.upper() for h in headers]))
    for row in data:
        click.echo(formatted_output.format(*row))


def get_user_analyses_dir(org, user):
    """Build the analyses directory path for certain user and organization.

    :param org: Organization which user is part of.
    :param user: Working directory owner.
    :return: Path to the user's analyses directory.
    """
    return fs.path.join(org, user, 'analyses')


def calculate_hash_of_dir(directory, verbose=0):
    """Calculate hash of directory."""
    SHAhash = hashlib.md5()
    if not os.path.exists(directory):
        return -1

    try:
        for root, dirs, files in os.walk(directory):
            for names in sorted(files):
                filepath = os.path.join(root, names)
                try:
                    f1 = open(filepath, 'rb')
                except Exception:
                    # You can't open the file for some reason
                    f1.close()
                    # We return -1 since we can not ensure that the file that
                    # can not be read will not change from one execution to
                    # another.
                    return -1
                while 1:
                    # Read file in as little chunks
                    buf = f1.read(4096)
                    if not buf:
                        break
                    SHAhash.update(hashlib.md5(buf).hexdigest().encode())
                f1.close()

    except Exception:
        return -1
    return SHAhash.hexdigest()


def calculate_job_input_hash(job_spec, workflow_json):
    """Calculate md5 hash of job specification and workflow json."""
    job_md5_buffer = hashlib.md5()
    job_md5_buffer.update(json.dumps(job_spec).encode('utf-8'))
    job_md5_buffer.update(json.dumps(workflow_json).encode('utf-8'))
    return job_md5_buffer.hexdigest()

1	# -- coding: utf-8 --
2	#
3	# This file is part of REANA.
4	# Copyright (C) 2018 CERN.
5	#
6	# REANA is free software; you can redistribute it and/or modify it under the
7	# terms of the GNU General Public License as published by the Free Software
8	# Foundation; either version 2 of the License, or (at your option) any later
9	# version.
10	#
11	# REANA is distributed in the hope that it will be useful, but WITHOUT ANY
12	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14	#
15	# You should have received a copy of the GNU General Public License along with
16	# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
17	# Suite 330, Boston, MA 02111-1307, USA.
18	#
19	# In applying this license, CERN does not waive the privileges and immunities
20	# granted to it by virtue of its status as an Intergovernmental Organization or
21	# submit itself to any jurisdiction.
22	"""REANA commons utils."""	×
23
24	import hashlib	×
25	import json	×
26	import os	×
27
28	import fs	×
29
30	import click	×
31
32
33	def click_table_printer(headers, _filter, data):	×
34	"""Generate space separated output for click commands."""
35	_filter = [h.lower() for h in _filter] + [h.upper() for h in _filter]	×
36	headers = [h for h in headers if not _filter or h in _filter]	×
37	# Maximum header width
38	header_widths = [len(h) for h in headers]	×
39
40	for row in data:	×
41	for idx in range(len(headers)):	×
42	# If a row contains an element which is wider update maximum width
43	if header_widths[idx] < len(str(row[idx])):	×
44	header_widths[idx] = len(str(row[idx]))	×
45	# Prepare the format string with the maximum widths
46	formatted_output_parts = ['{{:<{0}}}'.format(hw)	×
47	for hw in header_widths]
48	formatted_output = ' '.join(formatted_output_parts)	×
49	# Print the table with the headers capitalized
50	click.echo(formatted_output.format(*[h.upper() for h in headers]))	×
51	for row in data:	×
52	click.echo(formatted_output.format(*row))	×
53
54
55	def get_user_analyses_dir(org, user):	×
56	"""Build the analyses directory path for certain user and organization.
57
58	:param org: Organization which user is part of.
59	:param user: Working directory owner.
60	:return: Path to the user's analyses directory.
61	"""
62	return fs.path.join(org, user, 'analyses')	×
63
64
65	def calculate_hash_of_dir(directory, verbose=0):	×
66	"""Calculate hash of directory."""
67	SHAhash = hashlib.md5()	×
68	if not os.path.exists(directory):	×
69	return -1	×
70
71	try:	×
72	for root, dirs, files in os.walk(directory):	×
73	for names in sorted(files):	×
74	filepath = os.path.join(root, names)	×
75	try:	×
76	f1 = open(filepath, 'rb')	×
77	except Exception:	×
78	# You can't open the file for some reason
79	f1.close()	×
80	# We return -1 since we can not ensure that the file that
81	# can not be read will not change from one execution to
82	# another.
83	return -1	×
84	while 1:	×
85	# Read file in as little chunks
86	buf = f1.read(4096)	×
87	if not buf:	×
88	break	×
89	SHAhash.update(hashlib.md5(buf).hexdigest().encode())	×
90	f1.close()	×
91
92	except Exception:	×
93	return -1	×
94	return SHAhash.hexdigest()	×
95
96
97	def calculate_job_input_hash(job_spec, workflow_json):	×
98	"""Calculate md5 hash of job specification and workflow json."""
99	job_md5_buffer = hashlib.md5()	×
100	job_md5_buffer.update(json.dumps(job_spec).encode('utf-8'))	×
101	job_md5_buffer.update(json.dumps(workflow_json).encode('utf-8'))	×
102	return job_md5_buffer.hexdigest()	×

reanahub / reana-commons / 67

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous