IDSIA / sacred / 785

pending completion

Build # 785

Build Type

push

travis-ci

Committed by

JarnoRFB

Commit Message

<a href="https://github.com/IDSIA/sacred/commit/<a class=hub.com/IDSIA/sacred/commit/<a class="double-link" href="https://git"><a class=hub.com/IDSIA/sacred/commit/4c79484902b5cc03ff1f222594574b746acf18e1">4c7948490<a href="https://github.com/IDSIA/sacred/commit/4c79484902b5cc03ff1f222594574b746acf18e1">&lt;a href=&quot;https://github.com/IDSIA/sacred/commit/4c79484902b5cc03ff1f222594574b746acf18e1&quot;&gt;&amp;quot;&amp;gt;Some more attributes are Paths now. (#558)

* Added pathlib to more stuff.

* Some more attributes use pathlib.

* Fixed json serialization issue;

* Fix pep8

* Update sacred/observers/file_storage.py

Co-Authored-By: Rüdiger Busche &amp;amp;lt;ruedigerbusche@web.de&amp;amp;gt;

* Update sacred/observers/file_storage.py

Co-Authored-By: Rüdiger Busche &amp;lt;ruedigerbusche@web.de&amp;gt;

* Revert &amp;quot;Update sacred/observers/file_storage.py&amp;quot;

This reverts commit &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/IDSIA/sacred/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/IDSIA/sacred/commit/73a8d0ef7bff9e8b3d06c1d0f3da869e01807c07&quot;&gt;73a8d0ef7&lt;/a&gt;&quot;>73a8d0ef7</a><a href="https://github.com/IDSIA/sacred/commit/4c79484902b5cc03ff1f222594574b746acf18e1">.

* Revert &quot;Update sacred/observers/file_storage.py&quot;

This reverts commit </a><a class="double-link" href="https://github.com/IDSIA/sacred/commit/<a class="double-link" href="https://github.com/IDSIA/sacred/commit/7d3f263b80be33d5226320bd85c8e6b7a3d596e3">7d3f263b8">7d3f263b8.

Run Details

12 of 12 new or added lines in 1 file covered. (100.0%)

3222 of 3777 relevant lines covered (85.31%)

0.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.37

/sacred/observers/file_storage.py

#!/usr/bin/env python
# coding=utf-8

import json
import os
import os.path
from pathlib import Path
from typing import Optional

from shutil import copyfile

from sacred.commandline_options import CommandLineOption
from sacred.dependencies import get_digest
from sacred.observers.base import RunObserver
from sacred import optional as opt
from sacred.serializer import flatten
from sacred.utils import PathType


DEFAULT_FILE_STORAGE_PRIORITY = 20


class FileStorageObserver(RunObserver):
    VERSION = 'FileStorageObserver-0.7.0'

    @classmethod
    def create(cls, basedir: PathType,
               resource_dir: Optional[PathType] = None,
               source_dir: Optional[PathType] = None,
               template: Optional[PathType] = None,
               priority: int = DEFAULT_FILE_STORAGE_PRIORITY):
        basedir = Path(basedir)
        resource_dir = resource_dir or basedir / '_resources'
        source_dir = source_dir or basedir / '_sources'
        if template is not None:
            if not os.path.exists(template):
                raise FileNotFoundError("Couldn't find template file '{}'"
                                        .format(template))
        else:
            template = basedir / 'template.html'
            if not template.exists():
                template = None
        return cls(basedir, resource_dir, source_dir, template, priority)

    def __init__(self, basedir, resource_dir, source_dir, template,
                 priority=DEFAULT_FILE_STORAGE_PRIORITY):
        self.basedir = str(basedir)
        self.resource_dir = resource_dir
        self.source_dir = source_dir
        self.template = template
        self.priority = priority
        self.dir = None
        self.run_entry = None
        self.config = None
        self.info = None
        self.cout = ""
        self.cout_write_cursor = 0

    def _maximum_existing_run_id(self):
        dir_nrs = [int(d) for d in os.listdir(self.basedir)
                   if os.path.isdir(os.path.join(self.basedir, d)) and
                   d.isdigit()]
        if dir_nrs:
            return max(dir_nrs)
        else:
            return 0

    def _make_dir(self, _id):
        new_dir = os.path.join(self.basedir, str(_id))
        os.mkdir(new_dir)
        self.dir = new_dir  # set only if mkdir is successful

    def _make_run_dir(self, _id):
        os.makedirs(self.basedir, exist_ok=True)
        self.dir = None
        if _id is None:
            fail_count = 0
            _id = self._maximum_existing_run_id() + 1
            while self.dir is None:
                try:
                    self._make_dir(_id)
                except FileExistsError:  # Catch race conditions
                    if fail_count < 1000:
                        fail_count += 1
                        _id += 1
                    else:  # expect that something else went wrong
                        raise
        else:
            self.dir = os.path.join(self.basedir, str(_id))
            os.mkdir(self.dir)

    def queued_event(self, ex_info, command, host_info, queue_time, config,
                     meta_info, _id):
        self._make_run_dir(_id)

        self.run_entry = {
            'experiment': dict(ex_info),
            'command': command,
            'host': dict(host_info),
            'meta': meta_info,
            'status': 'QUEUED',
        }
        self.config = config
        self.info = {}

        self.save_json(self.run_entry, 'run.json')
        self.save_json(self.config, 'config.json')

        for s, m in ex_info['sources']:
            self.save_file(s)

        return os.path.relpath(self.dir, self.basedir) if _id is None else _id

    def save_sources(self, ex_info):
        base_dir = ex_info['base_dir']
        source_info = []
        for s, m in ex_info['sources']:
            abspath = os.path.join(base_dir, s)
            store_path, md5sum = self.find_or_save(abspath, self.source_dir)
            # assert m == md5sum
            relative_source = os.path.relpath(str(store_path), self.basedir)
            source_info.append([s, relative_source])
        return source_info

    def started_event(self, ex_info, command, host_info, start_time, config,
                      meta_info, _id):
        self._make_run_dir(_id)

        ex_info['sources'] = self.save_sources(ex_info)

        self.run_entry = {
            'experiment': dict(ex_info),
            'command': command,
            'host': dict(host_info),
            'start_time': start_time.isoformat(),
            'meta': meta_info,
            'status': 'RUNNING',
            'resources': [],
            'artifacts': [],
            'heartbeat': None
        }
        self.config = config
        self.info = {}
        self.cout = ""
        self.cout_write_cursor = 0

        self.save_json(self.run_entry, 'run.json')
        self.save_json(self.config, 'config.json')
        self.save_cout()

        return os.path.relpath(self.dir, self.basedir) if _id is None else _id

    def find_or_save(self, filename, store_dir: Path):
        os.makedirs(str(store_dir), exist_ok=True)
        source_name, ext = os.path.splitext(os.path.basename(filename))
        md5sum = get_digest(filename)
        store_name = source_name + '_' + md5sum + ext
        store_path = store_dir / store_name
        if not store_path.exists():
            copyfile(filename, str(store_path))
        return store_path, md5sum

    def save_json(self, obj, filename):
        with open(os.path.join(self.dir, filename), 'w') as f:
            json.dump(flatten(obj), f, sort_keys=True, indent=2)

    def save_file(self, filename, target_name=None):
        target_name = target_name or os.path.basename(filename)
        copyfile(filename, os.path.join(self.dir, target_name))

    def save_cout(self):
        with open(os.path.join(self.dir, 'cout.txt'), 'ab') as f:
            f.write(self.cout[self.cout_write_cursor:].encode("utf-8"))
            self.cout_write_cursor = len(self.cout)

    def render_template(self):
        if opt.has_mako and self.template:
            from mako.template import Template
            template = Template(filename=self.template)
            report = template.render(run=self.run_entry,
                                     config=self.config,
                                     info=self.info,
                                     cout=self.cout,
                                     savedir=self.dir)
            ext = self.template.suffix
            with open(os.path.join(self.dir, 'report' + ext), 'w') as f:
                f.write(report)

    def heartbeat_event(self, info, captured_out, beat_time, result):
        self.info = info
        self.run_entry['heartbeat'] = beat_time.isoformat()
        self.run_entry['result'] = result
        self.cout = captured_out
        self.save_cout()
        self.save_json(self.run_entry, 'run.json')
        if self.info:
            self.save_json(self.info, 'info.json')

    def completed_event(self, stop_time, result):
        self.run_entry['stop_time'] = stop_time.isoformat()
        self.run_entry['result'] = result
        self.run_entry['status'] = 'COMPLETED'

        self.save_json(self.run_entry, 'run.json')
        self.render_template()

    def interrupted_event(self, interrupt_time, status):
        self.run_entry['stop_time'] = interrupt_time.isoformat()
        self.run_entry['status'] = status
        self.save_json(self.run_entry, 'run.json')
        self.render_template()

    def failed_event(self, fail_time, fail_trace):
        self.run_entry['stop_time'] = fail_time.isoformat()
        self.run_entry['status'] = 'FAILED'
        self.run_entry['fail_trace'] = fail_trace
        self.save_json(self.run_entry, 'run.json')
        self.render_template()

    def resource_event(self, filename):
        store_path, md5sum = self.find_or_save(filename, self.resource_dir)
        self.run_entry['resources'].append([filename, str(store_path)])
        self.save_json(self.run_entry, 'run.json')

    def artifact_event(self, name, filename, metadata=None, content_type=None):
        self.save_file(filename, name)
        self.run_entry['artifacts'].append(name)
        self.save_json(self.run_entry, 'run.json')

    def log_metrics(self, metrics_by_name, info):
        """Store new measurements into metrics.json.
        """
        try:
            metrics_path = os.path.join(self.dir, "metrics.json")
            with open(metrics_path, 'r') as f:
                saved_metrics = json.load(f)
        except IOError:
            # We haven't recorded anything yet. Start Collecting.
            saved_metrics = {}

        for metric_name, metric_ptr in metrics_by_name.items():

            if metric_name not in saved_metrics:
                saved_metrics[metric_name] = {"values": [],
                                              "steps": [],
                                              "timestamps": []}

            saved_metrics[metric_name]["values"] += metric_ptr["values"]
            saved_metrics[metric_name]["steps"] += metric_ptr["steps"]

            # Manually convert them to avoid passing a datetime dtype handler
            # when we're trying to convert into json.
            timestamps_norm = [ts.isoformat()
                               for ts in metric_ptr["timestamps"]]
            saved_metrics[metric_name]["timestamps"] += timestamps_norm

        self.save_json(saved_metrics, 'metrics.json')

    def __eq__(self, other):
        if isinstance(other, FileStorageObserver):
            return self.basedir == other.basedir
        return False


class FileStorageOption(CommandLineOption):
    """Add a file-storage observer to the experiment."""

    short_flag = 'F'
    arg = 'BASEDIR'
    arg_description = "Base-directory to write the runs to"

    @classmethod
    def apply(cls, args, run):
        run.observers.append(FileStorageObserver.create(args))

1	#!/usr/bin/env python
2	# coding=utf-8
3
4	import json	1✔
5	import os	1✔
6	import os.path	1✔
7	from pathlib import Path	1✔
8	from typing import Optional	1✔
9
10	from shutil import copyfile	1✔
11
12	from sacred.commandline_options import CommandLineOption	1✔
13	from sacred.dependencies import get_digest	1✔
14	from sacred.observers.base import RunObserver	1✔
15	from sacred import optional as opt	1✔
16	from sacred.serializer import flatten	1✔
17	from sacred.utils import PathType	1✔
18
19
20	DEFAULT_FILE_STORAGE_PRIORITY = 20	1✔
21
22
23	class FileStorageObserver(RunObserver):	1✔
24	VERSION = 'FileStorageObserver-0.7.0'	1✔
25
26	@classmethod	1✔
27	def create(cls, basedir: PathType,	1✔
28	resource_dir: Optional[PathType] = None,
29	source_dir: Optional[PathType] = None,
30	template: Optional[PathType] = None,
31	priority: int = DEFAULT_FILE_STORAGE_PRIORITY):
32	basedir = Path(basedir)	1✔
33	resource_dir = resource_dir or basedir / '_resources'	1✔
34	source_dir = source_dir or basedir / '_sources'	1✔
35	if template is not None:	1✔
36	if not os.path.exists(template):	×
37	raise FileNotFoundError("Couldn't find template file '{}'"	×
38	.format(template))
39	else:
40	template = basedir / 'template.html'	1✔
41	if not template.exists():	1✔
42	template = None	1✔
43	return cls(basedir, resource_dir, source_dir, template, priority)	1✔
44
45	def __init__(self, basedir, resource_dir, source_dir, template,	1✔
46	priority=DEFAULT_FILE_STORAGE_PRIORITY):
47	self.basedir = str(basedir)	1✔
48	self.resource_dir = resource_dir	1✔
49	self.source_dir = source_dir	1✔
50	self.template = template	1✔
51	self.priority = priority	1✔
52	self.dir = None	1✔
53	self.run_entry = None	1✔
54	self.config = None	1✔
55	self.info = None	1✔
56	self.cout = ""	1✔
57	self.cout_write_cursor = 0	1✔
58
59	def _maximum_existing_run_id(self):	1✔
60	dir_nrs = [int(d) for d in os.listdir(self.basedir)	1✔
61	if os.path.isdir(os.path.join(self.basedir, d)) and
62	d.isdigit()]
63	if dir_nrs:	1✔
64	return max(dir_nrs)	×
65	else:
66	return 0	1✔
67
68	def _make_dir(self, _id):	1✔
69	new_dir = os.path.join(self.basedir, str(_id))	1✔
70	os.mkdir(new_dir)	1✔
71	self.dir = new_dir # set only if mkdir is successful	1✔
72
73	def _make_run_dir(self, _id):	1✔
74	os.makedirs(self.basedir, exist_ok=True)	1✔
75	self.dir = None	1✔
76	if _id is None:	1✔
77	fail_count = 0	1✔
78	_id = self._maximum_existing_run_id() + 1	1✔
79	while self.dir is None:	1✔
80	try:	1✔
81	self._make_dir(_id)	1✔
82	except FileExistsError: # Catch race conditions	1✔
83	if fail_count < 1000:	1✔
84	fail_count += 1	1✔
85	_id += 1	1✔
86	else: # expect that something else went wrong
87	raise	×
88	else:
89	self.dir = os.path.join(self.basedir, str(_id))	1✔
90	os.mkdir(self.dir)	1✔
91
92	def queued_event(self, ex_info, command, host_info, queue_time, config,	1✔
93	meta_info, _id):
94	self._make_run_dir(_id)	1✔
95
96	self.run_entry = {	1✔
97	'experiment': dict(ex_info),
98	'command': command,
99	'host': dict(host_info),
100	'meta': meta_info,
101	'status': 'QUEUED',
102	}
103	self.config = config	1✔
104	self.info = {}	1✔
105
106	self.save_json(self.run_entry, 'run.json')	1✔
107	self.save_json(self.config, 'config.json')	1✔
108
109	for s, m in ex_info['sources']:	1✔
110	self.save_file(s)	×
111
112	return os.path.relpath(self.dir, self.basedir) if _id is None else _id	1✔
113
114	def save_sources(self, ex_info):	1✔
115	base_dir = ex_info['base_dir']	1✔
116	source_info = []	1✔
117	for s, m in ex_info['sources']:	1✔
118	abspath = os.path.join(base_dir, s)	1✔
119	store_path, md5sum = self.find_or_save(abspath, self.source_dir)	1✔
120	# assert m == md5sum
121	relative_source = os.path.relpath(str(store_path), self.basedir)	1✔
122	source_info.append([s, relative_source])	1✔
123	return source_info	1✔
124
125	def started_event(self, ex_info, command, host_info, start_time, config,	1✔
126	meta_info, _id):
127	self._make_run_dir(_id)	1✔
128
129	ex_info['sources'] = self.save_sources(ex_info)	1✔
130
131	self.run_entry = {	1✔
132	'experiment': dict(ex_info),
133	'command': command,
134	'host': dict(host_info),
135	'start_time': start_time.isoformat(),
136	'meta': meta_info,
137	'status': 'RUNNING',
138	'resources': [],
139	'artifacts': [],
140	'heartbeat': None
141	}
142	self.config = config	1✔
143	self.info = {}	1✔
144	self.cout = ""	1✔
145	self.cout_write_cursor = 0	1✔
146
147	self.save_json(self.run_entry, 'run.json')	1✔
148	self.save_json(self.config, 'config.json')	1✔
149	self.save_cout()	1✔
150
151	return os.path.relpath(self.dir, self.basedir) if _id is None else _id	1✔
152
153	def find_or_save(self, filename, store_dir: Path):	1✔
154	os.makedirs(str(store_dir), exist_ok=True)	1✔
155	source_name, ext = os.path.splitext(os.path.basename(filename))	1✔
156	md5sum = get_digest(filename)	1✔
157	store_name = source_name + '_' + md5sum + ext	1✔
158	store_path = store_dir / store_name	1✔
159	if not store_path.exists():	1✔
160	copyfile(filename, str(store_path))	1✔
161	return store_path, md5sum	1✔
162
163	def save_json(self, obj, filename):	1✔
164	with open(os.path.join(self.dir, filename), 'w') as f:	1✔
165	json.dump(flatten(obj), f, sort_keys=True, indent=2)	1✔
166
167	def save_file(self, filename, target_name=None):	1✔
168	target_name = target_name or os.path.basename(filename)	1✔
169	copyfile(filename, os.path.join(self.dir, target_name))	1✔
170
171	def save_cout(self):	1✔
172	with open(os.path.join(self.dir, 'cout.txt'), 'ab') as f:	1✔
173	f.write(self.cout[self.cout_write_cursor:].encode("utf-8"))	1✔
174	self.cout_write_cursor = len(self.cout)	1✔
175
176	def render_template(self):	1✔
177	if opt.has_mako and self.template:	1✔
178	from mako.template import Template	×
179	template = Template(filename=self.template)	×
180	report = template.render(run=self.run_entry,	×
181	config=self.config,
182	info=self.info,
183	cout=self.cout,
184	savedir=self.dir)
185	ext = self.template.suffix	×
186	with open(os.path.join(self.dir, 'report' + ext), 'w') as f:	×
187	f.write(report)	×
188
189	def heartbeat_event(self, info, captured_out, beat_time, result):	1✔
190	self.info = info	1✔
191	self.run_entry['heartbeat'] = beat_time.isoformat()	1✔
192	self.run_entry['result'] = result	1✔
193	self.cout = captured_out	1✔
194	self.save_cout()	1✔
195	self.save_json(self.run_entry, 'run.json')	1✔
196	if self.info:	1✔
197	self.save_json(self.info, 'info.json')	1✔
198
199	def completed_event(self, stop_time, result):	1✔
200	self.run_entry['stop_time'] = stop_time.isoformat()	1✔
201	self.run_entry['result'] = result	1✔
202	self.run_entry['status'] = 'COMPLETED'	1✔
203
204	self.save_json(self.run_entry, 'run.json')	1✔
205	self.render_template()	1✔
206
207	def interrupted_event(self, interrupt_time, status):	1✔
208	self.run_entry['stop_time'] = interrupt_time.isoformat()	1✔
209	self.run_entry['status'] = status	1✔
210	self.save_json(self.run_entry, 'run.json')	1✔
211	self.render_template()	1✔
212
213	def failed_event(self, fail_time, fail_trace):	1✔
214	self.run_entry['stop_time'] = fail_time.isoformat()	1✔
215	self.run_entry['status'] = 'FAILED'	1✔
216	self.run_entry['fail_trace'] = fail_trace	1✔
217	self.save_json(self.run_entry, 'run.json')	1✔
218	self.render_template()	1✔
219
220	def resource_event(self, filename):	1✔
221	store_path, md5sum = self.find_or_save(filename, self.resource_dir)	1✔
222	self.run_entry['resources'].append([filename, str(store_path)])	1✔
223	self.save_json(self.run_entry, 'run.json')	1✔
224
225	def artifact_event(self, name, filename, metadata=None, content_type=None):	1✔
226	self.save_file(filename, name)	1✔
227	self.run_entry['artifacts'].append(name)	1✔
228	self.save_json(self.run_entry, 'run.json')	1✔
229
230	def log_metrics(self, metrics_by_name, info):	1✔
231	"""Store new measurements into metrics.json.
232	"""
233	try:	1✔
234	metrics_path = os.path.join(self.dir, "metrics.json")	1✔
235	with open(metrics_path, 'r') as f:	1✔
236	saved_metrics = json.load(f)	1✔
237	except IOError:	1✔
238	# We haven't recorded anything yet. Start Collecting.
239	saved_metrics = {}	1✔
240
241	for metric_name, metric_ptr in metrics_by_name.items():	1✔
242
243	if metric_name not in saved_metrics:	1✔
244	saved_metrics[metric_name] = {"values": [],	1✔
245	"steps": [],
246	"timestamps": []}
247
248	saved_metrics[metric_name]["values"] += metric_ptr["values"]	1✔
249	saved_metrics[metric_name]["steps"] += metric_ptr["steps"]	1✔
250
251	# Manually convert them to avoid passing a datetime dtype handler
252	# when we're trying to convert into json.
253	timestamps_norm = [ts.isoformat()	1✔
254	for ts in metric_ptr["timestamps"]]
255	saved_metrics[metric_name]["timestamps"] += timestamps_norm	1✔
256
257	self.save_json(saved_metrics, 'metrics.json')	1✔
258
259	def __eq__(self, other):	1✔
260	if isinstance(other, FileStorageObserver):	1✔
261	return self.basedir == other.basedir	1✔
262	return False	1✔
263
264
265	class FileStorageOption(CommandLineOption):	1✔
266	"""Add a file-storage observer to the experiment."""
267
268	short_flag = 'F'	1✔
269	arg = 'BASEDIR'	1✔
270	arg_description = "Base-directory to write the runs to"	1✔
271
272	@classmethod	1✔
273	def apply(cls, args, run):
274	run.observers.append(FileStorageObserver.create(args))	×

IDSIA / sacred / 785

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous