zopefoundation / zodbupdate / 135

pending completion

Build # 135

Build Type

Pull #15

travis-ci

Committed by

web-flow

Commit Message

python 2.7 fix and early exit on error if no fallbacks

Pull Request Pull Request #15: Fallback encodings

Run Details

93 of 200 branches covered (46.5%)

Branch coverage included in aggregate %.

80 of 80 new or added lines in 3 files covered. (100.0%)

768 of 1107 relevant lines covered (69.38%)

2.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

53.75

/src/zodbupdate/main.py

##############################################################################
#
# Copyright (c) 2009 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################

import ZODB.FileStorage
import ZODB.config
import ZODB.serialize
import logging
import argparse
import pkg_resources
import pprint
import six
import time
import zodbupdate.convert
import zodbupdate.update
import zodbupdate.utils

logger = logging.getLogger('zodbupdate')


parser = argparse.ArgumentParser(
    description=("Updates all references to classes to "
                 "their canonical location."))
exclusive_group = parser.add_mutually_exclusive_group()
exclusive_group.add_argument(
    "-f", "--file",
    help="load FileStorage")
exclusive_group.add_argument(
    "-c", "--config",
    help="load storage from config file")
parser.add_argument(
    "-n", "--dry-run", action="store_true",
    help="perform a trial run with no changes made")
parser.add_argument(
    "-s", "--save-renames",
    help="save automatically determined rename rules to file")
parser.add_argument(
    "-q", "--quiet", action="store_true",
    help="suppress non-error messages")
parser.add_argument(
    "-v", "--verbose", action="store_true",
    help="more verbose output")
parser.add_argument(
    "-o", "--oid",
    help="start with provided oid in hex format, ex: 0xaa1203")
parser.add_argument(
    "-d", "--debug", action="store_true",
    help="post mortem pdb on failure")
parser.add_argument(
    "--pack", action="store_true", dest="pack",
    help=("pack the storage when done. use in conjunction of -c "
          "if you have blobs storage"))
parser.add_argument(
    "--convert-py3", action="store_true", dest="convert_py3",
    help="convert pickle format to protocol 3 and adjust bytes")
parser.add_argument(
    '--encoding', dest="encoding",
    help="used for decoding pickled strings in py3"
)
parser.add_argument(
    '--encoding-fallback', dest="encoding_fallbacks", nargs="*",
    help="Older databases may have other encoding stored than 'utf-8', like latin1."
         "If an encoding error occurs, fallback to the given encodings "
         "and issue a warning.",
)


class DuplicateFilter(object):

    def __init__(self):
        self.reset()

    def filter(self, record):
        if record.msg in self.seen:
            return False
        self.seen.add(record.msg)
        return True

    def reset(self):
        self.seen = set()


duplicate_filter = DuplicateFilter()


def setup_logger(verbose=False, quiet=False, handler=None):
    logging.getLogger('zodbupdate.serialize').addFilter(duplicate_filter)
    if quiet:
        level = logging.ERROR
    elif verbose:
        level = logging.DEBUG
    else:
        level = logging.INFO
    if handler is None:
        handler = logging.StreamHandler()
    logger = logging.getLogger()
    logger.setLevel(level)
    logger.addHandler(handler)
    return logger


def load_renames():
    renames = {}
    for entry_point in pkg_resources.iter_entry_points('zodbupdate'):
        definition = entry_point.load()
        for old, new in definition.items():
            renames[tuple(old.split(' '))] = tuple(new.split(' '))
        logger.info('Loaded {} rename rules from {}:{}'.format(
            len(definition), entry_point.module_name, entry_point.name))
    return renames


def create_updater(
        storage,
        default_renames=None,
        default_decoders=None,
        start_at=None,
        convert_py3=False,
        encoding=None,
        encoding_fallbacks=None,
        dry_run=False,
        debug=False):
    if not start_at:
        start_at = '0x00'

    if six.PY2 and encoding:
        raise AssertionError(
            'Unpickling with a default encoding is only supported in Python 3.'
        )

    decoders = {}
    if default_decoders:
        decoders.update(default_decoders)
    renames = load_renames()
    if default_renames:
        renames.update(default_renames)
    repickle_all = False
    pickle_protocol = zodbupdate.utils.DEFAULT_PROTOCOL
    if convert_py3:
        pickle_protocol = 3
        repickle_all = True
        decoders.update(
            zodbupdate.convert.load_decoders(
                encoding_fallbacks=encoding_fallbacks
            )
        )
        renames.update(zodbupdate.convert.default_renames())

    return zodbupdate.update.Updater(
        storage,
        dry=dry_run,
        renames=renames,
        decoders=decoders,
        start_at=start_at,
        debug=debug,
        repickle_all=repickle_all,
        pickle_protocol=pickle_protocol,
        encoding=encoding,
    )


def format_renames(renames):
    formatted = {}
    for old, new in renames.items():
        formatted[' '.join(old)] = ' '.join(new)
    if not formatted:
        return ''
    return pprint.pformat(formatted)


def main():
    args = parser.parse_args()

    setup_logger(quiet=args.quiet, verbose=args.verbose)

    if args.file and args.config:
        raise AssertionError(
            'Exactly one of --file or --config must be given.')

    # Magic bytes need to be converted at the end when running in Python 2
    # but at the beginning when running in Python 3 so that FileStorage
    # doesn't complain.
    if args.convert_py3 and six.PY3 and not args.dry_run:
        zodbupdate.convert.update_magic_data_fs(args.file)

    if args.file:
        storage = ZODB.FileStorage.FileStorage(args.file)
    elif args.config:
        with open(args.config) as config:
            storage = ZODB.config.storageFromFile(config)
    else:
        raise AssertionError(
            'Exactly one of --file or --config must be given.')

    updater = create_updater(
        storage,
        start_at=args.oid,
        convert_py3=args.convert_py3,
        encoding=args.encoding,
        encoding_fallbacks=args.encoding_fallbacks,
        dry_run=args.dry_run,
        debug=args.debug)
    try:
        updater()
    except Exception as error:
        logging.info('An error occured', exc_info=True)
        logging.error('Stopped processing, due to: {}'.format(error))
        raise AssertionError()

    implicit_renames = format_renames(
        updater.processor.get_rules(implicit=True))
    if implicit_renames:
        logger.info('Found new rules: {}'.format(implicit_renames))
    if args.save_renames:
        logger.info('Saving rules into {}'.format(args.save_renames))
        with open(args.save_renames, 'w') as output:
            output.write('renames = {}'.format(
                format_renames(updater.processor.get_rules(
                    implicit=True, explicit=True))))
    if args.pack:
        logger.info('Packing storage ...')
        storage.pack(time.time(), ZODB.serialize.referencesf)
    storage.close()

    if args.convert_py3 and six.PY2 and not args.dry_run:
        zodbupdate.convert.update_magic_data_fs(args.file)

1	##############################################################################
2	#
3	# Copyright (c) 2009 Zope Corporation and Contributors.
4	# All Rights Reserved.
5	#
6	# This software is subject to the provisions of the Zope Public License,
7	# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
8	# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
9	# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
10	# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
11	# FOR A PARTICULAR PURPOSE.
12	#
13	##############################################################################
14
15	import ZODB.FileStorage	4✔
16	import ZODB.config	4✔
17	import ZODB.serialize	4✔
18	import logging	4✔
19	import argparse	4✔
20	import pkg_resources	4✔
21	import pprint	4✔
22	import six	4✔
23	import time	4✔
24	import zodbupdate.convert	4✔
25	import zodbupdate.update	4✔
26	import zodbupdate.utils	4✔
27
28	logger = logging.getLogger('zodbupdate')	4✔
29
30
31	parser = argparse.ArgumentParser(	4✔
32	description=("Updates all references to classes to "
33	"their canonical location."))
34	exclusive_group = parser.add_mutually_exclusive_group()	4✔
35	exclusive_group.add_argument(	4✔
36	"-f", "--file",
37	help="load FileStorage")
38	exclusive_group.add_argument(	4✔
39	"-c", "--config",
40	help="load storage from config file")
41	parser.add_argument(	4✔
42	"-n", "--dry-run", action="store_true",
43	help="perform a trial run with no changes made")
44	parser.add_argument(	4✔
45	"-s", "--save-renames",
46	help="save automatically determined rename rules to file")
47	parser.add_argument(	4✔
48	"-q", "--quiet", action="store_true",
49	help="suppress non-error messages")
50	parser.add_argument(	4✔
51	"-v", "--verbose", action="store_true",
52	help="more verbose output")
53	parser.add_argument(	4✔
54	"-o", "--oid",
55	help="start with provided oid in hex format, ex: 0xaa1203")
56	parser.add_argument(	4✔
57	"-d", "--debug", action="store_true",
58	help="post mortem pdb on failure")
59	parser.add_argument(	4✔
60	"--pack", action="store_true", dest="pack",
61	help=("pack the storage when done. use in conjunction of -c "
62	"if you have blobs storage"))
63	parser.add_argument(	4✔
64	"--convert-py3", action="store_true", dest="convert_py3",
65	help="convert pickle format to protocol 3 and adjust bytes")
66	parser.add_argument(	4✔
67	'--encoding', dest="encoding",
68	help="used for decoding pickled strings in py3"
69	)
70	parser.add_argument(	4✔
71	'--encoding-fallback', dest="encoding_fallbacks", nargs="*",
72	help="Older databases may have other encoding stored than 'utf-8', like latin1."
73	"If an encoding error occurs, fallback to the given encodings "
74	"and issue a warning.",
75	)
76
77
78	class DuplicateFilter(object):	4✔
79
80	def __init__(self):	4✔
81	self.reset()	4✔
82
83	def filter(self, record):	4✔
84	if record.msg in self.seen:	4✔
85	return False	4✔
86	self.seen.add(record.msg)	4✔
87	return True	4✔
88
89	def reset(self):	4✔
90	self.seen = set()	4✔
91
92
93	duplicate_filter = DuplicateFilter()	4✔
94
95
96	def setup_logger(verbose=False, quiet=False, handler=None):	4✔
97	logging.getLogger('zodbupdate.serialize').addFilter(duplicate_filter)	4✔
98	if quiet:	4!
99	level = logging.ERROR	×
100	elif verbose:	4!
101	level = logging.DEBUG	×
102	else:
103	level = logging.INFO	4✔
104	if handler is None:	4!
105	handler = logging.StreamHandler()	×
106	logger = logging.getLogger()	4✔
107	logger.setLevel(level)	4✔
108	logger.addHandler(handler)	4✔
109	return logger	4✔
110
111
112	def load_renames():	4✔
113	renames = {}	4✔
114	for entry_point in pkg_resources.iter_entry_points('zodbupdate'):	4!
115	definition = entry_point.load()	×
116	for old, new in definition.items():	×
117	renames[tuple(old.split(' '))] = tuple(new.split(' '))	×
118	logger.info('Loaded {} rename rules from {}:{}'.format(	×
119	len(definition), entry_point.module_name, entry_point.name))
120	return renames	4✔
121
122
123	def create_updater(	4✔
124	storage,
125	default_renames=None,
126	default_decoders=None,
127	start_at=None,
128	convert_py3=False,
129	encoding=None,
130	encoding_fallbacks=None,
131	dry_run=False,
132	debug=False):
133	if not start_at:	4!
134	start_at = '0x00'	4✔
135
136	if six.PY2 and encoding:	4!
137	raise AssertionError(	×
138	'Unpickling with a default encoding is only supported in Python 3.'
139	)
140
141	decoders = {}	4✔
142	if default_decoders:	4✔
143	decoders.update(default_decoders)	4✔
144	renames = load_renames()	4✔
145	if default_renames:	4✔
146	renames.update(default_renames)	4✔
147	repickle_all = False	4✔
148	pickle_protocol = zodbupdate.utils.DEFAULT_PROTOCOL	4✔
149	if convert_py3:	4✔
150	pickle_protocol = 3	4✔
151	repickle_all = True	4✔
152	decoders.update(	4✔
153	zodbupdate.convert.load_decoders(
154	encoding_fallbacks=encoding_fallbacks
155	)
156	)
157	renames.update(zodbupdate.convert.default_renames())	4✔
158
159	return zodbupdate.update.Updater(	4✔
160	storage,
161	dry=dry_run,
162	renames=renames,
163	decoders=decoders,
164	start_at=start_at,
165	debug=debug,
166	repickle_all=repickle_all,
167	pickle_protocol=pickle_protocol,
168	encoding=encoding,
169	)
170
171
172	def format_renames(renames):	4✔
173	formatted = {}	×
174	for old, new in renames.items():	×
175	formatted[' '.join(old)] = ' '.join(new)	×
176	if not formatted:	×
177	return ''	×
178	return pprint.pformat(formatted)	×
179
180
181	def main():	4✔
182	args = parser.parse_args()	×
183
184	setup_logger(quiet=args.quiet, verbose=args.verbose)	×
185
186	if args.file and args.config:	×
187	raise AssertionError(	×
188	'Exactly one of --file or --config must be given.')
189
190	# Magic bytes need to be converted at the end when running in Python 2
191	# but at the beginning when running in Python 3 so that FileStorage
192	# doesn't complain.
193	if args.convert_py3 and six.PY3 and not args.dry_run:	×
194	zodbupdate.convert.update_magic_data_fs(args.file)	×
195
196	if args.file:	×
197	storage = ZODB.FileStorage.FileStorage(args.file)	×
198	elif args.config:	×
199	with open(args.config) as config:	×
200	storage = ZODB.config.storageFromFile(config)	×
201	else:
202	raise AssertionError(	×
203	'Exactly one of --file or --config must be given.')
204
205	updater = create_updater(	×
206	storage,
207	start_at=args.oid,
208	convert_py3=args.convert_py3,
209	encoding=args.encoding,
210	encoding_fallbacks=args.encoding_fallbacks,
211	dry_run=args.dry_run,
212	debug=args.debug)
213	try:	×
214	updater()	×
215	except Exception as error:	×
216	logging.info('An error occured', exc_info=True)	×
217	logging.error('Stopped processing, due to: {}'.format(error))	×
218	raise AssertionError()	×
219
220	implicit_renames = format_renames(	×
221	updater.processor.get_rules(implicit=True))
222	if implicit_renames:	×
223	logger.info('Found new rules: {}'.format(implicit_renames))	×
224	if args.save_renames:	×
225	logger.info('Saving rules into {}'.format(args.save_renames))	×
226	with open(args.save_renames, 'w') as output:	×
227	output.write('renames = {}'.format(	×
228	format_renames(updater.processor.get_rules(
229	implicit=True, explicit=True))))
230	if args.pack:	×
231	logger.info('Packing storage ...')	×
232	storage.pack(time.time(), ZODB.serialize.referencesf)	×
233	storage.close()	×
234
235	if args.convert_py3 and six.PY2 and not args.dry_run:	×
236	zodbupdate.convert.update_magic_data_fs(args.file)	×

zopefoundation / zodbupdate / 135

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous