zopefoundation / zodbupdate / 111

##############################################################################
#
# Copyright (c) 2009-2010 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################

import contextlib
import io
import logging
import types
import sys
import six
import zodbpickle

from ZODB.broken import find_global, Broken, rebuild
from zodbupdate import utils

logger = logging.getLogger('zodbupdate.serialize')
known_broken_modules = {}

# types to skip when renaming/migrating databases
SKIP_SYMBS = [('ZODB.blob', 'Blob')]


def create_broken_module_for(symb):
    """If your pickle refer a broken class (not an instance of it, a
       reference to the class symbol itself) you have no choice than
       having this module available in the same symbol and with the
       same name, otherwise repickling doesn't work (as both pickle
       and cPickle __import__ the module, and verify the class symbol
       is the same than the one provided).
    """
    parts = symb.__module__.split('.')
    previous = None
    for fullname, name in reversed(
            [('.'.join(parts[0:p + 1]), parts[p])
             for p in range(0, len(parts))]):
        if fullname not in sys.modules:
            if fullname not in known_broken_modules:
                module = types.ModuleType(fullname)
                module.__name__ = name
                module.__file__ = '<broken module to pickle class reference>'
                module.__path__ = []
                known_broken_modules[fullname] = module
            else:
                if previous:
                    module = known_broken_modules[fullname]
                    setattr(module, *previous)
                break
            if previous:
                setattr(module, *previous)
            previous = (name, module)
        else:
            if previous:
                setattr(sys.modules[fullname], *previous)
                break
    if symb.__module__ in known_broken_modules:
        setattr(known_broken_modules[symb.__module__], symb.__name__, symb)
    elif symb.__module__ in sys.modules:
        setattr(sys.modules[symb.__module__], symb.__name__, symb)


class BrokenModuleFinder(object):
    """This broken module finder works with create_broken_module_for.
    """

    def load_module(self, fullname):
        module = known_broken_modules[fullname]
        if fullname not in sys.modules:
            sys.modules[fullname] = module
        module.__loader__ = self
        return module

    def find_module(self, fullname, path=None):
        if fullname in known_broken_modules:
            return self
        return None


sys.meta_path.append(BrokenModuleFinder())


class NullIterator(six.Iterator):
    """An empty iterator that doesn't gives any result.
    """

    def __iter__(self):
        return self

    def __next__(self):
        raise StopIteration()


class IterableClass(type):

    def __iter__(cls):
        """Define a empty iterator to fix unpickling of missing
        Interfaces that have been used to do alsoProvides on a another
        pickled object.
        """
        return NullIterator()


@six.add_metaclass(IterableClass)
class ZODBBroken(Broken):
    """Extend ZODB Broken to work with broken objects that doesn't
    have any __Broken_newargs__ sets (which happens if their __new__
    method is not called).
    """

    def __reduce__(self):
        """We pickle broken objects in hope of being able to fix them later.
        """
        return (rebuild,
                ((self.__class__.__module__, self.__class__.__name__)
                 + getattr(self, '__Broken_newargs__', tuple())),
                self.__Broken_state__)


class ZODBReference(object):
    """Class to remenber reference we don't want to touch.
    """

    def __init__(self, ref):
        self.ref = ref


class ObjectRenamer(object):
    """This load and save a ZODB record, modifying all references to
    renamed class according the given renaming rules:

    - in global symbols contained in the record,

    - in persistent reference information,

    - in class information (first pickle of the record).
    """

    def __init__(
            self, renames, decoders, pickle_protocol=3, repickle_all=False,
            encoding=None):
        self.__added = dict()
        self.__renames = renames
        self.__decoders = decoders
        self.__changed = False
        self.__protocol = pickle_protocol
        self.__repickle_all = repickle_all
        self.__encoding = encoding
        self.__unpickle_options = {}
        if encoding:
            self.__unpickle_options = {
                'encoding': encoding,
                'errors': 'bytes',
            }

    def __update_symb(self, symb_info):
        """This method look in a klass or symbol have been renamed or
        not. If the symbol have not been renamed explicitly, it's
        loaded and its location is checked to see if it have moved as
        well.
        """
        if symb_info in SKIP_SYMBS:
            self.__skipped = True

        if symb_info in self.__renames:
            self.__changed = True
            return self.__renames[symb_info]
        else:
            symb = find_global(*symb_info, Broken=ZODBBroken)
            if utils.is_broken(symb):
                logger.warning('Warning: Missing factory for {}'.format(
                    ' '.join(symb_info)))
                create_broken_module_for(symb)
            elif hasattr(symb, '__name__') and hasattr(symb, '__module__'):
                new_symb_info = (symb.__module__, symb.__name__)
                if new_symb_info != symb_info:
                    logger.info('New implicit rule detected {} to {}'.format(
                        ' '.join(symb_info), ' '.join(new_symb_info)))
                    self.__renames[symb_info] = new_symb_info
                    self.__added[symb_info] = new_symb_info
                    self.__changed = True
                    return new_symb_info
        return symb_info

    def __find_global(self, *klass_info):
        """Find a class with the given name, looking for a renaming
        rule first.

        Using ZODB find_global let us manage missing classes.
        """
        return find_global(*self.__update_symb(klass_info), Broken=ZODBBroken)

    def __persistent_load(self, reference):
        """Load a persistent reference. The reference might changed
        according a renaming rules. We give back a special object to
        represent that reference, and not the real object designated
        by the reference.
        """
        # This takes care of returning the OID as bytes in order to convert
        # a database to Python 3.
        if isinstance(reference, tuple):
            oid, cls_info = reference
            if isinstance(cls_info, tuple):
                cls_info = self.__update_symb(cls_info)
            return ZODBReference(
                (utils.safe_binary(oid), cls_info))
        if isinstance(reference, list):
            if len(reference) == 1:
                oid, = reference
                return ZODBReference(
                    ['w', (utils.safe_binary(oid))])
            mode, information = reference
            if mode == 'm':
                database_name, oid, cls_info = information
                if isinstance(cls_info, tuple):
                    cls_info = self.__update_symb(cls_info)
                return ZODBReference(
                    ['m', (database_name, utils.safe_binary(oid), cls_info)])
            if mode == 'n':
                database_name, oid = information
                return ZODBReference(
                    ['m', (database_name, utils.safe_binary(oid))])
            if mode == 'w':
                if len(information) == 1:
                    oid, = information
                    return ZODBReference(
                        ['w', (utils.safe_binary(oid))])
                oid, database_name = information
                return ZODBReference(
                    ['w', (utils.safe_binary(oid), database_name)])
        if isinstance(reference, (str, zodbpickle.binary)):
            oid = reference
            return ZODBReference(utils.safe_binary(oid))
        raise AssertionError('Unknown reference format.')

    def __persistent_id(self, obj):
        """Save the given object as a reference only if it was a
        reference before. We re-use the same information.
        """
        if not isinstance(obj, ZODBReference):
            return None
        return obj.ref

    def __unpickler(self, input_file):
        """Create an unpickler with our custom global symbol loader
        and reference resolver.
        """
        return utils.Unpickler(
            input_file,
            self.__persistent_load,
            self.__find_global,
            **self.__unpickle_options)

    def __pickler(self, output_file):
        """Create a pickler able to save to the given file, objects we
        loaded while paying attention to any reference we loaded.
        """
        return utils.Pickler(
            output_file, self.__persistent_id, self.__protocol)

    def __update_class_meta(self, class_meta):
        """Update class information, which can contain information
        about a renamed class.
        """
        if isinstance(class_meta, tuple):
            symb, args = class_meta
            if utils.is_broken(symb):
                symb_info = (symb.__module__, symb.__name__)
                logger.warning(
                    'Warning: Missing factory for {}'.format(
                        ' '.join(symb_info)))
                return (symb_info, args)
            elif isinstance(symb, tuple):
                return self.__update_symb(symb), args
        return class_meta

    def __decode_data(self, class_meta, data):
        if not self.__decoders:
            return
        key = None
        if isinstance(class_meta, six.class_types):
            key = (class_meta.__module__, class_meta.__name__)
        elif isinstance(class_meta, tuple):
            symb, args = class_meta
            if isinstance(symb, six.class_types):
                key = (symb.__module__, symb.__name__)
            elif isinstance(symb, tuple):
                key = symb
            else:
                raise AssertionError('Unknown class format.')
        else:
            raise AssertionError('Unknown class format.')
        for decoder in self.__decoders.get(key, []):
            self.__changed = decoder(data) or self.__changed

    @contextlib.contextmanager
    def __patched_encoding(self):
        if self.__encoding:
            orig = utils.ENCODING
            utils.ENCODING = self.__encoding
            try:
                yield
            finally:
                utils.ENCODING = orig
        else:
            yield

    def rename(self, input_file):
        """Take a ZODB record (as a file object) as input. We load it,
        replace any reference to renamed class we know of. If any
        modification are done, we save the record again and return it,
        return None otherwise.
        """
        self.__changed = False
        self.__skipped = False

        with self.__patched_encoding():
            unpickler = self.__unpickler(input_file)
            class_meta = unpickler.load()
            if self.__skipped:
                # do not do renames/conversions on blob records
                return None
            class_meta = self.__update_class_meta(class_meta)

            data = unpickler.load()
            self.__decode_data(class_meta, data)

            if not (self.__changed or self.__repickle_all):
                return None

            output_file = io.BytesIO()
            pickler = self.__pickler(output_file)
            try:
                pickler.dump(class_meta)
                pickler.dump(data)
            except utils.PicklingError as error:
                logger.error(
                    'Error: cannot pickle modified record: {}'.format(error))
                # Could not pickle that record, skip it.
                return None

            output_file.truncate()
            return output_file

    def get_rules(self, implicit=False, explicit=False):
        rules = {}
        if explicit:
            rules.update(self.__renames)
        if implicit:
            rules.update(self.__added)
        return rules

1	##############################################################################
2	#
3	# Copyright (c) 2009-2010 Zope Corporation and Contributors.
4	# All Rights Reserved.
5	#
6	# This software is subject to the provisions of the Zope Public License,
7	# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
8	# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
9	# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
10	# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
11	# FOR A PARTICULAR PURPOSE.
12	#
13	##############################################################################
14
15	import contextlib	6✔
16	import io	6✔
17	import logging	6✔
18	import types	6✔
19	import sys	6✔
20	import six	6✔
21	import zodbpickle	6✔
22
23	from ZODB.broken import find_global, Broken, rebuild	6✔
24	from zodbupdate import utils	6✔
25
26	logger = logging.getLogger('zodbupdate.serialize')	6✔
27	known_broken_modules = {}	6✔
28
29	# types to skip when renaming/migrating databases
30	SKIP_SYMBS = [('ZODB.blob', 'Blob')]	6✔
31
32
33	def create_broken_module_for(symb):	6✔
34	"""If your pickle refer a broken class (not an instance of it, a
35	reference to the class symbol itself) you have no choice than
36	having this module available in the same symbol and with the
37	same name, otherwise repickling doesn't work (as both pickle
38	and cPickle __import__ the module, and verify the class symbol
39	is the same than the one provided).
40	"""
41	parts = symb.__module__.split('.')	6✔
42	previous = None	6✔
43	for fullname, name in reversed(	6✔
44	[('.'.join(parts[0:p + 1]), parts[p])
45	for p in range(0, len(parts))]):
46	if fullname not in sys.modules:	6✔
47	if fullname not in known_broken_modules:	6!
48	module = types.ModuleType(fullname)	6✔
49	module.__name__ = name	6✔
50	module.__file__ = '<broken module to pickle class reference>'	6✔
51	module.__path__ = []	6✔
52	known_broken_modules[fullname] = module	6✔
53	else:
54	if previous:	×
55	module = known_broken_modules[fullname]	×
56	setattr(module, *previous)	×
57	break	×
58	if previous:	6!
59	setattr(module, *previous)	×
60	previous = (name, module)	6✔
61	else:
62	if previous:	6✔
63	setattr(sys.modules[fullname], *previous)	6✔
64	break	6✔
65	if symb.__module__ in known_broken_modules:	6!
66	setattr(known_broken_modules[symb.__module__], symb.__name__, symb)	6✔
67	elif symb.__module__ in sys.modules:	×
68	setattr(sys.modules[symb.__module__], symb.__name__, symb)	×
69
70
71	class BrokenModuleFinder(object):	6✔
72	"""This broken module finder works with create_broken_module_for.
73	"""
74
75	def load_module(self, fullname):	6✔
76	module = known_broken_modules[fullname]	6✔
77	if fullname not in sys.modules:	6!
78	sys.modules[fullname] = module	6✔
79	module.__loader__ = self	6✔
80	return module	6✔
81
82	def find_module(self, fullname, path=None):	6✔
83	if fullname in known_broken_modules:	6✔
84	return self	6✔
85	return None	6✔
86
87
88	sys.meta_path.append(BrokenModuleFinder())	6✔
89
90
91	class NullIterator(six.Iterator):	6✔
92	"""An empty iterator that doesn't gives any result.
93	"""
94
95	def __iter__(self):	6✔
96	return self	×
97
98	def __next__(self):	6✔
99	raise StopIteration()	6✔
100
101
102	class IterableClass(type):	6✔
103
104	def __iter__(cls):	6✔
105	"""Define a empty iterator to fix unpickling of missing
106	Interfaces that have been used to do alsoProvides on a another
107	pickled object.
108	"""
109	return NullIterator()	6✔
110
111
112	@six.add_metaclass(IterableClass)	6✔
113	class ZODBBroken(Broken):	6✔
114	"""Extend ZODB Broken to work with broken objects that doesn't
115	have any __Broken_newargs__ sets (which happens if their __new__
116	method is not called).
117	"""
118
119	def __reduce__(self):	6✔
120	"""We pickle broken objects in hope of being able to fix them later.
121	"""
122	return (rebuild,	×
123	((self.__class__.__module__, self.__class__.__name__)
124	+ getattr(self, '__Broken_newargs__', tuple())),
125	self.__Broken_state__)
126
127
128	class ZODBReference(object):	6✔
129	"""Class to remenber reference we don't want to touch.
130	"""
131
132	def __init__(self, ref):	6✔
133	self.ref = ref	6✔
134
135
136	class ObjectRenamer(object):	6✔
137	"""This load and save a ZODB record, modifying all references to
138	renamed class according the given renaming rules:
139
140	- in global symbols contained in the record,
141
142	- in persistent reference information,
143
144	- in class information (first pickle of the record).
145	"""
146
147	def __init__(	6✔
148	self, renames, decoders, pickle_protocol=3, repickle_all=False,
149	encoding=None):
150	self.__added = dict()	6✔
151	self.__renames = renames	6✔
152	self.__decoders = decoders	6✔
153	self.__changed = False	6✔
154	self.__protocol = pickle_protocol	6✔
155	self.__repickle_all = repickle_all	6✔
156	self.__encoding = encoding	6✔
157	self.__unpickle_options = {}	6✔
158	if encoding:	6✔
159	self.__unpickle_options = {	4✔
160	'encoding': encoding,
161	'errors': 'bytes',
162	}
163
164	def __update_symb(self, symb_info):	6✔
165	"""This method look in a klass or symbol have been renamed or
166	not. If the symbol have not been renamed explicitly, it's
167	loaded and its location is checked to see if it have moved as
168	well.
169	"""
170	if symb_info in SKIP_SYMBS:	6✔
171	self.__skipped = True	6✔
172
173	if symb_info in self.__renames:	6✔
174	self.__changed = True	6✔
175	return self.__renames[symb_info]	6✔
176	else:
177	symb = find_global(*symb_info, Broken=ZODBBroken)	6✔
178	if utils.is_broken(symb):	6✔
179	logger.warning('Warning: Missing factory for {}'.format(	6✔
180	' '.join(symb_info)))
181	create_broken_module_for(symb)	6✔
182	elif hasattr(symb, '__name__') and hasattr(symb, '__module__'):	6✔
183	new_symb_info = (symb.__module__, symb.__name__)	6✔
184	if new_symb_info != symb_info:	6✔
185	logger.info('New implicit rule detected {} to {}'.format(	6✔
186	' '.join(symb_info), ' '.join(new_symb_info)))
187	self.__renames[symb_info] = new_symb_info	6✔
188	self.__added[symb_info] = new_symb_info	6✔
189	self.__changed = True	6✔
190	return new_symb_info	6✔
191	return symb_info	6✔
192
193	def __find_global(self, *klass_info):	6✔
194	"""Find a class with the given name, looking for a renaming
195	rule first.
196
197	Using ZODB find_global let us manage missing classes.
198	"""
199	return find_global(*self.__update_symb(klass_info), Broken=ZODBBroken)	6✔
200
201	def __persistent_load(self, reference):	6✔
202	"""Load a persistent reference. The reference might changed
203	according a renaming rules. We give back a special object to
204	represent that reference, and not the real object designated
205	by the reference.
206	"""
207	# This takes care of returning the OID as bytes in order to convert
208	# a database to Python 3.
209	if isinstance(reference, tuple):	6!
210	oid, cls_info = reference	6✔
211	if isinstance(cls_info, tuple):	6!
212	cls_info = self.__update_symb(cls_info)	×
213	return ZODBReference(	6✔
214	(utils.safe_binary(oid), cls_info))
215	if isinstance(reference, list):	×
216	if len(reference) == 1:	×
217	oid, = reference	×
218	return ZODBReference(	×
219	['w', (utils.safe_binary(oid))])
220	mode, information = reference	×
221	if mode == 'm':	×
222	database_name, oid, cls_info = information	×
223	if isinstance(cls_info, tuple):	×
224	cls_info = self.__update_symb(cls_info)	×
225	return ZODBReference(	×
226	['m', (database_name, utils.safe_binary(oid), cls_info)])
227	if mode == 'n':	×
228	database_name, oid = information	×
229	return ZODBReference(	×
230	['m', (database_name, utils.safe_binary(oid))])
231	if mode == 'w':	×
232	if len(information) == 1:	×
233	oid, = information	×
234	return ZODBReference(	×
235	['w', (utils.safe_binary(oid))])
236	oid, database_name = information	×
237	return ZODBReference(	×
238	['w', (utils.safe_binary(oid), database_name)])
239	if isinstance(reference, (str, zodbpickle.binary)):	×
240	oid = reference	×
241	return ZODBReference(utils.safe_binary(oid))	×
242	raise AssertionError('Unknown reference format.')	×
243
244	def __persistent_id(self, obj):	6✔
245	"""Save the given object as a reference only if it was a
246	reference before. We re-use the same information.
247	"""
248	if not isinstance(obj, ZODBReference):	6✔
249	return None	6✔
250	return obj.ref	6✔
251
252	def __unpickler(self, input_file):	6✔
253	"""Create an unpickler with our custom global symbol loader
254	and reference resolver.
255	"""
256	return utils.Unpickler(	6✔
257	input_file,
258	self.__persistent_load,
259	self.__find_global,
260	**self.__unpickle_options)
261
262	def __pickler(self, output_file):	6✔
263	"""Create a pickler able to save to the given file, objects we
264	loaded while paying attention to any reference we loaded.
265	"""
266	return utils.Pickler(	6✔
267	output_file, self.__persistent_id, self.__protocol)
268
269	def __update_class_meta(self, class_meta):	6✔
270	"""Update class information, which can contain information
271	about a renamed class.
272	"""
273	if isinstance(class_meta, tuple):	6!
274	symb, args = class_meta	×
275	if utils.is_broken(symb):	×
276	symb_info = (symb.__module__, symb.__name__)	×
277	logger.warning(	×
278	'Warning: Missing factory for {}'.format(
279	' '.join(symb_info)))
280	return (symb_info, args)	×
281	elif isinstance(symb, tuple):	×
282	return self.__update_symb(symb), args	×
283	return class_meta	6✔
284
285	def __decode_data(self, class_meta, data):	6✔
286	if not self.__decoders:	6✔
287	return	6✔
288	key = None	6✔
289	if isinstance(class_meta, six.class_types):	6!
290	key = (class_meta.__module__, class_meta.__name__)	6✔
291	elif isinstance(class_meta, tuple):	×
292	symb, args = class_meta	×
293	if isinstance(symb, six.class_types):	×
294	key = (symb.__module__, symb.__name__)	×
295	elif isinstance(symb, tuple):	×
296	key = symb	×
297	else:
298	raise AssertionError('Unknown class format.')	×
299	else:
300	raise AssertionError('Unknown class format.')	×
301	for decoder in self.__decoders.get(key, []):	6✔
302	self.__changed = decoder(data) or self.__changed	6✔
303
304	@contextlib.contextmanager	6✔
305	def __patched_encoding(self):
306	if self.__encoding:	6✔
307	orig = utils.ENCODING	4✔
308	utils.ENCODING = self.__encoding	4✔
309	try:	4✔
310	yield	4✔
311	finally:
312	utils.ENCODING = orig	4✔
313	else:
314	yield	6✔
315
316	def rename(self, input_file):	6✔
317	"""Take a ZODB record (as a file object) as input. We load it,
318	replace any reference to renamed class we know of. If any
319	modification are done, we save the record again and return it,
320	return None otherwise.
321	"""
322	self.__changed = False	6✔
323	self.__skipped = False	6✔
324
325	with self.__patched_encoding():	6✔
326	unpickler = self.__unpickler(input_file)	6✔
327	class_meta = unpickler.load()	6✔
328	if self.__skipped:	6✔
329	# do not do renames/conversions on blob records
330	return None	6✔
331	class_meta = self.__update_class_meta(class_meta)	6✔
332
333	data = unpickler.load()	6✔
334	self.__decode_data(class_meta, data)	6✔
335
336	if not (self.__changed or self.__repickle_all):	6✔
337	return None	6✔
338
339	output_file = io.BytesIO()	6✔
340	pickler = self.__pickler(output_file)	6✔
341	try:	6✔
342	pickler.dump(class_meta)	6✔
343	pickler.dump(data)	6✔
344	except utils.PicklingError as error:	×
345	logger.error(	×
346	'Error: cannot pickle modified record: {}'.format(error))
347	# Could not pickle that record, skip it.
348	return None	×
349
350	output_file.truncate()	6✔
351	return output_file	6✔
352
353	def get_rules(self, implicit=False, explicit=False):	6✔
354	rules = {}	6✔
355	if explicit:	6!
356	rules.update(self.__renames)	×
357	if implicit:	6!
358	rules.update(self.__added)	6✔
359	return rules	6✔

zopefoundation / zodbupdate / 111

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous