8679957491

Committed 30 Mar 2024 11:42AM CUT coverage: 83.744%. Remained the same

Build # 8679957491

Build Type

push

github

Committed by

dataflake

Commit Message

- vb [ci skip]

Run Details

2878 of 4051 branches covered (71.04%)

13348 of 15939 relevant lines covered (83.74%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/ZODB/scripts/fsrefs.py

#!/usr/bin/env python
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################

"""Check FileStorage for dangling references.

usage: fsrefs.py [-v] data.fs

fsrefs.py checks object sanity by trying to load the current revision of
every object O in the database, and also verifies that every object
directly reachable from each such O exists in the database.

It's hard to explain exactly what it does because it relies on undocumented
features in Python's cPickle module:  many of the crucial steps of loading
an object are taken, but application objects aren't actually created.  This
saves a lot of time, and allows fsrefs to be run even if the code
implementing the object classes isn't available.

A read-only connection to the specified FileStorage is made, but it is not
recommended to run fsrefs against a live FileStorage.  Because a live
FileStorage is mutating while fsrefs runs, it's not possible for fsrefs to
get a wholly consistent view of the database across the entire time fsrefs
is running; spurious error messages may result.

fsrefs doesn't normally produce any output.  If an object fails to load, the
oid of the object is given in a message saying so, and if -v was specified
then the traceback corresponding to the load failure is also displayed
(this is the only effect of the -v flag).

Three other kinds of errors are also detected, when an object O loads OK,
and directly refers to a persistent object P but there's a problem with P:

 - If P doesn't exist in the database, a message saying so is displayed.
   The unsatisifiable reference to P is often called a "dangling
   reference"; P is called "missing" in the error output.

 - If the current state of the database is such that P's creation has
   been undone, then P can't be loaded either.  This is also a kind of
   dangling reference, but is identified as "object creation was undone".

 - If P can't be loaded (but does exist in the database), a message saying
   that O refers to an object that can't be loaded is displayed.

fsrefs also (indirectly) checks that the .index file is sane, because
fsrefs uses the index to get its idea of what constitutes "all the objects
in the database".

Note these limitations:  because fsrefs only looks at the current revision
of objects, it does not attempt to load objects in versions, or non-current
revisions of objects; therefore fsrefs cannot find problems in versions or
in non-current revisions.
"""

import traceback

from BTrees.QQBTree import QQBTree

from ZODB.FileStorage import FileStorage
from ZODB.POSException import POSKeyError
from ZODB.serialize import get_refs
from ZODB.TimeStamp import TimeStamp
from ZODB.utils import get_pickle_metadata
from ZODB.utils import load_current
from ZODB.utils import oid_repr
from ZODB.utils import p64
from ZODB.utils import u64


# There's a problem with oid.  'data' is its pickle, and 'serial' its
# serial number.  'missing' is a list of (oid, class, reason) triples,
# explaining what the problem(s) is(are).


def report(oid, data, serial, missing):
    from_mod, from_class = get_pickle_metadata(data)
    if len(missing) > 1:
        plural = "s"
    else:
        plural = ""
    ts = TimeStamp(serial)
    print("oid {} {}.{}".format(hex(u64(oid)), from_mod, from_class))
    print("last updated: {}, tid={}".format(ts, hex(u64(serial))))
    print("refers to invalid object%s:" % plural)
    for oid, info, reason in missing:
        if isinstance(info, tuple):
            description = "%s.%s" % info
        else:
            description = str(info)
        print("\toid {} {}: {!r}".format(oid_repr(oid), reason, description))
    print()


def main(path=None):
    verbose = 0
    if path is None:
        import getopt
        import sys

        opts, args = getopt.getopt(sys.argv[1:], "v")
        for k, v in opts:
            if k == "-v":
                verbose += 1

        path, = args

    fs = FileStorage(path, read_only=1)

    # Set of oids in the index that failed to load due to POSKeyError.
    # This is what happens if undo is applied to the transaction creating
    # the object (the oid is still in the index, but its current data
    # record has a backpointer of 0, and POSKeyError is raised then
    # because of that backpointer).
    undone = {}

    # Set of oids that were present in the index but failed to load.
    # This does not include oids in undone.
    noload = {}

    # build {pos -> oid} index that is reverse to {oid -> pos} fs._index
    # we'll need this to iterate objects in order of ascending file position to
    # optimize disk IO.
    pos2oid = QQBTree()  # pos -> u64(oid)
    for oid, pos in fs._index.iteritems():
        pos2oid[pos] = u64(oid)

    # pass 1: load all objects listed in the index and remember those objects
    # that are deleted or load with an error. Iterate objects in order of
    # ascending file position to optimize disk IO.
    for oid64 in pos2oid.itervalues():
        oid = p64(oid64)
        try:
            data, serial = load_current(fs, oid)
        except (KeyboardInterrupt, SystemExit):
            raise
        except POSKeyError:
            undone[oid] = 1
        except:  # noqa: E722 do not use bare 'except'
            if verbose:
                traceback.print_exc()
            noload[oid] = 1

    # pass 2: go through all objects again and verify that their references do
    # not point to problematic object set. Iterate objects in order of
    # ascending file position to optimize disk IO.
    inactive = noload.copy()
    inactive.update(undone)
    for oid64 in pos2oid.itervalues():
        oid = p64(oid64)
        if oid in inactive:
            continue
        data, serial = load_current(fs, oid)
        refs = get_refs(data)
        missing = []  # contains 3-tuples of oid, klass-metadata, reason
        for ref, klass in refs:
            if klass is None:
                klass = '<unknown>'
            if ref not in fs._index:
                missing.append((ref, klass, "missing"))
            if ref in noload:
                missing.append((ref, klass, "failed to load"))
            if ref in undone:
                missing.append((ref, klass, "object creation was undone"))
        if missing:
            report(oid, data, serial, missing)


if __name__ == "__main__":
    main()

1	#!/usr/bin/env python
2	##############################################################################
3	#
4	# Copyright (c) 2002 Zope Foundation and Contributors.
5	# All Rights Reserved.
6	#
7	# This software is subject to the provisions of the Zope Public License,
8	# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
9	# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
10	# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
11	# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
12	# FOR A PARTICULAR PURPOSE
13	#
14	##############################################################################
15
16	"""Check FileStorage for dangling references.	×
17
18	usage: fsrefs.py [-v] data.fs
19
20	fsrefs.py checks object sanity by trying to load the current revision of
21	every object O in the database, and also verifies that every object
22	directly reachable from each such O exists in the database.
23
24	It's hard to explain exactly what it does because it relies on undocumented
25	features in Python's cPickle module: many of the crucial steps of loading
26	an object are taken, but application objects aren't actually created. This
27	saves a lot of time, and allows fsrefs to be run even if the code
28	implementing the object classes isn't available.
29
30	A read-only connection to the specified FileStorage is made, but it is not
31	recommended to run fsrefs against a live FileStorage. Because a live
32	FileStorage is mutating while fsrefs runs, it's not possible for fsrefs to
33	get a wholly consistent view of the database across the entire time fsrefs
34	is running; spurious error messages may result.
35
36	fsrefs doesn't normally produce any output. If an object fails to load, the
37	oid of the object is given in a message saying so, and if -v was specified
38	then the traceback corresponding to the load failure is also displayed
39	(this is the only effect of the -v flag).
40
41	Three other kinds of errors are also detected, when an object O loads OK,
42	and directly refers to a persistent object P but there's a problem with P:
43
44	- If P doesn't exist in the database, a message saying so is displayed.
45	The unsatisifiable reference to P is often called a "dangling
46	reference"; P is called "missing" in the error output.
47
48	- If the current state of the database is such that P's creation has
49	been undone, then P can't be loaded either. This is also a kind of
50	dangling reference, but is identified as "object creation was undone".
51
52	- If P can't be loaded (but does exist in the database), a message saying
53	that O refers to an object that can't be loaded is displayed.
54
55	fsrefs also (indirectly) checks that the .index file is sane, because
56	fsrefs uses the index to get its idea of what constitutes "all the objects
57	in the database".
58
59	Note these limitations: because fsrefs only looks at the current revision
60	of objects, it does not attempt to load objects in versions, or non-current
61	revisions of objects; therefore fsrefs cannot find problems in versions or
62	in non-current revisions.
63	"""
64
65	import traceback	×
66
67	from BTrees.QQBTree import QQBTree	×
68
69	from ZODB.FileStorage import FileStorage	×
70	from ZODB.POSException import POSKeyError	×
71	from ZODB.serialize import get_refs	×
72	from ZODB.TimeStamp import TimeStamp	×
73	from ZODB.utils import get_pickle_metadata	×
74	from ZODB.utils import load_current	×
75	from ZODB.utils import oid_repr	×
76	from ZODB.utils import p64	×
77	from ZODB.utils import u64	×
78
79
80	# There's a problem with oid. 'data' is its pickle, and 'serial' its
81	# serial number. 'missing' is a list of (oid, class, reason) triples,
82	# explaining what the problem(s) is(are).
83
84
85	def report(oid, data, serial, missing):	×
86	from_mod, from_class = get_pickle_metadata(data)	×
87	if len(missing) > 1:	×
88	plural = "s"	×
89	else:
90	plural = ""	×
91	ts = TimeStamp(serial)	×
92	print("oid {} {}.{}".format(hex(u64(oid)), from_mod, from_class))	×
93	print("last updated: {}, tid={}".format(ts, hex(u64(serial))))	×
94	print("refers to invalid object%s:" % plural)	×
95	for oid, info, reason in missing:	×
96	if isinstance(info, tuple):	×
97	description = "%s.%s" % info	×
98	else:
99	description = str(info)	×
100	print("\toid {} {}: {!r}".format(oid_repr(oid), reason, description))	×
101	print()	×
102
103
104	def main(path=None):	×
105	verbose = 0	×
106	if path is None:	×
107	import getopt	×
108	import sys	×
109
110	opts, args = getopt.getopt(sys.argv[1:], "v")	×
111	for k, v in opts:	×
112	if k == "-v":	×
113	verbose += 1	×
114
115	path, = args	×
116
117	fs = FileStorage(path, read_only=1)	×
118
119	# Set of oids in the index that failed to load due to POSKeyError.
120	# This is what happens if undo is applied to the transaction creating
121	# the object (the oid is still in the index, but its current data
122	# record has a backpointer of 0, and POSKeyError is raised then
123	# because of that backpointer).
124	undone = {}	×
125
126	# Set of oids that were present in the index but failed to load.
127	# This does not include oids in undone.
128	noload = {}	×
129
130	# build {pos -> oid} index that is reverse to {oid -> pos} fs._index
131	# we'll need this to iterate objects in order of ascending file position to
132	# optimize disk IO.
133	pos2oid = QQBTree() # pos -> u64(oid)	×
134	for oid, pos in fs._index.iteritems():	×
135	pos2oid[pos] = u64(oid)	×
136
137	# pass 1: load all objects listed in the index and remember those objects
138	# that are deleted or load with an error. Iterate objects in order of
139	# ascending file position to optimize disk IO.
140	for oid64 in pos2oid.itervalues():	×
141	oid = p64(oid64)	×
142	try:	×
143	data, serial = load_current(fs, oid)	×
144	except (KeyboardInterrupt, SystemExit):	×
145	raise	×
146	except POSKeyError:	×
147	undone[oid] = 1	×
148	except: # noqa: E722 do not use bare 'except'	×
149	if verbose:	×
150	traceback.print_exc()	×
151	noload[oid] = 1	×
152
153	# pass 2: go through all objects again and verify that their references do
154	# not point to problematic object set. Iterate objects in order of
155	# ascending file position to optimize disk IO.
156	inactive = noload.copy()	×
157	inactive.update(undone)	×
158	for oid64 in pos2oid.itervalues():	×
159	oid = p64(oid64)	×
160	if oid in inactive:	×
161	continue	×
162	data, serial = load_current(fs, oid)	×
163	refs = get_refs(data)	×
164	missing = [] # contains 3-tuples of oid, klass-metadata, reason	×
165	for ref, klass in refs:	×
166	if klass is None:	×
167	klass = '<unknown>'	×
168	if ref not in fs._index:	×
169	missing.append((ref, klass, "missing"))	×
170	if ref in noload:	×
171	missing.append((ref, klass, "failed to load"))	×
172	if ref in undone:	×
173	missing.append((ref, klass, "object creation was undone"))	×
174	if missing:	×
175	report(oid, data, serial, missing)	×
176
177
178	if __name__ == "__main__":	×
179	main()	×

zopefoundation / ZODB / 8679957491

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous