8679957491

Committed 30 Mar 2024 11:42AM CUT coverage: 83.744%. Remained the same

Build # 8679957491

Build Type

push

github

Committed by

dataflake

Commit Message

- vb [ci skip]

Run Details

2878 of 4051 branches covered (71.04%)

13348 of 15939 relevant lines covered (83.74%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/ZODB/scripts/netspace.py

#!/usr/bin/env python
"""Report on the net size of objects counting subobjects.

usage: netspace.py [-P | -v] data.fs

-P: do a pack first
-v: print info for all objects, even if a traversal path isn't found
"""


import ZODB
from ZODB.FileStorage import FileStorage
from ZODB.serialize import referencesf
from ZODB.utils import U64
from ZODB.utils import get_pickle_metadata
from ZODB.utils import load_current


def find_paths(root, maxdist):
    """Find Python attribute traversal paths for objects to maxdist distance.

    Starting at a root object, traverse attributes up to distance levels
    from the root, looking for persistent objects.  Return a dict
    mapping oids to traversal paths.

    TODO:  Assumes that the keys of the root are not themselves
    persistent objects.

    TODO:  Doesn't traverse containers.
    """
    paths = {}

    # Handle the root as a special case because it's a dict
    objs = []
    for k, v in root.items():
        oid = getattr(v, '_p_oid', None)
        objs.append((k, v, oid, 0))

    for path, obj, oid, dist in objs:
        if oid is not None:
            paths[oid] = path
        if dist < maxdist:
            getattr(obj, 'foo', None)  # unghostify
            try:
                items = obj.__dict__.items()
            except AttributeError:
                continue
            for k, v in items:
                oid = getattr(v, '_p_oid', None)
                objs.append(("{}.{}".format(path, k), v, oid, dist + 1))

    return paths


def main(path):
    fs = FileStorage(path, read_only=1)
    if PACK:
        fs.pack()

    db = ZODB.DB(fs)
    rt = db.open().root()
    paths = find_paths(rt, 3)

    def total_size(oid):
        cache = {}
        cache_size = 1000

        def _total_size(oid, seen):
            v = cache.get(oid)
            if v is not None:
                return v
            data, serialno = load_current(fs, oid)
            size = len(data)
            for suboid in referencesf(data):
                if suboid in seen:
                    continue
                seen[suboid] = 1
                size += _total_size(suboid, seen)
            cache[oid] = size
            if len(cache) == cache_size:
                cache.popitem()
            return size
        return _total_size(oid, {})

    keys = fs._index.keys()
    keys.sort()
    keys.reverse()

    if not VERBOSE:
        # If not running verbosely, don't print an entry for an object
        # unless it has an entry in paths.
        keys = filter(paths.has_key, keys)

    fmt = "%8s %5d %8d %s %s.%s"

    for oid in keys:
        data, serialno = load_current(fs, oid)
        mod, klass = get_pickle_metadata(data)
        referencesf(data)
        path = paths.get(oid, '-')
        print(fmt % (U64(oid), len(data), total_size(oid), path, mod, klass))


def Main():
    import getopt
    import sys

    global PACK
    global VERBOSE

    PACK = 0
    VERBOSE = 0
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'Pv')
        path, = args
    except getopt.error as err:
        print(err)
        print(__doc__)
        sys.exit(2)
    except ValueError:
        print("expected one argument, got", len(args))
        print(__doc__)
        sys.exit(2)
    for o, v in opts:
        if o == '-P':
            PACK = 1
        if o == '-v':
            VERBOSE += 1
    main(path)


if __name__ == "__main__":
    Main()

1	#!/usr/bin/env python
2	"""Report on the net size of objects counting subobjects.	×
3
4	usage: netspace.py [-P \| -v] data.fs
5
6	-P: do a pack first
7	-v: print info for all objects, even if a traversal path isn't found
8	"""
9
10
11	import ZODB	×
12	from ZODB.FileStorage import FileStorage	×
13	from ZODB.serialize import referencesf	×
14	from ZODB.utils import U64	×
15	from ZODB.utils import get_pickle_metadata	×
16	from ZODB.utils import load_current	×
17
18
19	def find_paths(root, maxdist):	×
20	"""Find Python attribute traversal paths for objects to maxdist distance.
21
22	Starting at a root object, traverse attributes up to distance levels
23	from the root, looking for persistent objects. Return a dict
24	mapping oids to traversal paths.
25
26	TODO: Assumes that the keys of the root are not themselves
27	persistent objects.
28
29	TODO: Doesn't traverse containers.
30	"""
31	paths = {}	×
32
33	# Handle the root as a special case because it's a dict
34	objs = []	×
35	for k, v in root.items():	×
36	oid = getattr(v, '_p_oid', None)	×
37	objs.append((k, v, oid, 0))	×
38
39	for path, obj, oid, dist in objs:	×
40	if oid is not None:	×
41	paths[oid] = path	×
42	if dist < maxdist:	×
43	getattr(obj, 'foo', None) # unghostify	×
44	try:	×
45	items = obj.__dict__.items()	×
46	except AttributeError:	×
47	continue	×
48	for k, v in items:	×
49	oid = getattr(v, '_p_oid', None)	×
50	objs.append(("{}.{}".format(path, k), v, oid, dist + 1))	×
51
52	return paths	×
53
54
55	def main(path):	×
56	fs = FileStorage(path, read_only=1)	×
57	if PACK:	×
58	fs.pack()	×
59
60	db = ZODB.DB(fs)	×
61	rt = db.open().root()	×
62	paths = find_paths(rt, 3)	×
63
64	def total_size(oid):	×
65	cache = {}	×
66	cache_size = 1000	×
67
68	def _total_size(oid, seen):	×
69	v = cache.get(oid)	×
70	if v is not None:	×
71	return v	×
72	data, serialno = load_current(fs, oid)	×
73	size = len(data)	×
74	for suboid in referencesf(data):	×
75	if suboid in seen:	×
76	continue	×
77	seen[suboid] = 1	×
78	size += _total_size(suboid, seen)	×
79	cache[oid] = size	×
80	if len(cache) == cache_size:	×
81	cache.popitem()	×
82	return size	×
83	return _total_size(oid, {})	×
84
85	keys = fs._index.keys()	×
86	keys.sort()	×
87	keys.reverse()	×
88
89	if not VERBOSE:	×
90	# If not running verbosely, don't print an entry for an object
91	# unless it has an entry in paths.
92	keys = filter(paths.has_key, keys)	×
93
94	fmt = "%8s %5d %8d %s %s.%s"	×
95
96	for oid in keys:	×
97	data, serialno = load_current(fs, oid)	×
98	mod, klass = get_pickle_metadata(data)	×
99	referencesf(data)	×
100	path = paths.get(oid, '-')	×
101	print(fmt % (U64(oid), len(data), total_size(oid), path, mod, klass))	×
102
103
104	def Main():	×
105	import getopt	×
106	import sys	×
107
108	global PACK
109	global VERBOSE
110
111	PACK = 0	×
112	VERBOSE = 0	×
113	try:	×
114	opts, args = getopt.getopt(sys.argv[1:], 'Pv')	×
115	path, = args	×
116	except getopt.error as err:	×
117	print(err)	×
118	print(__doc__)	×
119	sys.exit(2)	×
120	except ValueError:	×
121	print("expected one argument, got", len(args))	×
122	print(__doc__)	×
123	sys.exit(2)	×
124	for o, v in opts:	×
125	if o == '-P':	×
126	PACK = 1	×
127	if o == '-v':	×
128	VERBOSE += 1	×
129	main(path)	×
130
131
132	if __name__ == "__main__":	×
133	Main()	×

zopefoundation / ZODB / 8679957491

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous