6507626273

Committed 13 Oct 2023 11:25AM UTC coverage: 42.006% (-53.7%) from 95.72%

Build # 6507626273

Build Type

Pull #433

github

Committed by

odscjames

Commit Message

New "Geo" optional dependencies

https://github.com/OpenDataServices/flatten-tool/issues/424

Pull Request Pull Request #433: New "Geo" optional dependencies

Run Details

38 of 38 new or added lines in 6 files covered. (100.0%)

1466 of 3490 relevant lines covered (42.01%)

4.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.35

/flattentool/sheet.py

import copy

import BTrees.IOBTree


class Sheet(object):
    """
    An abstract representation of a single sheet of a spreadsheet.

    """

    def __init__(self, columns=None, root_id="", name=None):
        self.id_columns = []
        self.columns = columns if columns else []
        self.titles = {}
        self._lines = []
        self.root_id = root_id
        self.name = name

    @property
    def lines(self):
        return self._lines

    def add_field(self, field, id_field=False):
        columns = self.id_columns if id_field else self.columns
        if field not in columns:
            columns.append(field)

    def append(self, item):
        self.add_field(item)

    def __iter__(self):
        if self.root_id:
            yield self.root_id
        for column in self.id_columns:
            yield column
        for column in self.columns:
            yield column

    def append_line(self, flattened_dict):
        self._lines.append(flattened_dict)


class PersistentSheet(Sheet):
    """
    A sheet that is persisted in ZODB database.

    """

    def __init__(self, columns=None, root_id="", name=None, connection=None):
        super().__init__(columns=columns, root_id=root_id, name=name)
        self.connection = connection
        self.index = 0
        # Integer key and object value btree.  Store sequential index in order to preserve input order.
        connection.root.sheet_store[self.name] = BTrees.IOBTree.BTree()

    @property
    def lines(self):
        # btrees iterate in key order.
        for key, value in self.connection.root.sheet_store[self.name].items():
            # 5000 chosen by trial and error.  The written row
            # data is removed from memory as is no loner needed.
            # All new sheets clear out previous sheets data from memory.
            if key % 5000 == 0:
                self.connection.cacheMinimize()
            yield value

    def append_line(self, flattened_dict):
        self.connection.root.sheet_store[self.name][self.index] = flattened_dict
        self.index += 1

    @classmethod
    def from_sheet(cls, sheet, connection):
        instance = cls(name=sheet.name, connection=connection)
        instance.id_columns = copy.deepcopy(sheet.id_columns)
        instance.columns = copy.deepcopy(sheet.columns)
        instance.titles = copy.deepcopy(sheet.titles)
        instance.root_id = sheet.root_id
        return instance

1	import copy	10✔
2
3	import BTrees.IOBTree	10✔
4
5
6	class Sheet(object):	10✔
7	"""
8	An abstract representation of a single sheet of a spreadsheet.
9
10	"""
11
12	def __init__(self, columns=None, root_id="", name=None):	10✔
13	self.id_columns = []	10✔
14	self.columns = columns if columns else []	10✔
15	self.titles = {}	10✔
16	self._lines = []	10✔
17	self.root_id = root_id	10✔
18	self.name = name	10✔
19
20	@property	10✔
21	def lines(self):	8✔
22	return self._lines	10✔
23
24	def add_field(self, field, id_field=False):	10✔
25	columns = self.id_columns if id_field else self.columns	10✔
26	if field not in columns:	10✔
27	columns.append(field)	10✔
28
29	def append(self, item):	10✔
30	self.add_field(item)	10✔
31
32	def __iter__(self):	10✔
33	if self.root_id:	10✔
34	yield self.root_id	×
35	for column in self.id_columns:	10✔
36	yield column	×
37	for column in self.columns:	10✔
38	yield column	10✔
39
40	def append_line(self, flattened_dict):	10✔
41	self._lines.append(flattened_dict)	×
42
43
44	class PersistentSheet(Sheet):	10✔
45	"""
46	A sheet that is persisted in ZODB database.
47
48	"""
49
50	def __init__(self, columns=None, root_id="", name=None, connection=None):	10✔
51	super().__init__(columns=columns, root_id=root_id, name=name)	10✔
52	self.connection = connection	10✔
53	self.index = 0	10✔
54	# Integer key and object value btree. Store sequential index in order to preserve input order.
55	connection.root.sheet_store[self.name] = BTrees.IOBTree.BTree()	10✔
56
57	@property	10✔
58	def lines(self):	8✔
59	# btrees iterate in key order.
60	for key, value in self.connection.root.sheet_store[self.name].items():	10✔
61	# 5000 chosen by trial and error. The written row
62	# data is removed from memory as is no loner needed.
63	# All new sheets clear out previous sheets data from memory.
64	if key % 5000 == 0:	10✔
65	self.connection.cacheMinimize()	10✔
66	yield value	10✔
67
68	def append_line(self, flattened_dict):	10✔
69	self.connection.root.sheet_store[self.name][self.index] = flattened_dict	10✔
70	self.index += 1	10✔
71
72	@classmethod	10✔
73	def from_sheet(cls, sheet, connection):	8✔
74	instance = cls(name=sheet.name, connection=connection)	×
75	instance.id_columns = copy.deepcopy(sheet.id_columns)	×
76	instance.columns = copy.deepcopy(sheet.columns)	×
77	instance.titles = copy.deepcopy(sheet.titles)	×
78	instance.root_id = sheet.root_id	×
79	return instance	×

OpenDataServices / flatten-tool / 6507626273

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous