4399176145

Build Type

github

Committed by GitHub

Commit Message

Merge ee2cf80d9 into 7590ad02c

Pull Request Pull Request #30: Fuse

Run Details

474 of 474 new or added lines in 13 files covered. (100.0%)

1304 of 1691 relevant lines covered (77.11%)

3.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.56

/catcli/walker.py

"""
author: deadc0de6 (https://github.com/deadc0de6)
Copyright (c) 2017, deadc0de6

Catcli filesystem indexer
"""

import os
from typing import Tuple, Optional

# local imports
from catcli.noder import Noder
from catcli.logger import Logger
from catcli.nodes import NodeAny, NodeTop


class Walker:
    """a filesystem walker"""

    MAXLINELEN = 80 - 15

    def __init__(self, noder: Noder,
                 usehash: bool = True,
                 debug: bool = False,
                 logpath: str = ''):
        """
        @noder: the noder to use
        @hash: calculate hash of nodes
        @debug: debug mode
        @logpath: path where to log catalog changes on reindex
        """
        self.noder = noder
        self.usehash = usehash
        self.noder.do_hashing(self.usehash)
        self.debug = debug
        self.lpath = logpath

    def index(self, path: str,
              parent: NodeAny,
              name: str,
              storagepath: str = '') -> Tuple[str, int]:
        """
        index a directory and store in tree
        @path: path to index
        @parent: parent node
        @name: this stoarge name
        """
        self._debug(f'indexing starting at {path}')
        if not parent:
            parent = self.noder.new_dir_node(name, path,
                                             parent, storagepath)

        if os.path.islink(path):
            rel = os.readlink(path)
            abspath = os.path.join(path, rel)
            if os.path.isdir(abspath):
                return parent, 0

        cnt = 0
        for (root, dirs, files) in os.walk(path):
            for file in files:
                self._debug(f'found file {file} under {path}')
                sub = os.path.join(root, file)
                if not os.path.exists(sub):
                    continue
                self._progress(file)
                self._debug(f'index file {sub}')
                node = self.noder.new_file_node(os.path.basename(file), sub,
                                                parent, storagepath)
                if node:
                    cnt += 1
            for adir in dirs:
                self._debug(f'found dir {adir} under {path}')
                base = os.path.basename(adir)
                sub = os.path.join(root, adir)
                self._debug(f'index directory {sub}')
                if not os.path.exists(sub):
                    continue
                dummy = self.noder.new_dir_node(base, sub, parent, storagepath)
                if not dummy:
                    continue
                cnt += 1
                nstoragepath = os.sep.join([storagepath, base])
                if not storagepath:
                    nstoragepath = base
                _, cnt2 = self.index(sub, dummy, base, nstoragepath)
                cnt += cnt2
            break
        self._progress('')
        return parent, cnt

    def reindex(self, path: str, parent: NodeAny, top: NodeTop) -> int:
        """reindex a directory and store in tree"""
        cnt = self._reindex(path, parent, top)
        cnt += self.noder.clean_not_flagged(parent)
        return cnt

    def _reindex(self, path: str,
                 parent: NodeAny,
                 top: NodeTop,
                 storagepath: str = '') -> int:
        """
        reindex a directory and store in tree
        @path: directory path to re-index
        @top: top node (storage)
        @storagepath: rel path relative to indexed directory
        """
        self._debug(f'reindexing starting at {path}')
        cnt = 0
        for (root, dirs, files) in os.walk(path):
            for file in files:
                self._debug(f'found file \"{file}\" under {path}')
                sub = os.path.join(root, file)
                treepath = os.path.join(storagepath, file)
                reindex, node = self._need_reindex(parent, sub, treepath)
                if not reindex:
                    self._debug(f'\tskip file {sub}')
                    if node:
                        node.flag()
                    continue
                node = self.noder.new_file_node(os.path.basename(file), sub,
                                                parent, storagepath)
                if node:
                    node.flag()
                    cnt += 1
            for adir in dirs:
                self._debug(f'found dir \"{adir}\" under {path}')
                base = os.path.basename(adir)
                sub = os.path.join(root, adir)
                treepath = os.path.join(storagepath, adir)
                reindex, dummy = self._need_reindex(parent, sub, treepath)
                if reindex:
                    dummy = self.noder.new_dir_node(base, sub,
                                                    parent, storagepath)
                    cnt += 1
                if dummy:
                    dummy.flag()
                self._debug(f'reindexing deeper under {sub}')
                nstoragepath = os.sep.join([storagepath, base])
                if not storagepath:
                    nstoragepath = base
                if dummy:
                    cnt2 = self._reindex(sub, dummy, top, nstoragepath)
                    cnt += cnt2
            break
        return cnt

    def _need_reindex(self,
                      top: NodeTop,
                      path: str,
                      treepath: str) -> Tuple[bool, Optional[NodeTop]]:
        """
        test if node needs re-indexing
        @top: top node (storage)
        @path: abs path to file
        @treepath: rel path from indexed directory
        """
        node, changed = self.noder.get_node_if_changed(top, path, treepath)
        if not node:
            self._debug(f'\t{path} does not exist')
            return True, node
        if node and not changed:
            # ignore this node
            self._debug(f'\t{path} has not changed')
            return False, node
        if node and changed:
            # remove this node and re-add
            self._debug(f'\t{path} has changed')
            self._debug(f'\tremoving node {node.name} for {path}')
            node.parent = None
        return True, node

    def _debug(self, string: str) -> None:
        """print to debug"""
        if not self.debug:
            return
        Logger.debug(string)

    def _progress(self, string: str) -> None:
        """print progress"""
        if self.debug:
            return
        if not string:
            # clean
            Logger.progr(' ' * 80)
            return
        if len(string) > self.MAXLINELEN:
            string = string[:self.MAXLINELEN] + '...'
        Logger.progr(f'indexing: {string:80}')

1	"""
2	author: deadc0de6 (https://github.com/deadc0de6)
3	Copyright (c) 2017, deadc0de6
4
5	Catcli filesystem indexer
6	"""
7
8	import os	5✔
9	from typing import Tuple, Optional	5✔
10
11	# local imports
12	from catcli.noder import Noder	5✔
13	from catcli.logger import Logger	5✔
14	from catcli.nodes import NodeAny, NodeTop	5✔
15
16
17	class Walker:	5✔
18	"""a filesystem walker"""
19
20	MAXLINELEN = 80 - 15	5✔
21
22	def __init__(self, noder: Noder,	5✔
23	usehash: bool = True,
24	debug: bool = False,
25	logpath: str = ''):
26	"""
27	@noder: the noder to use
28	@hash: calculate hash of nodes
29	@debug: debug mode
30	@logpath: path where to log catalog changes on reindex
31	"""
32	self.noder = noder	5✔
33	self.usehash = usehash	5✔
34	self.noder.do_hashing(self.usehash)	5✔
35	self.debug = debug	5✔
36	self.lpath = logpath	5✔
37
38	def index(self, path: str,	5✔
39	parent: NodeAny,
40	name: str,
41	storagepath: str = '') -> Tuple[str, int]:
42	"""
43	index a directory and store in tree
44	@path: path to index
45	@parent: parent node
46	@name: this stoarge name
47	"""
48	self._debug(f'indexing starting at {path}')	5✔
49	if not parent:	5✔
50	parent = self.noder.new_dir_node(name, path,	×
51	parent, storagepath)
52
53	if os.path.islink(path):	5✔
54	rel = os.readlink(path)	×
55	abspath = os.path.join(path, rel)	×
56	if os.path.isdir(abspath):	×
57	return parent, 0	×
58
59	cnt = 0	5✔
60	for (root, dirs, files) in os.walk(path):	5✔
61	for file in files:	5✔
62	self._debug(f'found file {file} under {path}')	5✔
63	sub = os.path.join(root, file)	5✔
64	if not os.path.exists(sub):	5✔
65	continue	×
66	self._progress(file)	5✔
67	self._debug(f'index file {sub}')	5✔
68	node = self.noder.new_file_node(os.path.basename(file), sub,	5✔
69	parent, storagepath)
70	if node:	5✔
71	cnt += 1	5✔
72	for adir in dirs:	5✔
73	self._debug(f'found dir {adir} under {path}')	5✔
74	base = os.path.basename(adir)	5✔
75	sub = os.path.join(root, adir)	5✔
76	self._debug(f'index directory {sub}')	5✔
77	if not os.path.exists(sub):	5✔
78	continue	×
79	dummy = self.noder.new_dir_node(base, sub, parent, storagepath)	5✔
80	if not dummy:	5✔
81	continue	×
82	cnt += 1	5✔
83	nstoragepath = os.sep.join([storagepath, base])	5✔
84	if not storagepath:	5✔
85	nstoragepath = base	5✔
86	_, cnt2 = self.index(sub, dummy, base, nstoragepath)	5✔
87	cnt += cnt2	5✔
88	break	5✔
89	self._progress('')	5✔
90	return parent, cnt	5✔
91
92	def reindex(self, path: str, parent: NodeAny, top: NodeTop) -> int:	5✔
93	"""reindex a directory and store in tree"""
94	cnt = self._reindex(path, parent, top)	5✔
95	cnt += self.noder.clean_not_flagged(parent)	5✔
96	return cnt	5✔
97
98	def _reindex(self, path: str,	5✔
99	parent: NodeAny,
100	top: NodeTop,
101	storagepath: str = '') -> int:
102	"""
103	reindex a directory and store in tree
104	@path: directory path to re-index
105	@top: top node (storage)
106	@storagepath: rel path relative to indexed directory
107	"""
108	self._debug(f'reindexing starting at {path}')	5✔
109	cnt = 0	5✔
110	for (root, dirs, files) in os.walk(path):	5✔
111	for file in files:	5✔
112	self._debug(f'found file \"{file}\" under {path}')	5✔
113	sub = os.path.join(root, file)	5✔
114	treepath = os.path.join(storagepath, file)	5✔
115	reindex, node = self._need_reindex(parent, sub, treepath)	5✔
116	if not reindex:	5✔
117	self._debug(f'\tskip file {sub}')	5✔
118	if node:	5✔
119	node.flag()	5✔
120	continue	5✔
121	node = self.noder.new_file_node(os.path.basename(file), sub,	5✔
122	parent, storagepath)
123	if node:	5✔
124	node.flag()	5✔
125	cnt += 1	5✔
126	for adir in dirs:	5✔
127	self._debug(f'found dir \"{adir}\" under {path}')	5✔
128	base = os.path.basename(adir)	5✔
129	sub = os.path.join(root, adir)	5✔
130	treepath = os.path.join(storagepath, adir)	5✔
131	reindex, dummy = self._need_reindex(parent, sub, treepath)	5✔
132	if reindex:	5✔
133	dummy = self.noder.new_dir_node(base, sub,	5✔
134	parent, storagepath)
135	cnt += 1	5✔
136	if dummy:	5✔
137	dummy.flag()	5✔
138	self._debug(f'reindexing deeper under {sub}')	5✔
139	nstoragepath = os.sep.join([storagepath, base])	5✔
140	if not storagepath:	5✔
141	nstoragepath = base	5✔
142	if dummy:	5✔
143	cnt2 = self._reindex(sub, dummy, top, nstoragepath)	5✔
144	cnt += cnt2	5✔
145	break	5✔
146	return cnt	5✔
147
148	def _need_reindex(self,	5✔
149	top: NodeTop,
150	path: str,
151	treepath: str) -> Tuple[bool, Optional[NodeTop]]:
152	"""
153	test if node needs re-indexing
154	@top: top node (storage)
155	@path: abs path to file
156	@treepath: rel path from indexed directory
157	"""
158	node, changed = self.noder.get_node_if_changed(top, path, treepath)	5✔
159	if not node:	5✔
160	self._debug(f'\t{path} does not exist')	5✔
161	return True, node	5✔
162	if node and not changed:	5✔
163	# ignore this node
164	self._debug(f'\t{path} has not changed')	5✔
165	return False, node	5✔
166	if node and changed:	5✔
167	# remove this node and re-add
168	self._debug(f'\t{path} has changed')	5✔
169	self._debug(f'\tremoving node {node.name} for {path}')	5✔
170	node.parent = None	5✔
171	return True, node	5✔
172
173	def _debug(self, string: str) -> None:	5✔
174	"""print to debug"""
175	if not self.debug:	5✔
176	return	5✔
177	Logger.debug(string)	5✔
178
179	def _progress(self, string: str) -> None:	5✔
180	"""print progress"""
181	if self.debug:	5✔
182	return	5✔
183	if not string:	5✔
184	# clean
185	Logger.progr(' ' * 80)	5✔
186	return	5✔
187	if len(string) > self.MAXLINELEN:	5✔
188	string = string[:self.MAXLINELEN] + '...'	×
189	Logger.progr(f'indexing: {string:80}')	5✔

deadc0de6 / catcli / 4399176145

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous