39931488734

Committed 23 Oct 2026 09:23AM UTC coverage: 58.172% (+0.4%) from 57.812%

Build # 39931488734

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #6921 from eXist-db/dependabot/maven/ai.djl.huggingface-tokenizers-0.39.0

Bump ai.djl.huggingface:tokenizers from 0.38.0 to 0.39.0

Coverage Stats

32478 of 61732 branches covered (52.61%)

Branch coverage included in aggregate %.

85569 of 141197 relevant lines covered (60.6%)

2.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.44

/exist-core/src/main/java/org/exist/storage/statistics/DataGuide.java

/*
 * eXist-db Open Source Native XML Database
 * Copyright (C) 2001 The eXist-db Authors
 *
 * info@exist-db.org
 * http://www.exist-db.org
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */
package org.exist.storage.statistics;

import org.exist.Namespaces;
import org.exist.dom.QName;
import org.exist.dom.persistent.SymbolTable;
import org.exist.storage.ElementValue;
import org.exist.storage.NodePath;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
import java.util.ArrayList;
import java.util.List;

/**
 * Collects statistics about the distribution of elements in a document or
 * even the entire database. The class creates a graph structure which describes
 * all possible element paths and their frequency. For example, for a TEI document, a typical
 * path could be:
 *
 * <pre>TEI[44,63330] -&gt; text[44,62757] -gt; body[44,44206] -gt; div[300,5584] -gt; p[5336,820]</pre>
 *
 * which means there are 44 TEI, text and body elements in the db with 300 div children and
 * 5336 paragraphs below them. The second number indicates the size of the largest element,
 * expressed as the number of descendant elements below the node. The largest p node in this
 * distribution has 820 elements below it.
 */
public class DataGuide {

    private final static int BYTES_PER_NODE = 16;

    // the (virtual) root of the tree whose name will always be null.
    private NodeStats root = new NodeStatsRoot();

    public int getSize() {
        return root.getSize();
    }
    
    /**
     * Add the given node path (a path like /root/childA/childB) to the data guide.
     * The frequency for the target element (i.e. the last component in the path)
     * is incremented by one.
     *
     * @param path the node path
     *
     * @return the node statistics
     */
    public NodeStats add(NodePath path) {
        return add(path, null);
    }

    /**
     * Add the given node path using the frequency and size information
     * given in the second argument. Used to merge two DataGuides.
     *
     * @param path the node path
     * @param mergeWith the existing node statistics
     *
     * @return the node statistics
     */
    protected NodeStats add(NodePath path, NodeStats mergeWith) {
        NodeStats current = root;
        for (int i = 0; i < path.length(); i++) {
            final QName qn = path.getComponent(i);
            if (qn.getNameType() != ElementValue.ELEMENT) {
                return null;
            }
            current = current.addChild(qn);
        }
        if (mergeWith != null) {
            current.mergeStats(mergeWith);
        } else
            {current.addOccurrence();}
        return current;
    }

    /**
     * Merge paths and statistics from this instance into the
     * other instance.
     *
     * @param other the other data guide
     * @return the other instance containing the merged graphs
     */
    public DataGuide mergeInto(DataGuide other) {
        root.mergeInto(other, new NodePath());
        return other;
    }

    public int getMaxParentDepth(QName qname) {
        final NodeStats temp = new NodeStats(qname);
        root.getMaxParentDepth(qname, temp);
        return temp.getMaxDepth();
    }

    public String toString() {
        final List<StringBuilder> paths = new ArrayList<>();
        root.dump(new StringBuilder(), paths);

        final StringBuilder buf = new StringBuilder();
        for (StringBuilder path : paths) {
            buf.append(path);
            buf.append('\n');
        }
        return buf.toString();
    }

    public void toSAX(ContentHandler handler) throws SAXException {
        root.toSAX(handler);
    }

    public void write(SeekableByteChannel chan, SymbolTable symbols) throws IOException {
        final int nodeCount = root.getSize();
        final ByteBuffer buffer = ByteBuffer.allocate(nodeCount * BYTES_PER_NODE + 4);
        root.write(buffer, symbols);
        buffer.flip();
        chan.write(buffer);
    }

    public void read(SeekableByteChannel chan, SymbolTable symbols) throws IOException {
        final ByteBuffer buffer = ByteBuffer.allocate((int) chan.size());
        chan.read(buffer);
        buffer.flip();
        root.read(buffer, symbols);
    }

    private static class NodeStatsRoot extends NodeStats {

        private NodeStatsRoot() {
            super(null);
        }

        protected void write(ByteBuffer buffer, SymbolTable symbols) {
            if (children == null)
                {buffer.putInt(0);}
            else {
                buffer.putInt(children.length);
                for (NodeStats child : children) {
                    child.write(buffer, symbols);
                }
            }
        }

        protected void read(ByteBuffer buffer, SymbolTable symbols) {
            final int childCount = buffer.getInt();
            if (childCount > 0) {
                children = new NodeStats[childCount];
                for (int i = 0; i < childCount; i++) {
                    children[i] = new NodeStats(null);
                    children[i].read(buffer, symbols);
                }
            }
        }

        public void toSAX(ContentHandler handler) throws SAXException {
            handler.startElement(Namespaces.EXIST_NS, "distribution", "distribution", new AttributesImpl());
            if (children != null) {
                for (NodeStats child : children) {
                    child.toSAX(handler);
                }
            }
            handler.endElement(Namespaces.EXIST_NS, "distribution", "distribution");
        }
    }
}

1	/*
2	* eXist-db Open Source Native XML Database
3	* Copyright (C) 2001 The eXist-db Authors
4	*
5	* info@exist-db.org
6	* http://www.exist-db.org
7	*
8	* This library is free software; you can redistribute it and/or
9	* modify it under the terms of the GNU Lesser General Public
10	* License as published by the Free Software Foundation; either
11	* version 2.1 of the License, or (at your option) any later version.
12	*
13	* This library is distributed in the hope that it will be useful,
14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16	* Lesser General Public License for more details.
17	*
18	* You should have received a copy of the GNU Lesser General Public
19	* License along with this library; if not, write to the Free Software
20	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21	*/
22	package org.exist.storage.statistics;
23
24	import org.exist.Namespaces;
25	import org.exist.dom.QName;
26	import org.exist.dom.persistent.SymbolTable;
27	import org.exist.storage.ElementValue;
28	import org.exist.storage.NodePath;
29	import org.xml.sax.ContentHandler;
30	import org.xml.sax.SAXException;
31	import org.xml.sax.helpers.AttributesImpl;
32
33	import java.io.IOException;
34	import java.nio.ByteBuffer;
35	import java.nio.channels.SeekableByteChannel;
36	import java.util.ArrayList;
37	import java.util.List;
38
39	/**
40	* Collects statistics about the distribution of elements in a document or
41	* even the entire database. The class creates a graph structure which describes
42	* all possible element paths and their frequency. For example, for a TEI document, a typical
43	* path could be:
44	*
45	* <pre>TEI[44,63330] -> text[44,62757] -gt; body[44,44206] -gt; div[300,5584] -gt; p[5336,820]</pre>
46	*
47	* which means there are 44 TEI, text and body elements in the db with 300 div children and
48	* 5336 paragraphs below them. The second number indicates the size of the largest element,
49	* expressed as the number of descendant elements below the node. The largest p node in this
50	* distribution has 820 elements below it.
51	*/
52	public class DataGuide {	3 ✔
53
54	private final static int BYTES_PER_NODE = 16;
55
56	// the (virtual) root of the tree whose name will always be null.
57	private NodeStats root = new NodeStatsRoot();	5 ✔
58
59	public int getSize() {
60	return root.getSize();	×
61	}
62
63	/**
64	* Add the given node path (a path like /root/childA/childB) to the data guide.
65	* The frequency for the target element (i.e. the last component in the path)
66	* is incremented by one.
67	*
68	* @param path the node path
69	*
70	* @return the node statistics
71	*/
72	public NodeStats add(NodePath path) {
73	return add(path, null);	5 ✔
74	}
75
76	/**
77	* Add the given node path using the frequency and size information
78	* given in the second argument. Used to merge two DataGuides.
79	*
80	* @param path the node path
81	* @param mergeWith the existing node statistics
82	*
83	* @return the node statistics
84	*/
85	protected NodeStats add(NodePath path, NodeStats mergeWith) {
86	NodeStats current = root;	3 ✔
87	for (int i = 0; i < path.length(); i++) {	8 ✔
88	final QName qn = path.getComponent(i);	4 ✔
89	if (qn.getNameType() != ElementValue.ELEMENT) {	3 !
90	return null;	×
91	}
92	current = current.addChild(qn);	4 ✔
93	}
94	if (mergeWith != null) {	2 ✔
95	current.mergeStats(mergeWith);	3 ✔
96	} else	1 ✔
97	{current.addOccurrence();}	2 ✔
98	return current;	2 ✔
99	}
100
101	/**
102	* Merge paths and statistics from this instance into the
103	* other instance.
104	*
105	* @param other the other data guide
106	* @return the other instance containing the merged graphs
107	*/
108	public DataGuide mergeInto(DataGuide other) {
109	root.mergeInto(other, new NodePath());	7 ✔
110	return other;	2 ✔
111	}
112
113	public int getMaxParentDepth(QName qname) {
114	final NodeStats temp = new NodeStats(qname);	×
115	root.getMaxParentDepth(qname, temp);	×
116	return temp.getMaxDepth();	×
117	}
118
119	public String toString() {
120	final List<StringBuilder> paths = new ArrayList<>();	×
121	root.dump(new StringBuilder(), paths);	×
122
123	final StringBuilder buf = new StringBuilder();	×
124	for (StringBuilder path : paths) {	×
125	buf.append(path);	×
126	buf.append('\n');	×
127	}
128	return buf.toString();	×
129	}
130
131	public void toSAX(ContentHandler handler) throws SAXException {
132	root.toSAX(handler);	×
133	}	×
134
135	public void write(SeekableByteChannel chan, SymbolTable symbols) throws IOException {
136	final int nodeCount = root.getSize();	4 ✔
137	final ByteBuffer buffer = ByteBuffer.allocate(nodeCount * BYTES_PER_NODE + 4);	7 ✔
138	root.write(buffer, symbols);	5 ✔
139	buffer.flip();	3 ✔
140	chan.write(buffer);	4 ✔
141	}	1 ✔
142
143	public void read(SeekableByteChannel chan, SymbolTable symbols) throws IOException {
144	final ByteBuffer buffer = ByteBuffer.allocate((int) chan.size());	×
145	chan.read(buffer);	×
146	buffer.flip();	×
147	root.read(buffer, symbols);	×
148	}	×
149
150	private static class NodeStatsRoot extends NodeStats {
151
152	private NodeStatsRoot() {
153	super(null);	3 ✔
154	}	1 ✔
155
156	protected void write(ByteBuffer buffer, SymbolTable symbols) {
157	if (children == null)	3 !
158	{buffer.putInt(0);}	×
159	else {
160	buffer.putInt(children.length);	6 ✔
161	for (NodeStats child : children) {	17 ✔
162	child.write(buffer, symbols);	4 ✔
163	}
164	}
165	}	1 ✔
166
167	protected void read(ByteBuffer buffer, SymbolTable symbols) {
168	final int childCount = buffer.getInt();	×
169	if (childCount > 0) {	×
170	children = new NodeStats[childCount];	×
171	for (int i = 0; i < childCount; i++) {	×
172	children[i] = new NodeStats(null);	×
173	children[i].read(buffer, symbols);	×
174	}
175	}
176	}	×
177
178	public void toSAX(ContentHandler handler) throws SAXException {
179	handler.startElement(Namespaces.EXIST_NS, "distribution", "distribution", new AttributesImpl());	×
180	if (children != null) {	×
181	for (NodeStats child : children) {	×
182	child.toSAX(handler);	×
183	}
184	}
185	handler.endElement(Namespaces.EXIST_NS, "distribution", "distribution");	×
186	}	×
187	}
188	}

eXist-db / exist / 39931488734

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous