• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

apache / iotdb / #9996

05 Sep 2023 02:57AM UTC coverage: 47.738% (+0.006%) from 47.732%
#9996

push

travis_ci

web-flow
Pipe: Improve performance for 10000+ pipes (#11021)

Improvements:

* Cache the File objects to reduce the size of TsFileInsertionEvents.

* Limit the frequency of flushing TsFiles when cocurrently creating historical extractors.

* If cluster heartbeat doesn't need PipeMetaList, do not get into PipeTaskAgent. This will prevent datanode from turning UNKNOWN when many pipes are being created.

---------

Co-authored-by: Steve Yurong Su <rong@apache.org>

28 of 28 new or added lines in 3 files covered. (100.0%)

80382 of 168381 relevant lines covered (47.74%)

0.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.1
/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
1
/*
2
 * Licensed to the Apache Software Foundation (ASF) under one
3
 * or more contributor license agreements.  See the NOTICE file
4
 * distributed with this work for additional information
5
 * regarding copyright ownership.  The ASF licenses this file
6
 * to you under the Apache License, Version 2.0 (the
7
 * "License"); you may not use this file except in compliance
8
 * with the License.  You may obtain a copy of the License at
9
 *
10
 *     http://www.apache.org/licenses/LICENSE-2.0
11
 *
12
 * Unless required by applicable law or agreed to in writing,
13
 * software distributed under the License is distributed on an
14
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
 * KIND, either express or implied.  See the License for the
16
 * specific language governing permissions and limitations
17
 * under the License.
18
 */
19

20
package org.apache.iotdb.commons.path;
21

22
import org.apache.iotdb.commons.conf.IoTDBConstant;
23
import org.apache.iotdb.commons.path.PathPatternNode.VoidSerializer;
24
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
25
import org.apache.iotdb.tsfile.utils.PublicBAOS;
26

27
import java.io.DataOutputStream;
28
import java.io.IOException;
29
import java.nio.ByteBuffer;
30
import java.util.ArrayDeque;
31
import java.util.ArrayList;
32
import java.util.Deque;
33
import java.util.HashSet;
34
import java.util.Iterator;
35
import java.util.LinkedList;
36
import java.util.List;
37
import java.util.Objects;
38
import java.util.Set;
39

40
public class PathPatternTree {
41

42
  private PathPatternNode<Void, VoidSerializer> root;
43

44
  private List<PartialPath> pathPatternList;
45

46
  // set the default value to TRUE to ensure correctness
47
  private boolean useWildcard = true;
1✔
48

49
  public PathPatternTree(boolean useWildcard) {
50
    this();
1✔
51
    this.useWildcard = useWildcard;
1✔
52
  }
1✔
53

54
  public PathPatternTree() {
1✔
55
    this.root = new PathPatternNode<>(IoTDBConstant.PATH_ROOT, VoidSerializer.getInstance());
1✔
56
    this.pathPatternList = new LinkedList<>();
1✔
57
  }
1✔
58

59
  public PathPatternNode<Void, VoidSerializer> getRoot() {
60
    return root;
×
61
  }
62

63
  public void setRoot(PathPatternNode<Void, VoidSerializer> root) {
64
    this.root = root;
1✔
65
  }
1✔
66

67
  /////////////////////////////////////////////////////////////////////////////////////////////////
68
  // Operations for constructing tree
69
  /////////////////////////////////////////////////////////////////////////////////////////////////
70

71
  /** Append a fullPath (without wildcards) as a branch on the tree. */
72
  public void appendFullPath(PartialPath fullPath) {
73
    appendBranchWithoutPrune(root, fullPath.getNodes(), 0);
1✔
74
  }
1✔
75

76
  /** Append a fullPath consisting of device and measurement as a branch on the tree. */
77
  public void appendFullPath(PartialPath devicePath, String measurement) {
78
    int deviceNodeLength = devicePath.getNodeLength();
1✔
79
    String[] pathNodes = new String[deviceNodeLength + 1];
1✔
80
    System.arraycopy(devicePath.getNodes(), 0, pathNodes, 0, deviceNodeLength);
1✔
81
    pathNodes[deviceNodeLength] = measurement;
1✔
82

83
    appendBranchWithoutPrune(root, pathNodes, 0);
1✔
84
  }
1✔
85

86
  /** Add a pathPattern (may contain wildcards) to pathPatternList. */
87
  public void appendPathPattern(PartialPath pathPattern) {
88
    if (useWildcard) {
1✔
89
      boolean isExist = false;
1✔
90
      for (PartialPath path : pathPatternList) {
1✔
91
        if (path.include(pathPattern)) {
1✔
92
          // path already exists in pathPatternList
93
          isExist = true;
1✔
94
          break;
1✔
95
        }
96
      }
1✔
97
      if (!isExist) {
1✔
98
        // remove duplicate path in pathPatternList
99
        pathPatternList.removeIf(pathPattern::include);
1✔
100
        pathPatternList.add(pathPattern);
1✔
101
      }
102
    } else {
1✔
103
      appendBranchWithoutPrune(root, pathPattern.getNodes(), 0);
1✔
104
    }
105
  }
1✔
106

107
  /** Construct tree according to the pathPatternList. */
108
  public void constructTree() {
109
    for (PartialPath path : pathPatternList) {
1✔
110
      appendBranchWithoutPrune(root, path.getNodes(), 0);
1✔
111
    }
1✔
112
    pathPatternList.clear();
1✔
113
  }
1✔
114

115
  private void appendBranchWithoutPrune(
116
      PathPatternNode<Void, VoidSerializer> curNode, String[] pathNodes, int pos) {
117
    if (pos == pathNodes.length - 1) {
1✔
118
      curNode.markPathPattern(true);
1✔
119
      return;
1✔
120
    }
121

122
    PathPatternNode<Void, VoidSerializer> nextNode = curNode.getChildren(pathNodes[pos + 1]);
1✔
123

124
    if (nextNode != null) {
1✔
125
      appendBranchWithoutPrune(nextNode, pathNodes, pos + 1);
1✔
126
    } else {
127
      constructBranch(curNode, pathNodes, pos + 1);
1✔
128
    }
129
  }
1✔
130

131
  private void constructBranch(
132
      PathPatternNode<Void, VoidSerializer> curNode, String[] pathNodes, int pos) {
133
    for (int i = pos; i < pathNodes.length; i++) {
1✔
134
      PathPatternNode<Void, VoidSerializer> newNode =
1✔
135
          new PathPatternNode<>(pathNodes[i], VoidSerializer.getInstance());
1✔
136
      curNode.addChild(newNode);
1✔
137
      curNode = newNode;
1✔
138
    }
139
    curNode.markPathPattern(true);
1✔
140
  }
1✔
141

142
  /////////////////////////////////////////////////////////////////////////////////////////////////
143
  // Operations for querying tree
144
  /////////////////////////////////////////////////////////////////////////////////////////////////
145

146
  public boolean isEmpty() {
147
    return (root.getChildren() == null || root.getChildren().isEmpty())
×
148
        && (pathPatternList == null || pathPatternList.isEmpty());
×
149
  }
150

151
  public List<String> getAllDevicePatterns() {
152
    List<String> nodes = new ArrayList<>();
1✔
153
    Set<String> results = new HashSet<>();
1✔
154
    searchDevicePattern(root, nodes, results);
1✔
155
    return new ArrayList<>(results);
1✔
156
  }
157

158
  public List<PartialPath> getAllDevicePaths() {
159
    List<String> nodes = new ArrayList<>();
1✔
160
    Set<List<String>> resultNodesSet = new HashSet<>();
1✔
161
    searchDevicePath(root, nodes, resultNodesSet);
1✔
162

163
    Set<PartialPath> resultPaths = new HashSet<>();
1✔
164
    for (List<String> resultNodes : resultNodesSet) {
1✔
165
      resultPaths.add(new PartialPath(resultNodes.toArray(new String[0])));
1✔
166
    }
1✔
167
    return new ArrayList<>(resultPaths);
1✔
168
  }
169

170
  private void searchDevicePattern(
171
      PathPatternNode<Void, VoidSerializer> curNode, List<String> nodes, Set<String> results) {
172
    nodes.add(curNode.getName());
1✔
173
    if (curNode.isPathPattern()) {
1✔
174
      if (!curNode.getName().equals(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD)) {
1✔
175
        results.add(
1✔
176
            nodes.size() == 1 ? "" : convertNodesToString(nodes.subList(0, nodes.size() - 1)));
1✔
177
      } else {
178
        // the device of root.sg.d.** is root.sg.d and root.sg.d.**
179
        if (nodes.size() > 2) {
1✔
180
          results.add(convertNodesToString(nodes.subList(0, nodes.size() - 1)));
1✔
181
        }
182
        results.add(convertNodesToString(nodes));
1✔
183
      }
184
      if (curNode.isLeaf()) {
1✔
185
        nodes.remove(nodes.size() - 1);
1✔
186
        return;
1✔
187
      }
188
    }
189
    for (PathPatternNode<Void, VoidSerializer> childNode : curNode.getChildren().values()) {
1✔
190
      searchDevicePattern(childNode, nodes, results);
1✔
191
    }
1✔
192
    nodes.remove(nodes.size() - 1);
1✔
193
  }
1✔
194

195
  private void searchDevicePath(
196
      PathPatternNode<Void, VoidSerializer> curNode,
197
      List<String> nodes,
198
      Set<List<String>> resultNodesSet) {
199
    nodes.add(curNode.getName());
1✔
200
    if (curNode.isPathPattern()) {
1✔
201
      if (!curNode.getName().equals(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD)) {
1✔
202
        resultNodesSet.add(
1✔
203
            nodes.size() == 1
1✔
204
                ? new ArrayList<>()
×
205
                : new ArrayList<>(nodes.subList(0, nodes.size() - 1)));
1✔
206
      } else {
207
        // the device of root.sg.d.** is root.sg.d and root.sg.d.**
208
        if (nodes.size() > 2) {
1✔
209
          resultNodesSet.add(new ArrayList<>(nodes.subList(0, nodes.size() - 1)));
1✔
210
        }
211
        resultNodesSet.add(new ArrayList<>(nodes));
1✔
212
      }
213
      if (curNode.isLeaf()) {
1✔
214
        nodes.remove(nodes.size() - 1);
1✔
215
        return;
1✔
216
      }
217
    }
218
    for (PathPatternNode<Void, VoidSerializer> childNode : curNode.getChildren().values()) {
1✔
219
      searchDevicePath(childNode, nodes, resultNodesSet);
1✔
220
    }
1✔
221
    nodes.remove(nodes.size() - 1);
1✔
222
  }
1✔
223

224
  public List<PartialPath> getAllPathPatterns() {
225
    List<PartialPath> result = new ArrayList<>();
1✔
226
    Deque<String> ancestors = new ArrayDeque<>();
1✔
227
    searchPathPattern(root, ancestors, result);
1✔
228
    return result;
1✔
229
  }
230

231
  private void searchPathPattern(
232
      PathPatternNode<Void, VoidSerializer> node,
233
      Deque<String> ancestors,
234
      List<PartialPath> fullPaths) {
235
    if (node.isPathPattern()) {
1✔
236
      fullPaths.add(convertNodesToPartialPath(node, ancestors));
1✔
237
      if (node.isLeaf()) {
1✔
238
        return;
1✔
239
      }
240
    }
241

242
    ancestors.push(node.getName());
1✔
243
    for (PathPatternNode<Void, VoidSerializer> child : node.getChildren().values()) {
1✔
244
      searchPathPattern(child, ancestors, fullPaths);
1✔
245
    }
1✔
246
    ancestors.pop();
1✔
247
  }
1✔
248

249
  public List<PartialPath> getOverlappedPathPatterns(PartialPath pattern) {
250
    if (pathPatternList.isEmpty()) {
×
251
      pathPatternList = getAllPathPatterns();
×
252
    }
253

254
    List<PartialPath> results = new ArrayList<>();
×
255
    for (PartialPath path : pathPatternList) {
×
256
      if (pattern.overlapWith(path)) {
×
257
        results.add(path);
×
258
      }
259
    }
×
260
    return results;
×
261
  }
262

263
  private String convertNodesToString(List<String> nodes) {
264
    StringBuilder fullPathBuilder = new StringBuilder(nodes.get(0));
1✔
265
    for (int i = 1; i < nodes.size(); i++) {
1✔
266
      fullPathBuilder.append(TsFileConstant.PATH_SEPARATOR).append(nodes.get(i));
1✔
267
    }
268
    return fullPathBuilder.toString();
1✔
269
  }
270

271
  private PartialPath convertNodesToPartialPath(
272
      PathPatternNode<Void, VoidSerializer> node, Deque<String> ancestors) {
273
    Iterator<String> iterator = ancestors.descendingIterator();
1✔
274
    List<String> nodeList = new ArrayList<>(ancestors.size() + 1);
1✔
275
    while (iterator.hasNext()) {
1✔
276
      nodeList.add(iterator.next());
1✔
277
    }
278
    nodeList.add(node.getName());
1✔
279
    return new PartialPath(nodeList.toArray(new String[0]));
1✔
280
  }
281

282
  public boolean isOverlapWith(PathPatternTree patternTree) {
283
    // todo improve this implementation
284
    for (PartialPath pathPattern : getAllPathPatterns()) {
×
285
      if (!patternTree.getOverlappedPathPatterns(pathPattern).isEmpty()) {
×
286
        return true;
×
287
      }
288
    }
×
289
    return false;
×
290
  }
291

292
  /////////////////////////////////////////////////////////////////////////////////////////////////
293
  // serialize & deserialize
294
  /////////////////////////////////////////////////////////////////////////////////////////////////
295

296
  public void serialize(PublicBAOS outputStream) throws IOException {
297
    constructTree();
1✔
298
    root.serialize(outputStream);
1✔
299
  }
1✔
300

301
  public void serialize(DataOutputStream stream) throws IOException {
302
    constructTree();
1✔
303
    root.serialize(stream);
1✔
304
  }
1✔
305

306
  public void serialize(ByteBuffer buffer) {
307
    constructTree();
1✔
308
    root.serialize(buffer);
1✔
309
  }
1✔
310

311
  public static PathPatternTree deserialize(ByteBuffer buffer) {
312
    PathPatternNode<Void, VoidSerializer> root =
1✔
313
        PathPatternNode.deserializeNode(buffer, VoidSerializer.getInstance());
1✔
314
    PathPatternTree deserializedPatternTree = new PathPatternTree();
1✔
315
    deserializedPatternTree.setRoot(root);
1✔
316
    return deserializedPatternTree;
1✔
317
  }
318

319
  @Override
320
  public boolean equals(Object o) {
321
    if (this == o) return true;
1✔
322
    if (o == null || getClass() != o.getClass()) return false;
1✔
323
    PathPatternTree that = (PathPatternTree) o;
1✔
324
    return Objects.equals(root, that.root);
1✔
325
  }
326

327
  @Override
328
  public int hashCode() {
329
    return Objects.hash(root);
1✔
330
  }
331
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc