#9919

Committed 25 Aug 2023 07:08AM UTC coverage: 47.802% (+0.007%) from 47.795%

Build # #9919

Build Type

push

travis_ci

Committed by

web-flow

Commit Message

Remove some useless configs (#10950)

Run Details

80023 of 167404 relevant lines covered (47.8%)

0.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.57

/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.iotdb.db.utils;

import org.apache.iotdb.commons.path.AlignedPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.pipe.agent.PipeAgent;
import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey;
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.DiskAlignedChunkMetadataLoader;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.DiskChunkMetadataLoader;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.MemAlignedChunkMetadataLoader;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.MemChunkMetadataLoader;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.IChunkReader;
import org.apache.iotdb.tsfile.read.reader.IPageReader;
import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_ALIGNED_DISK;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_ALIGNED_MEM;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_DISK;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_MEM;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_ALIGNED;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_NONALIGNED;

public class FileLoaderUtils {

  private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
      SeriesScanCostMetricSet.getInstance();

  private FileLoaderUtils() {
    // empty constructor
  }

  public static void updateTsFileResource(
      TsFileSequenceReader reader, TsFileResource tsFileResource) throws IOException {
    updateTsFileResource(reader.getAllTimeseriesMetadata(false), tsFileResource);
    tsFileResource.updatePlanIndexes(reader.getMinPlanIndex());
    tsFileResource.updatePlanIndexes(reader.getMaxPlanIndex());
  }

  public static void updateTsFileResource(
      Map<String, List<TimeseriesMetadata>> device2Metadata, TsFileResource tsFileResource) {
    for (Entry<String, List<TimeseriesMetadata>> entry : device2Metadata.entrySet()) {
      for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) {
        tsFileResource.updateStartTime(
            entry.getKey(), timeseriesMetaData.getStatistics().getStartTime());
        tsFileResource.updateEndTime(
            entry.getKey(), timeseriesMetaData.getStatistics().getEndTime());
      }
    }
  }

  /**
   * Generate {@link TsFileResource} from a closed {@link TsFileIOWriter}. Notice that the writer
   * should have executed {@link TsFileIOWriter#endFile()}. And this method will not record plan
   * Index of this writer.
   *
   * @param writer a {@link TsFileIOWriter}
   * @return a updated {@link TsFileResource}
   */
  public static TsFileResource generateTsFileResource(TsFileIOWriter writer) {
    TsFileResource resource = new TsFileResource(writer.getFile());
    for (ChunkGroupMetadata chunkGroupMetadata : writer.getChunkGroupMetadataList()) {
      String device = chunkGroupMetadata.getDevice();
      for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) {
        resource.updateStartTime(device, chunkMetadata.getStartTime());
        resource.updateEndTime(device, chunkMetadata.getEndTime());
      }
    }
    resource.setStatus(TsFileResourceStatus.NORMAL);
    PipeAgent.runtime().assignRecoverProgressIndexForTsFileRecovery(resource);
    return resource;
  }

  /**
   * Load TimeSeriesMetadata for non-aligned time series
   *
   * @param resource TsFile
   * @param seriesPath Timeseries path
   * @param allSensors measurements queried at the same time of this device
   * @param filter any filter, only used to check time range
   * @throws IOException IOException may be thrown while reading it from disk.
   */
  @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
  public static TimeseriesMetadata loadTimeSeriesMetadata(
      TsFileResource resource,
      PartialPath seriesPath,
      QueryContext context,
      Filter filter,
      Set<String> allSensors)
      throws IOException {
    long t1 = System.nanoTime();
    boolean loadFromMem = false;
    try {
      // common path
      TimeseriesMetadata timeSeriesMetadata;
      // If the tsfile is closed, we need to load from tsfile
      if (resource.isClosed()) {
        // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
        // we should not ignore the non-exist of device in TsFileMetadata
        timeSeriesMetadata =
            TimeSeriesMetadataCache.getInstance()
                .get(
                    new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
                        resource.getTsFilePath(),
                        seriesPath.getDevice(),
                        seriesPath.getMeasurement()),
                    allSensors,
                    resource.getTimeIndexType() != 1,
                    context.isDebug());
        if (timeSeriesMetadata != null) {
          timeSeriesMetadata.setChunkMetadataLoader(
              new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
        }
      } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
        loadFromMem = true;

        timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
        if (timeSeriesMetadata != null) {
          timeSeriesMetadata.setChunkMetadataLoader(
              new MemChunkMetadataLoader(resource, seriesPath, context, filter));
        }
      }

      if (timeSeriesMetadata != null) {
        long t2 = System.nanoTime();
        try {
          List<Modification> pathModifications =
              context.getPathModifications(resource.getModFile(), seriesPath);
          timeSeriesMetadata.setModified(!pathModifications.isEmpty());
          if (timeSeriesMetadata.getStatistics().getStartTime()
              > timeSeriesMetadata.getStatistics().getEndTime()) {
            return null;
          }
          if (filter != null
              && !filter.satisfyStartEndTime(
                  timeSeriesMetadata.getStatistics().getStartTime(),
                  timeSeriesMetadata.getStatistics().getEndTime())) {
            return null;
          }
        } finally {
          SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
              TIMESERIES_METADATA_MODIFICATION_NONALIGNED, System.nanoTime() - t2);
        }
      }
      return timeSeriesMetadata;
    } finally {
      SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
          loadFromMem
              ? LOAD_TIMESERIES_METADATA_NONALIGNED_MEM
              : LOAD_TIMESERIES_METADATA_NONALIGNED_DISK,
          System.nanoTime() - t1);
    }
  }

  /**
   * Load AlignedTimeSeriesMetadata for aligned time series.
   *
   * @param resource corresponding TsFileResource
   * @param alignedPath instance of VectorPartialPath, vector's full path, e.g. (root.sg1.d1.vector,
   *     [root.sg1.d1.vector.s1, root.sg1.d1.vector.s2])
   * @throws IOException IOException may be thrown while reading it from disk.
   */
  public static AlignedTimeSeriesMetadata loadTimeSeriesMetadata(
      TsFileResource resource,
      AlignedPath alignedPath,
      QueryContext context,
      Filter filter,
      boolean queryAllSensors)
      throws IOException {
    final long t1 = System.nanoTime();
    boolean loadFromMem = false;
    try {
      AlignedTimeSeriesMetadata alignedTimeSeriesMetadata;
      // If the tsfile is closed, we need to load from tsfile
      if (resource.isClosed()) {
        alignedTimeSeriesMetadata =
            loadFromDisk(resource, alignedPath, context, filter, queryAllSensors);
      } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
        loadFromMem = true;
        alignedTimeSeriesMetadata =
            (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(alignedPath);
        if (alignedTimeSeriesMetadata != null) {
          alignedTimeSeriesMetadata.setChunkMetadataLoader(
              new MemAlignedChunkMetadataLoader(
                  resource, alignedPath, context, filter, queryAllSensors));
        }
      }

      if (alignedTimeSeriesMetadata != null) {
        final long t2 = System.nanoTime();
        try {
          if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()
              > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {
            return null;
          }
          if (filter != null
              && !filter.satisfyStartEndTime(
                  alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(),
                  alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) {
            return null;
          }

          // set modifications to each aligned path
          setModifications(resource, alignedTimeSeriesMetadata, alignedPath, context);
        } finally {
          SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
              TIMESERIES_METADATA_MODIFICATION_ALIGNED, System.nanoTime() - t2);
        }
      }
      return alignedTimeSeriesMetadata;
    } finally {
      SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
          loadFromMem
              ? LOAD_TIMESERIES_METADATA_ALIGNED_MEM
              : LOAD_TIMESERIES_METADATA_ALIGNED_DISK,
          System.nanoTime() - t1);
    }
  }

  private static AlignedTimeSeriesMetadata loadFromDisk(
      TsFileResource resource,
      AlignedPath alignedPath,
      QueryContext context,
      Filter filter,
      boolean queryAllSensors)
      throws IOException {
    AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
    // load all the TimeseriesMetadata of vector, the first one is for time column and the
    // remaining is for sub sensors
    // the order of timeSeriesMetadata list is same as subSensorList's order
    TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
    List<String> valueMeasurementList = alignedPath.getMeasurementList();
    Set<String> allSensors = new HashSet<>(valueMeasurementList);
    allSensors.add("");
    boolean isDebug = context.isDebug();
    String filePath = resource.getTsFilePath();
    String deviceId = alignedPath.getDevice();

    // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
    // we should not ignore the non-exist of device in TsFileMetadata
    TimeseriesMetadata timeColumn =
        cache.get(
            new TimeSeriesMetadataCacheKey(filePath, deviceId, ""),
            allSensors,
            resource.getTimeIndexType() != 1,
            isDebug);
    if (timeColumn != null) {
      // only need time column, like count_time aggregation
      if (valueMeasurementList.isEmpty()) {
        alignedTimeSeriesMetadata =
            new AlignedTimeSeriesMetadata(timeColumn, Collections.emptyList());
        alignedTimeSeriesMetadata.setChunkMetadataLoader(
            new DiskAlignedChunkMetadataLoader(
                resource, alignedPath, context, filter, queryAllSensors));
      } else {
        List<TimeseriesMetadata> valueTimeSeriesMetadataList =
            new ArrayList<>(valueMeasurementList.size());
        // if all the queried aligned sensors does not exist, we will return null
        boolean exist = false;
        for (String valueMeasurement : valueMeasurementList) {
          TimeseriesMetadata valueColumn =
              cache.get(
                  new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement),
                  allSensors,
                  resource.getTimeIndexType() != 1,
                  isDebug);
          exist = (exist || (valueColumn != null));
          valueTimeSeriesMetadataList.add(valueColumn);
        }
        if (exist) {
          alignedTimeSeriesMetadata =
              new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList);
          alignedTimeSeriesMetadata.setChunkMetadataLoader(
              new DiskAlignedChunkMetadataLoader(
                  resource, alignedPath, context, filter, queryAllSensors));
        }
      }
    }
    return alignedTimeSeriesMetadata;
  }

  private static void setModifications(
      TsFileResource resource,
      AlignedTimeSeriesMetadata alignedTimeSeriesMetadata,
      AlignedPath alignedPath,
      QueryContext context) {
    List<TimeseriesMetadata> valueTimeSeriesMetadataList =
        alignedTimeSeriesMetadata.getValueTimeseriesMetadataList();
    boolean modified = false;
    for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
      if (valueTimeSeriesMetadataList.get(i) != null) {
        List<Modification> pathModifications =
            context.getPathModifications(
                resource.getModFile(), alignedPath.getPathWithMeasurement(i));
        valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty());
        modified = (modified || !pathModifications.isEmpty());
      }
    }
    alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified);
  }

  /**
   * load all chunk metadata of one time series in one file.
   *
   * @param timeSeriesMetadata the corresponding TimeSeriesMetadata in that file.
   */
  public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) {
    return timeSeriesMetadata.loadChunkMetadataList();
  }

  /**
   * load all page readers in one chunk that satisfying the timeFilter.
   *
   * @param chunkMetaData the corresponding chunk metadata
   * @param timeFilter it should be a TimeFilter instead of a ValueFilter
   * @throws IOException if chunkMetaData is null or errors happened while loading page readers,
   *     IOException will be thrown
   */
  public static List<IPageReader> loadPageReaderList(
      IChunkMetadata chunkMetaData, Filter timeFilter) throws IOException {
    if (chunkMetaData == null) {
      throw new IOException("Can't init null chunkMeta");
    }
    IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
    IChunkReader chunkReader = chunkLoader.getChunkReader(chunkMetaData, timeFilter);
    return chunkReader.loadPageReaderList();
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one
3	* or more contributor license agreements. See the NOTICE file
4	* distributed with this work for additional information
5	* regarding copyright ownership. The ASF licenses this file
6	* to you under the Apache License, Version 2.0 (the
7	* "License"); you may not use this file except in compliance
8	* with the License. You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing,
13	* software distributed under the License is distributed on an
14	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15	* KIND, either express or implied. See the License for the
16	* specific language governing permissions and limitations
17	* under the License.
18	*/
19
20	package org.apache.iotdb.db.utils;
21
22	import org.apache.iotdb.commons.path.AlignedPath;
23	import org.apache.iotdb.commons.path.PartialPath;
24	import org.apache.iotdb.db.pipe.agent.PipeAgent;
25	import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
26	import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
27	import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
28	import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey;
29	import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
30	import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.DiskAlignedChunkMetadataLoader;
31	import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.DiskChunkMetadataLoader;
32	import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.MemAlignedChunkMetadataLoader;
33	import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata.MemChunkMetadataLoader;
34	import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
35	import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
36	import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata;
37	import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
38	import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
39	import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
40	import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
41	import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
42	import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
43	import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
44	import org.apache.iotdb.tsfile.read.filter.basic.Filter;
45	import org.apache.iotdb.tsfile.read.reader.IChunkReader;
46	import org.apache.iotdb.tsfile.read.reader.IPageReader;
47	import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
48
49	import java.io.IOException;
50	import java.util.ArrayList;
51	import java.util.Collections;
52	import java.util.HashSet;
53	import java.util.List;
54	import java.util.Map;
55	import java.util.Map.Entry;
56	import java.util.Set;
57
58	import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_ALIGNED_DISK;
59	import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_ALIGNED_MEM;
60	import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_DISK;
61	import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_MEM;
62	import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_ALIGNED;
63	import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_NONALIGNED;
64
65	public class FileLoaderUtils {
66
67	private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =	1✔
68	SeriesScanCostMetricSet.getInstance();	1✔
69
70	private FileLoaderUtils() {
71	// empty constructor
72	}
73
74	public static void updateTsFileResource(
75	TsFileSequenceReader reader, TsFileResource tsFileResource) throws IOException {
76	updateTsFileResource(reader.getAllTimeseriesMetadata(false), tsFileResource);	1✔
77	tsFileResource.updatePlanIndexes(reader.getMinPlanIndex());	1✔
78	tsFileResource.updatePlanIndexes(reader.getMaxPlanIndex());	1✔
79	}	1✔
80
81	public static void updateTsFileResource(
82	Map<String, List<TimeseriesMetadata>> device2Metadata, TsFileResource tsFileResource) {
83	for (Entry<String, List<TimeseriesMetadata>> entry : device2Metadata.entrySet()) {	1✔
84	for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) {	1✔
85	tsFileResource.updateStartTime(	1✔
86	entry.getKey(), timeseriesMetaData.getStatistics().getStartTime());	1✔
87	tsFileResource.updateEndTime(	1✔
88	entry.getKey(), timeseriesMetaData.getStatistics().getEndTime());	1✔
89	}	1✔
90	}	1✔
91	}	1✔
92
93	/**
94	* Generate {@link TsFileResource} from a closed {@link TsFileIOWriter}. Notice that the writer
95	* should have executed {@link TsFileIOWriter#endFile()}. And this method will not record plan
96	* Index of this writer.
97	*
98	* @param writer a {@link TsFileIOWriter}
99	* @return a updated {@link TsFileResource}
100	*/
101	public static TsFileResource generateTsFileResource(TsFileIOWriter writer) {
102	TsFileResource resource = new TsFileResource(writer.getFile());	×
103	for (ChunkGroupMetadata chunkGroupMetadata : writer.getChunkGroupMetadataList()) {	×
104	String device = chunkGroupMetadata.getDevice();	×
105	for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) {	×
106	resource.updateStartTime(device, chunkMetadata.getStartTime());	×
107	resource.updateEndTime(device, chunkMetadata.getEndTime());	×
108	}	×
109	}	×
110	resource.setStatus(TsFileResourceStatus.NORMAL);	×
111	PipeAgent.runtime().assignRecoverProgressIndexForTsFileRecovery(resource);	×
112	return resource;	×
113	}
114
115	/**
116	* Load TimeSeriesMetadata for non-aligned time series
117	*
118	* @param resource TsFile
119	* @param seriesPath Timeseries path
120	* @param allSensors measurements queried at the same time of this device
121	* @param filter any filter, only used to check time range
122	* @throws IOException IOException may be thrown while reading it from disk.
123	*/
124	@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
125	public static TimeseriesMetadata loadTimeSeriesMetadata(
126	TsFileResource resource,
127	PartialPath seriesPath,
128	QueryContext context,
129	Filter filter,
130	Set<String> allSensors)
131	throws IOException {
132	long t1 = System.nanoTime();	1✔
133	boolean loadFromMem = false;	1✔
134	try {
135	// common path
136	TimeseriesMetadata timeSeriesMetadata;
137	// If the tsfile is closed, we need to load from tsfile
138	if (resource.isClosed()) {	1✔
139	// when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
140	// we should not ignore the non-exist of device in TsFileMetadata
141	timeSeriesMetadata =
142	TimeSeriesMetadataCache.getInstance()	1✔
143	.get(	1✔
144	new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
145	resource.getTsFilePath(),	1✔
146	seriesPath.getDevice(),	1✔
147	seriesPath.getMeasurement()),	1✔
148	allSensors,
149	resource.getTimeIndexType() != 1,	1✔
150	context.isDebug());	1✔
151	if (timeSeriesMetadata != null) {	1✔
152	timeSeriesMetadata.setChunkMetadataLoader(	1✔
153	new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
154	}
155	} else { // if the tsfile is unclosed, we just get it directly from TsFileResource
156	loadFromMem = true;	×
157
158	timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);	×
159	if (timeSeriesMetadata != null) {	×
160	timeSeriesMetadata.setChunkMetadataLoader(	×
161	new MemChunkMetadataLoader(resource, seriesPath, context, filter));
162	}
163	}
164
165	if (timeSeriesMetadata != null) {	1✔
166	long t2 = System.nanoTime();	1✔
167	try {
168	List<Modification> pathModifications =	1✔
169	context.getPathModifications(resource.getModFile(), seriesPath);	1✔
170	timeSeriesMetadata.setModified(!pathModifications.isEmpty());	1✔
171	if (timeSeriesMetadata.getStatistics().getStartTime()	1✔
172	> timeSeriesMetadata.getStatistics().getEndTime()) {	1✔
173	return null;	×
174	}
175	if (filter != null	1✔
176	&& !filter.satisfyStartEndTime(	1✔
177	timeSeriesMetadata.getStatistics().getStartTime(),	1✔
178	timeSeriesMetadata.getStatistics().getEndTime())) {	1✔
179	return null;	×
180	}
181	} finally {
182	SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(	1✔
183	TIMESERIES_METADATA_MODIFICATION_NONALIGNED, System.nanoTime() - t2);	1✔
184	}
185	}
186	return timeSeriesMetadata;	1✔
187	} finally {
188	SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(	1✔
189	loadFromMem	1✔
190	? LOAD_TIMESERIES_METADATA_NONALIGNED_MEM	×
191	: LOAD_TIMESERIES_METADATA_NONALIGNED_DISK,	1✔
192	System.nanoTime() - t1);	1✔
193	}
194	}
195
196	/**
197	* Load AlignedTimeSeriesMetadata for aligned time series.
198	*
199	* @param resource corresponding TsFileResource
200	* @param alignedPath instance of VectorPartialPath, vector's full path, e.g. (root.sg1.d1.vector,
201	* [root.sg1.d1.vector.s1, root.sg1.d1.vector.s2])
202	* @throws IOException IOException may be thrown while reading it from disk.
203	*/
204	public static AlignedTimeSeriesMetadata loadTimeSeriesMetadata(
205	TsFileResource resource,
206	AlignedPath alignedPath,
207	QueryContext context,
208	Filter filter,
209	boolean queryAllSensors)
210	throws IOException {
211	final long t1 = System.nanoTime();	1✔
212	boolean loadFromMem = false;	1✔
213	try {
214	AlignedTimeSeriesMetadata alignedTimeSeriesMetadata;
215	// If the tsfile is closed, we need to load from tsfile
216	if (resource.isClosed()) {	1✔
217	alignedTimeSeriesMetadata =	1✔
218	loadFromDisk(resource, alignedPath, context, filter, queryAllSensors);	1✔
219	} else { // if the tsfile is unclosed, we just get it directly from TsFileResource
220	loadFromMem = true;	×
221	alignedTimeSeriesMetadata =	×
222	(AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(alignedPath);	×
223	if (alignedTimeSeriesMetadata != null) {	×
224	alignedTimeSeriesMetadata.setChunkMetadataLoader(	×
225	new MemAlignedChunkMetadataLoader(
226	resource, alignedPath, context, filter, queryAllSensors));
227	}
228	}
229
230	if (alignedTimeSeriesMetadata != null) {	1✔
231	final long t2 = System.nanoTime();	1✔
232	try {
233	if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()	1✔
234	> alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {	1✔
235	return null;	×
236	}
237	if (filter != null	1✔
238	&& !filter.satisfyStartEndTime(	1✔
239	alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(),	1✔
240	alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) {	1✔
241	return null;	×
242	}
243
244	// set modifications to each aligned path
245	setModifications(resource, alignedTimeSeriesMetadata, alignedPath, context);	1✔
246	} finally {
247	SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(	1✔
248	TIMESERIES_METADATA_MODIFICATION_ALIGNED, System.nanoTime() - t2);	1✔
249	}
250	}
251	return alignedTimeSeriesMetadata;	1✔
252	} finally {
253	SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(	1✔
254	loadFromMem	1✔
255	? LOAD_TIMESERIES_METADATA_ALIGNED_MEM	×
256	: LOAD_TIMESERIES_METADATA_ALIGNED_DISK,	1✔
257	System.nanoTime() - t1);	1✔
258	}
259	}
260
261	private static AlignedTimeSeriesMetadata loadFromDisk(
262	TsFileResource resource,
263	AlignedPath alignedPath,
264	QueryContext context,
265	Filter filter,
266	boolean queryAllSensors)
267	throws IOException {
268	AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;	1✔
269	// load all the TimeseriesMetadata of vector, the first one is for time column and the
270	// remaining is for sub sensors
271	// the order of timeSeriesMetadata list is same as subSensorList's order
272	TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();	1✔
273	List<String> valueMeasurementList = alignedPath.getMeasurementList();	1✔
274	Set<String> allSensors = new HashSet<>(valueMeasurementList);	1✔
275	allSensors.add("");	1✔
276	boolean isDebug = context.isDebug();	1✔
277	String filePath = resource.getTsFilePath();	1✔
278	String deviceId = alignedPath.getDevice();	1✔
279
280	// when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
281	// we should not ignore the non-exist of device in TsFileMetadata
282	TimeseriesMetadata timeColumn =	1✔
283	cache.get(	1✔
284	new TimeSeriesMetadataCacheKey(filePath, deviceId, ""),
285	allSensors,
286	resource.getTimeIndexType() != 1,	1✔
287	isDebug);
288	if (timeColumn != null) {	1✔
289	// only need time column, like count_time aggregation
290	if (valueMeasurementList.isEmpty()) {	1✔
291	alignedTimeSeriesMetadata =	×
292	new AlignedTimeSeriesMetadata(timeColumn, Collections.emptyList());	×
293	alignedTimeSeriesMetadata.setChunkMetadataLoader(	×
294	new DiskAlignedChunkMetadataLoader(
295	resource, alignedPath, context, filter, queryAllSensors));
296	} else {
297	List<TimeseriesMetadata> valueTimeSeriesMetadataList =	1✔
298	new ArrayList<>(valueMeasurementList.size());	1✔
299	// if all the queried aligned sensors does not exist, we will return null
300	boolean exist = false;	1✔
301	for (String valueMeasurement : valueMeasurementList) {	1✔
302	TimeseriesMetadata valueColumn =	1✔
303	cache.get(	1✔
304	new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement),
305	allSensors,
306	resource.getTimeIndexType() != 1,	1✔
307	isDebug);
308	exist = (exist \|\| (valueColumn != null));	1✔
309	valueTimeSeriesMetadataList.add(valueColumn);	1✔
310	}	1✔
311	if (exist) {	1✔
312	alignedTimeSeriesMetadata =	1✔
313	new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList);
314	alignedTimeSeriesMetadata.setChunkMetadataLoader(	1✔
315	new DiskAlignedChunkMetadataLoader(
316	resource, alignedPath, context, filter, queryAllSensors));
317	}
318	}
319	}
320	return alignedTimeSeriesMetadata;	1✔
321	}
322
323	private static void setModifications(
324	TsFileResource resource,
325	AlignedTimeSeriesMetadata alignedTimeSeriesMetadata,
326	AlignedPath alignedPath,
327	QueryContext context) {
328	List<TimeseriesMetadata> valueTimeSeriesMetadataList =	1✔
329	alignedTimeSeriesMetadata.getValueTimeseriesMetadataList();	1✔
330	boolean modified = false;	1✔
331	for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {	1✔
332	if (valueTimeSeriesMetadataList.get(i) != null) {	1✔
333	List<Modification> pathModifications =	1✔
334	context.getPathModifications(	1✔
335	resource.getModFile(), alignedPath.getPathWithMeasurement(i));	1✔
336	valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty());	1✔
337	modified = (modified \|\| !pathModifications.isEmpty());	1✔
338	}
339	}
340	alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified);	1✔
341	}	1✔
342
343	/**
344	* load all chunk metadata of one time series in one file.
345	*
346	* @param timeSeriesMetadata the corresponding TimeSeriesMetadata in that file.
347	*/
348	public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) {
349	return timeSeriesMetadata.loadChunkMetadataList();	1✔
350	}
351
352	/**
353	* load all page readers in one chunk that satisfying the timeFilter.
354	*
355	* @param chunkMetaData the corresponding chunk metadata
356	* @param timeFilter it should be a TimeFilter instead of a ValueFilter
357	* @throws IOException if chunkMetaData is null or errors happened while loading page readers,
358	* IOException will be thrown
359	*/
360	public static List<IPageReader> loadPageReaderList(
361	IChunkMetadata chunkMetaData, Filter timeFilter) throws IOException {
362	if (chunkMetaData == null) {	1✔
363	throw new IOException("Can't init null chunkMeta");	×
364	}
365	IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();	1✔
366	IChunkReader chunkReader = chunkLoader.getChunkReader(chunkMetaData, timeFilter);	1✔
367	return chunkReader.loadPageReaderList();	1✔
368	}
369	}

apache / iotdb / #9919

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous