#9905

Committed 23 Aug 2023 06:20AM UTC coverage: 47.785% (-0.1%) from 47.922%

Build # #9905

Build Type

push

travis_ci

Committed by

web-flow

Commit Message

[To rel/1.2][Metric] Fix flush point statistics (#10934)

Run Details

23 of 23 new or added lines in 1 file covered. (100.0%)

79851 of 167106 relevant lines covered (47.78%)

0.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.iotdb.db.service;

import org.apache.iotdb.commons.client.exception.ClientManagerException;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.schemaengine.SchemaEngineMode;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.db.storageengine.rescon.disk.DirectoryChecker;
import org.apache.iotdb.db.utils.MemUtils;
import org.apache.iotdb.rpc.TSStatusCode;

import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

public class IoTDBShutdownHook extends Thread {

  private static final Logger logger = LoggerFactory.getLogger(IoTDBShutdownHook.class);

  public IoTDBShutdownHook() {
    super(ThreadName.IOTDB_SHUTDOWN_HOOK.getName());
  }

  @Override
  public void run() {
    // close rocksdb if possible to avoid lose data
    if (SchemaEngineMode.valueOf(CommonDescriptor.getInstance().getConfig().getSchemaEngineMode())
        .equals(SchemaEngineMode.Rocksdb_based)) {
      SchemaEngine.getInstance().clear();
    }

    // reject write operations to make sure all tsfiles will be sealed
    CommonDescriptor.getInstance().getConfig().setStopping(true);
    CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
    // wait all wal are flushed
    WALManager.getInstance().waitAllWALFlushed();

    // flush data to Tsfile and remove WAL log files
    if (!IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
      StorageEngine.getInstance().syncCloseAllProcessor();
    }
    WALManager.getInstance().deleteOutdatedWALFiles();

    // We did this work because the RatisConsensus recovery mechanism is different from other
    // consensus algorithms, which will replace the underlying storage engine based on its
    // own
    // latest snapshot, while other consensus algorithms will not. This judgement ensures that
    // compaction work is not discarded even if there are frequent restarts
    if (IoTDBDescriptor.getInstance().getConfig().isClusterMode()
        && IoTDBDescriptor.getInstance()
            .getConfig()
            .getDataRegionConsensusProtocolClass()
            .equals(ConsensusFactory.RATIS_CONSENSUS)) {
      DataRegionConsensusImpl.getInstance()
          .getAllConsensusGroupIds()
          .parallelStream()
          .forEach(
              id -> {
                try {
                  DataRegionConsensusImpl.getInstance().triggerSnapshot(id);
                } catch (ConsensusException e) {
                  logger.warn(
                      "Something wrong happened while calling consensus layer's "
                          + "triggerSnapshot API.",
                      e);
                }
              });
    }

    // close consensusImpl
    try {
      SchemaRegionConsensusImpl.getInstance().stop();
      DataRegionConsensusImpl.getInstance().stop();
    } catch (IOException e) {
      logger.error("Stop ConsensusImpl error in IoTDBShutdownHook", e);
    }

    // clear lock file
    DirectoryChecker.getInstance().deregisterAll();

    // Set and report shutdown to cluster ConfigNode-leader
    CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Unknown);
    boolean isReportSuccess = false;
    try (ConfigNodeClient client =
        ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
      isReportSuccess =
          client.reportDataNodeShutdown(DataNode.generateDataNodeLocation()).getCode()
              == TSStatusCode.SUCCESS_STATUS.getStatusCode();
    } catch (ClientManagerException e) {
      logger.error("Failed to borrow ConfigNodeClient", e);
    } catch (TException e) {
      logger.error("Failed to report shutdown", e);
    }
    if (!isReportSuccess) {
      logger.error(
          "Reporting DataNode shutdown failed. The cluster will still take the current DataNode as Running for a few seconds.");
    }

    if (logger.isInfoEnabled()) {
      logger.info(
          "IoTDB exits. Jvm memory usage: {}",
          MemUtils.bytesCntToStr(
              Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()));
    }
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one
3	* or more contributor license agreements. See the NOTICE file
4	* distributed with this work for additional information
5	* regarding copyright ownership. The ASF licenses this file
6	* to you under the Apache License, Version 2.0 (the
7	* "License"); you may not use this file except in compliance
8	* with the License. You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing,
13	* software distributed under the License is distributed on an
14	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15	* KIND, either express or implied. See the License for the
16	* specific language governing permissions and limitations
17	* under the License.
18	*/
19
20	package org.apache.iotdb.db.service;
21
22	import org.apache.iotdb.commons.client.exception.ClientManagerException;
23	import org.apache.iotdb.commons.cluster.NodeStatus;
24	import org.apache.iotdb.commons.concurrent.ThreadName;
25	import org.apache.iotdb.commons.conf.CommonDescriptor;
26	import org.apache.iotdb.consensus.ConsensusFactory;
27	import org.apache.iotdb.consensus.exception.ConsensusException;
28	import org.apache.iotdb.db.conf.IoTDBDescriptor;
29	import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
30	import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl;
31	import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
32	import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
33	import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
34	import org.apache.iotdb.db.schemaengine.SchemaEngine;
35	import org.apache.iotdb.db.schemaengine.SchemaEngineMode;
36	import org.apache.iotdb.db.storageengine.StorageEngine;
37	import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
38	import org.apache.iotdb.db.storageengine.rescon.disk.DirectoryChecker;
39	import org.apache.iotdb.db.utils.MemUtils;
40	import org.apache.iotdb.rpc.TSStatusCode;
41
42	import org.apache.thrift.TException;
43	import org.slf4j.Logger;
44	import org.slf4j.LoggerFactory;
45
46	import java.io.IOException;
47
48	public class IoTDBShutdownHook extends Thread {
49
50	private static final Logger logger = LoggerFactory.getLogger(IoTDBShutdownHook.class);	×
51
52	public IoTDBShutdownHook() {
53	super(ThreadName.IOTDB_SHUTDOWN_HOOK.getName());	×
54	}	×
55
56	@Override
57	public void run() {
58	// close rocksdb if possible to avoid lose data
59	if (SchemaEngineMode.valueOf(CommonDescriptor.getInstance().getConfig().getSchemaEngineMode())	×
60	.equals(SchemaEngineMode.Rocksdb_based)) {	×
61	SchemaEngine.getInstance().clear();	×
62	}
63
64	// reject write operations to make sure all tsfiles will be sealed
65	CommonDescriptor.getInstance().getConfig().setStopping(true);	×
66	CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);	×
67	// wait all wal are flushed
68	WALManager.getInstance().waitAllWALFlushed();	×
69
70	// flush data to Tsfile and remove WAL log files
71	if (!IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {	×
72	StorageEngine.getInstance().syncCloseAllProcessor();	×
73	}
74	WALManager.getInstance().deleteOutdatedWALFiles();	×
75
76	// We did this work because the RatisConsensus recovery mechanism is different from other
77	// consensus algorithms, which will replace the underlying storage engine based on its
78	// own
79	// latest snapshot, while other consensus algorithms will not. This judgement ensures that
80	// compaction work is not discarded even if there are frequent restarts
81	if (IoTDBDescriptor.getInstance().getConfig().isClusterMode()	×
82	&& IoTDBDescriptor.getInstance()	×
83	.getConfig()	×
84	.getDataRegionConsensusProtocolClass()	×
85	.equals(ConsensusFactory.RATIS_CONSENSUS)) {	×
86	DataRegionConsensusImpl.getInstance()	×
87	.getAllConsensusGroupIds()	×
88	.parallelStream()	×
89	.forEach(	×
90	id -> {
91	try {
92	DataRegionConsensusImpl.getInstance().triggerSnapshot(id);	×
93	} catch (ConsensusException e) {	×
94	logger.warn(	×
95	"Something wrong happened while calling consensus layer's "
96	+ "triggerSnapshot API.",
97	e);
98	}	×
99	});	×
100	}
101
102	// close consensusImpl
103	try {
104	SchemaRegionConsensusImpl.getInstance().stop();	×
105	DataRegionConsensusImpl.getInstance().stop();	×
106	} catch (IOException e) {	×
107	logger.error("Stop ConsensusImpl error in IoTDBShutdownHook", e);	×
108	}	×
109
110	// clear lock file
111	DirectoryChecker.getInstance().deregisterAll();	×
112
113	// Set and report shutdown to cluster ConfigNode-leader
114	CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Unknown);	×
115	boolean isReportSuccess = false;	×
116	try (ConfigNodeClient client =
117	ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {	×
118	isReportSuccess =	×
119	client.reportDataNodeShutdown(DataNode.generateDataNodeLocation()).getCode()	×
120	== TSStatusCode.SUCCESS_STATUS.getStatusCode();	×
121	} catch (ClientManagerException e) {	×
122	logger.error("Failed to borrow ConfigNodeClient", e);	×
123	} catch (TException e) {	×
124	logger.error("Failed to report shutdown", e);	×
125	}	×
126	if (!isReportSuccess) {	×
127	logger.error(	×
128	"Reporting DataNode shutdown failed. The cluster will still take the current DataNode as Running for a few seconds.");
129	}
130
131	if (logger.isInfoEnabled()) {	×
132	logger.info(	×
133	"IoTDB exits. Jvm memory usage: {}",
134	MemUtils.bytesCntToStr(	×
135	Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()));	×
136	}
137	}	×
138	}

apache / iotdb / #9905

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous