#9957

Committed 29 Aug 2023 11:56AM CUT coverage: 47.757%. Remained the same

Build # #9957

Build Type

push

travis_ci

Committed by

web-flow

Commit Message

Pipe: fix some minor concurrent issues & collect pipe's logs into a separate file (#10990)

* Pipe: fix some minor concurrent issues

* Logback: seperate pipe's logs

* enabled -> enable

* Update IoTDBConfig.java

Run Details

5 of 5 new or added lines in 4 files covered. (100.0%)

80368 of 168284 relevant lines covered (47.76%)

0.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

41.38

/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/IoTDBConnector.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.iotdb.db.pipe.connector.protocol;

import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.pipe.api.PipeConnector;
import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.session.util.SessionUtils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE;
import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY;
import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_IP_KEY;
import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_NODE_URLS_KEY;
import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PORT_KEY;

public abstract class IoTDBConnector implements PipeConnector {

  private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConnector.class);

  protected final List<TEndPoint> nodeUrls = new ArrayList<>();

  protected boolean isTabletBatchModeEnabled = true;

  @Override
  public void validate(PipeParameterValidator validator) throws Exception {
    final PipeParameters parameters = validator.getParameters();
    validator.validate(
        args -> (boolean) args[0] || ((boolean) args[1] && (boolean) args[2]),
        String.format(
            "Either %s or %s:%s must be specified",
            CONNECTOR_IOTDB_NODE_URLS_KEY, CONNECTOR_IOTDB_IP_KEY, CONNECTOR_IOTDB_PORT_KEY),
        parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY),
        parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY),
        parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY));
  }

  @Override
  public void customize(PipeParameters parameters, PipeConnectorRuntimeConfiguration configuration)
      throws Exception {
    final Set<TEndPoint> givenNodeUrls = new HashSet<>(nodeUrls);

    if (parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY)
        && parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY)) {
      givenNodeUrls.add(
          new TEndPoint(
              parameters.getString(CONNECTOR_IOTDB_IP_KEY),
              parameters.getInt(CONNECTOR_IOTDB_PORT_KEY)));
    }

    if (parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY)) {
      givenNodeUrls.addAll(
          SessionUtils.parseSeedNodeUrls(
              Arrays.asList(parameters.getString(CONNECTOR_IOTDB_NODE_URLS_KEY).split(","))));
    }

    nodeUrls.clear();
    nodeUrls.addAll(givenNodeUrls);
    LOGGER.info("IoTDBConnector nodeUrls: {}", nodeUrls);

    isTabletBatchModeEnabled =
        parameters.getBooleanOrDefault(
            CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY, CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE);
    LOGGER.info("IoTDBConnector isTabletBatchModeEnabled: {}", isTabletBatchModeEnabled);
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one
3	* or more contributor license agreements. See the NOTICE file
4	* distributed with this work for additional information
5	* regarding copyright ownership. The ASF licenses this file
6	* to you under the Apache License, Version 2.0 (the
7	* "License"); you may not use this file except in compliance
8	* with the License. You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing,
13	* software distributed under the License is distributed on an
14	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15	* KIND, either express or implied. See the License for the
16	* specific language governing permissions and limitations
17	* under the License.
18	*/
19
20	package org.apache.iotdb.db.pipe.connector.protocol;
21
22	import org.apache.iotdb.common.rpc.thrift.TEndPoint;
23	import org.apache.iotdb.pipe.api.PipeConnector;
24	import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
25	import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
26	import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
27	import org.apache.iotdb.session.util.SessionUtils;
28
29	import org.slf4j.Logger;
30	import org.slf4j.LoggerFactory;
31
32	import java.util.ArrayList;
33	import java.util.Arrays;
34	import java.util.HashSet;
35	import java.util.List;
36	import java.util.Set;
37
38	import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE;
39	import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY;
40	import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_IP_KEY;
41	import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_NODE_URLS_KEY;
42	import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_PORT_KEY;
43
44	public abstract class IoTDBConnector implements PipeConnector {	1✔
45
46	private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConnector.class);	1✔
47
48	protected final List<TEndPoint> nodeUrls = new ArrayList<>();	1✔
49
50	protected boolean isTabletBatchModeEnabled = true;	1✔
51
52	@Override
53	public void validate(PipeParameterValidator validator) throws Exception {
54	final PipeParameters parameters = validator.getParameters();	1✔
55	validator.validate(	1✔
56	args -> (boolean) args[0] \|\| ((boolean) args[1] && (boolean) args[2]),	1✔
57	String.format(	1✔
58	"Either %s or %s:%s must be specified",
59	CONNECTOR_IOTDB_NODE_URLS_KEY, CONNECTOR_IOTDB_IP_KEY, CONNECTOR_IOTDB_PORT_KEY),
60	parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY),	1✔
61	parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY),	1✔
62	parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY));	1✔
63	}	1✔
64
65	@Override
66	public void customize(PipeParameters parameters, PipeConnectorRuntimeConfiguration configuration)
67	throws Exception {
68	final Set<TEndPoint> givenNodeUrls = new HashSet<>(nodeUrls);	×
69
70	if (parameters.hasAttribute(CONNECTOR_IOTDB_IP_KEY)	×
71	&& parameters.hasAttribute(CONNECTOR_IOTDB_PORT_KEY)) {	×
72	givenNodeUrls.add(	×
73	new TEndPoint(
74	parameters.getString(CONNECTOR_IOTDB_IP_KEY),	×
75	parameters.getInt(CONNECTOR_IOTDB_PORT_KEY)));	×
76	}
77
78	if (parameters.hasAttribute(CONNECTOR_IOTDB_NODE_URLS_KEY)) {	×
79	givenNodeUrls.addAll(	×
80	SessionUtils.parseSeedNodeUrls(	×
81	Arrays.asList(parameters.getString(CONNECTOR_IOTDB_NODE_URLS_KEY).split(","))));	×
82	}
83
84	nodeUrls.clear();	×
85	nodeUrls.addAll(givenNodeUrls);	×
86	LOGGER.info("IoTDBConnector nodeUrls: {}", nodeUrls);	×
87
88	isTabletBatchModeEnabled =	×
89	parameters.getBooleanOrDefault(	×
90	CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY, CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE);
91	LOGGER.info("IoTDBConnector isTabletBatchModeEnabled: {}", isTabletBatchModeEnabled);	×
92	}	×
93	}

apache / iotdb / #9957

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous