2436

Build # 2436

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/688d78cee3c51835a8b3dd4abbb00ea411987ac0">688d78cee<a href="https://github.com/apache/carbondata/commit/688d78cee3c51835a8b3dd4abbb00ea411987ac0">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/3cbe239f54b21ed5ab63b536226326b73f2f11db">3cbe239f5</a>">3cbe239f5</a><a href="https://github.com/apache/carbondata/commit/688d78cee3c51835a8b3dd4abbb00ea411987ac0"> into 4e27b86df">4e27b86df</a>

Run Details

3 of 3 new or added lines in 2 files covered. (100.0%)

62214 of 78578 relevant lines covered (79.17%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.11

/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.processing.loading.model;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.apache.carbondata.common.Maps;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.loading.ComplexDelimitersEnum;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
import org.apache.carbondata.processing.util.CarbonLoaderUtil;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.log4j.Logger;

/**
 * Provide utilities to populate loading options
 */
@InterfaceAudience.Internal
public class LoadOption {

  private static final Logger LOG = LogServiceFactory.getLogService(LoadOption.class.getName());

  /**
   * Based on the input options, fill and return data loading options with default value
   */
  public static Map<String, String> fillOptionWithDefaultValue(
      Map<String, String> options) throws InvalidLoadOptionException {
    Map<String, String> optionsFinal = new HashMap<>();
    optionsFinal.put("delimiter", Maps.getOrDefault(options, "delimiter", ","));
    optionsFinal.put("quotechar", Maps.getOrDefault(options, "quotechar", "\""));
    optionsFinal.put("fileheader", Maps.getOrDefault(options, "fileheader", ""));
    optionsFinal.put("commentchar", Maps.getOrDefault(options, "commentchar", "#"));
    optionsFinal.put("columndict", Maps.getOrDefault(options, "columndict", null));

    optionsFinal.put(
        "escapechar",
        CarbonLoaderUtil.getEscapeChar(Maps.getOrDefault(options,"escapechar", "\\")));

    optionsFinal.put(
        "serialization_null_format",
        Maps.getOrDefault(options, "serialization_null_format", "\\N"));

    optionsFinal.put(
        "bad_records_logger_enable",
        Maps.getOrDefault(
            options,
            "bad_records_logger_enable",
            CarbonProperties.getInstance().getProperty(
                CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
                CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT)));

    String badRecordActionValue = CarbonProperties.getInstance().getProperty(
        CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
        CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT);

    optionsFinal.put(
        "bad_records_action",
        Maps.getOrDefault(
            options,
            "bad_records_action",
            CarbonProperties.getInstance().getProperty(
                CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
                badRecordActionValue)));

    optionsFinal.put(
        "is_empty_data_bad_record",
        Maps.getOrDefault(
            options,
            "is_empty_data_bad_record",
            CarbonProperties.getInstance().getProperty(
                CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
                CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT)));

    optionsFinal.put(
        "skip_empty_line",
        Maps.getOrDefault(
            options,
            "skip_empty_line",
            CarbonProperties.getInstance().getProperty(
                CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));

    optionsFinal.put(
        "all_dictionary_path",
        Maps.getOrDefault(options, "all_dictionary_path", ""));

    optionsFinal.put("complex_delimiter_level_1",
        Maps.getOrDefault(options, "complex_delimiter_level_1",
            ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_1.value()));

    optionsFinal.put("complex_delimiter_level_2",
        Maps.getOrDefault(options, "complex_delimiter_level_2",
            ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_2.value()));

    optionsFinal.put("complex_delimiter_level_3",
        Maps.getOrDefault(options, "complex_delimiter_level_3",
            ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_3.value()));

    optionsFinal.put(
        "dateformat",
        Maps.getOrDefault(
            options,
            "dateformat",
            CarbonProperties.getInstance().getProperty(
                CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
                CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)));

    optionsFinal.put(
        "timestampformat",
        Maps.getOrDefault(
            options,
            "timestampformat",
            CarbonProperties.getInstance().getProperty(
                CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT,
                CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT)));

    optionsFinal.put(
        "global_sort_partitions",
        Maps.getOrDefault(
            options,
            "global_sort_partitions",
            CarbonProperties.getInstance().getProperty(
                CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
                null)));

    optionsFinal.put("maxcolumns", Maps.getOrDefault(options, "maxcolumns", null));

    optionsFinal.put(
        "batch_sort_size_inmb",
        Maps.getOrDefault(
            options,
            "batch_sort_size_inmb",
            CarbonProperties.getInstance().getProperty(
                CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
                CarbonProperties.getInstance().getProperty(
                    CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
                    CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))));

    String useOnePass = Maps.getOrDefault(
        options,
        "single_pass",
        CarbonProperties.getInstance().getProperty(
            CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
            CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT)).trim().toLowerCase();

    boolean singlePass;

    if (useOnePass.equalsIgnoreCase("true")) {
      singlePass = true;
    } else {
      // when single_pass = false  and if either alldictionarypath
      // or columnDict is configured the do not allow load
      if (StringUtils.isNotEmpty(optionsFinal.get("all_dictionary_path")) ||
          StringUtils.isNotEmpty(optionsFinal.get("columndict"))) {
        throw new InvalidLoadOptionException(
            "Can not use all_dictionary_path or columndict without single_pass.");
      } else {
        singlePass = false;
      }
    }

    optionsFinal.put("single_pass", String.valueOf(singlePass));
    optionsFinal.put("sort_scope", CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT);
    optionsFinal.put("sort_column_bounds", Maps.getOrDefault(options, "sort_column_bounds", ""));
    optionsFinal.put(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB,
        Maps.getOrDefault(options, CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB,
            CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB_DEFAULT));

    optionsFinal.put("range_column", Maps.getOrDefault(options, "range_column", null));
    optionsFinal.put("scale_factor", Maps.getOrDefault(options, "scale_factor", null));
    return optionsFinal;
  }

  /**
   * Return CSV header field names
   */
  public static String[] getCsvHeaderColumns(
      CarbonLoadModel carbonLoadModel,
      Configuration hadoopConf) throws IOException {
    return getCsvHeaderColumns(carbonLoadModel, hadoopConf, new LinkedList<String>());
  }

  /**
   * Return CSV header field names, with partition column
   */
  public static String[] getCsvHeaderColumns(
      CarbonLoadModel carbonLoadModel,
      Configuration hadoopConf,
      List<String> staticPartitionCols) throws IOException {
    String delimiter;
    if (StringUtils.isEmpty(carbonLoadModel.getCsvDelimiter())) {
      delimiter = CarbonCommonConstants.COMMA;
    } else {
      delimiter = CarbonUtil.delimiterConverter(carbonLoadModel.getCsvDelimiter());
    }
    String csvFile = null;
    String csvHeader = carbonLoadModel.getCsvHeader();
    String[] csvColumns;
    if (StringUtils.isBlank(csvHeader)) {
      // read header from csv file
      csvFile = carbonLoadModel.getFactFilePath().split(",")[0];
      csvHeader = CarbonUtil.readHeader(csvFile, hadoopConf);
      if (StringUtils.isBlank(csvHeader)) {
        throw new CarbonDataLoadingException("First line of the csv is not valid.");
      }
      String[] headers = csvHeader.toLowerCase().split(delimiter);
      csvColumns = new String[headers.length];
      for (int i = 0; i < csvColumns.length; i++) {
        csvColumns[i] = headers[i].replaceAll("\"", "").trim();
      }
    } else {
      String[] headers = csvHeader.toLowerCase().split(CarbonCommonConstants.COMMA);
      csvColumns = new String[headers.length];
      for (int i = 0; i < csvColumns.length; i++) {
        csvColumns[i] = headers[i].trim();
      }
    }

    // In SDK flow, hadoopConf will always be null,
    // hence FileHeader check is not required for nontransactional table
    if (hadoopConf != null && !CarbonDataProcessorUtil
        .isHeaderValid(carbonLoadModel.getTableName(), csvColumns,
            carbonLoadModel.getCarbonDataLoadSchema(), staticPartitionCols)) {
      if (csvFile == null) {
        LOG.error("CSV header in DDL is not proper."
            + " Column names in schema and CSV header are not the same.");
        throw new CarbonDataLoadingException(
            "CSV header in DDL is not proper. Column names in schema and CSV header are "
                + "not the same.");
      } else {
        LOG.error(
            "CSV header in input file is not proper. Column names in schema and csv header are not "
                + "the same. Input file : " + CarbonUtil.removeAKSK(csvFile));
        throw new CarbonDataLoadingException(
            "CSV header in input file is not proper. Column names in schema and csv header are not "
                + "the same. Input file : " + CarbonUtil.removeAKSK(csvFile));
      }
    }

    // In case of static partition columns just change the name of header if already exists as
    // we should not take the column from csv file and add them as new columns at the end.
    if (staticPartitionCols.size() > 0) {
      List<String> updatedColumns = new ArrayList<>();
      for (int i = 0; i < csvColumns.length; i++) {
        if (staticPartitionCols.contains(csvColumns[i])) {
          updatedColumns.add(csvColumns[i] + "1");
        } else {
          updatedColumns.add(csvColumns[i]);
        }
      }
      updatedColumns.addAll(staticPartitionCols);
      return updatedColumns.toArray(new String[updatedColumns.size()]);
    } else {
      return csvColumns;
    }
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.processing.loading.model;
19
20	import java.io.IOException;
21	import java.util.ArrayList;
22	import java.util.HashMap;
23	import java.util.LinkedList;
24	import java.util.List;
25	import java.util.Map;
26
27	import org.apache.carbondata.common.Maps;
28	import org.apache.carbondata.common.annotations.InterfaceAudience;
29	import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
30	import org.apache.carbondata.common.logging.LogServiceFactory;
31	import org.apache.carbondata.core.constants.CarbonCommonConstants;
32	import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
33	import org.apache.carbondata.core.util.CarbonProperties;
34	import org.apache.carbondata.core.util.CarbonUtil;
35	import org.apache.carbondata.processing.loading.ComplexDelimitersEnum;
36	import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
37	import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
38	import org.apache.carbondata.processing.util.CarbonLoaderUtil;
39
40	import org.apache.commons.lang.StringUtils;
41	import org.apache.hadoop.conf.Configuration;
42	import org.apache.log4j.Logger;
43
44	/**
45	* Provide utilities to populate loading options
46	*/
47	@InterfaceAudience.Internal
48	public class LoadOption {	×
49
50	private static final Logger LOG = LogServiceFactory.getLogService(LoadOption.class.getName());	1✔
51
52	/**
53	* Based on the input options, fill and return data loading options with default value
54	*/
55	public static Map<String, String> fillOptionWithDefaultValue(
56	Map<String, String> options) throws InvalidLoadOptionException {
57	Map<String, String> optionsFinal = new HashMap<>();	1✔
58	optionsFinal.put("delimiter", Maps.getOrDefault(options, "delimiter", ","));	1✔
59	optionsFinal.put("quotechar", Maps.getOrDefault(options, "quotechar", "\""));	1✔
60	optionsFinal.put("fileheader", Maps.getOrDefault(options, "fileheader", ""));	1✔
61	optionsFinal.put("commentchar", Maps.getOrDefault(options, "commentchar", "#"));	1✔
62	optionsFinal.put("columndict", Maps.getOrDefault(options, "columndict", null));	1✔
63
64	optionsFinal.put(	1✔
65	"escapechar",
66	CarbonLoaderUtil.getEscapeChar(Maps.getOrDefault(options,"escapechar", "\\")));	1✔
67
68	optionsFinal.put(	1✔
69	"serialization_null_format",
70	Maps.getOrDefault(options, "serialization_null_format", "\\N"));	1✔
71
72	optionsFinal.put(	1✔
73	"bad_records_logger_enable",
74	Maps.getOrDefault(	1✔
75	options,
76	"bad_records_logger_enable",
77	CarbonProperties.getInstance().getProperty(	1✔
78	CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
79	CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT)));
80
81	String badRecordActionValue = CarbonProperties.getInstance().getProperty(	1✔
82	CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
83	CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT);
84
85	optionsFinal.put(	1✔
86	"bad_records_action",
87	Maps.getOrDefault(	1✔
88	options,
89	"bad_records_action",
90	CarbonProperties.getInstance().getProperty(	1✔
91	CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
92	badRecordActionValue)));
93
94	optionsFinal.put(	1✔
95	"is_empty_data_bad_record",
96	Maps.getOrDefault(	1✔
97	options,
98	"is_empty_data_bad_record",
99	CarbonProperties.getInstance().getProperty(	1✔
100	CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
101	CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT)));
102
103	optionsFinal.put(	1✔
104	"skip_empty_line",
105	Maps.getOrDefault(	1✔
106	options,
107	"skip_empty_line",
108	CarbonProperties.getInstance().getProperty(	1✔
109	CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
110
111	optionsFinal.put(	1✔
112	"all_dictionary_path",
113	Maps.getOrDefault(options, "all_dictionary_path", ""));	1✔
114
115	optionsFinal.put("complex_delimiter_level_1",	1✔
116	Maps.getOrDefault(options, "complex_delimiter_level_1",	1✔
117	ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_1.value()));	1✔
118
119	optionsFinal.put("complex_delimiter_level_2",	1✔
120	Maps.getOrDefault(options, "complex_delimiter_level_2",	1✔
121	ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_2.value()));	1✔
122
123	optionsFinal.put("complex_delimiter_level_3",	1✔
124	Maps.getOrDefault(options, "complex_delimiter_level_3",	1✔
125	ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_3.value()));	1✔
126
127	optionsFinal.put(	1✔
128	"dateformat",
129	Maps.getOrDefault(	1✔
130	options,
131	"dateformat",
132	CarbonProperties.getInstance().getProperty(	1✔
133	CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
134	CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)));
135
136	optionsFinal.put(	1✔
137	"timestampformat",
138	Maps.getOrDefault(	1✔
139	options,
140	"timestampformat",
141	CarbonProperties.getInstance().getProperty(	1✔
142	CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT,
143	CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT)));
144
145	optionsFinal.put(	1✔
146	"global_sort_partitions",
147	Maps.getOrDefault(	1✔
148	options,
149	"global_sort_partitions",
150	CarbonProperties.getInstance().getProperty(	1✔
151	CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
152	null)));
153
154	optionsFinal.put("maxcolumns", Maps.getOrDefault(options, "maxcolumns", null));	1✔
155
156	optionsFinal.put(	1✔
157	"batch_sort_size_inmb",
158	Maps.getOrDefault(	1✔
159	options,
160	"batch_sort_size_inmb",
161	CarbonProperties.getInstance().getProperty(	1✔
162	CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
163	CarbonProperties.getInstance().getProperty(	1✔
164	CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
165	CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))));
166
167	String useOnePass = Maps.getOrDefault(	1✔
168	options,
169	"single_pass",
170	CarbonProperties.getInstance().getProperty(	1✔
171	CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
172	CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT)).trim().toLowerCase();	1✔
173
174	boolean singlePass;
175
176	if (useOnePass.equalsIgnoreCase("true")) {	1✔
177	singlePass = true;	1✔
178	} else {
179	// when single_pass = false and if either alldictionarypath
180	// or columnDict is configured the do not allow load
181	if (StringUtils.isNotEmpty(optionsFinal.get("all_dictionary_path")) \|\|	1✔
182	StringUtils.isNotEmpty(optionsFinal.get("columndict"))) {	1✔
183	throw new InvalidLoadOptionException(	1✔
184	"Can not use all_dictionary_path or columndict without single_pass.");
185	} else {
186	singlePass = false;	1✔
187	}
188	}
189
190	optionsFinal.put("single_pass", String.valueOf(singlePass));	1✔
191	optionsFinal.put("sort_scope", CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT);	1✔
192	optionsFinal.put("sort_column_bounds", Maps.getOrDefault(options, "sort_column_bounds", ""));	1✔
193	optionsFinal.put(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB,	1✔
194	Maps.getOrDefault(options, CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB,	1✔
195	CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB_DEFAULT));
196
197	optionsFinal.put("range_column", Maps.getOrDefault(options, "range_column", null));	1✔
198	optionsFinal.put("scale_factor", Maps.getOrDefault(options, "scale_factor", null));	1✔
199	return optionsFinal;	1✔
200	}
201
202	/**
203	* Return CSV header field names
204	*/
205	public static String[] getCsvHeaderColumns(
206	CarbonLoadModel carbonLoadModel,
207	Configuration hadoopConf) throws IOException {
208	return getCsvHeaderColumns(carbonLoadModel, hadoopConf, new LinkedList<String>());	1✔
209	}
210
211	/**
212	* Return CSV header field names, with partition column
213	*/
214	public static String[] getCsvHeaderColumns(
215	CarbonLoadModel carbonLoadModel,
216	Configuration hadoopConf,
217	List<String> staticPartitionCols) throws IOException {
218	String delimiter;
219	if (StringUtils.isEmpty(carbonLoadModel.getCsvDelimiter())) {	1✔
220	delimiter = CarbonCommonConstants.COMMA;	×
221	} else {
222	delimiter = CarbonUtil.delimiterConverter(carbonLoadModel.getCsvDelimiter());	1✔
223	}
224	String csvFile = null;	1✔
225	String csvHeader = carbonLoadModel.getCsvHeader();	1✔
226	String[] csvColumns;
227	if (StringUtils.isBlank(csvHeader)) {	1✔
228	// read header from csv file
229	csvFile = carbonLoadModel.getFactFilePath().split(",")[0];	1✔
230	csvHeader = CarbonUtil.readHeader(csvFile, hadoopConf);	1✔
231	if (StringUtils.isBlank(csvHeader)) {	1✔
232	throw new CarbonDataLoadingException("First line of the csv is not valid.");	1✔
233	}
234	String[] headers = csvHeader.toLowerCase().split(delimiter);	1✔
235	csvColumns = new String[headers.length];	1✔
236	for (int i = 0; i < csvColumns.length; i++) {	1✔
237	csvColumns[i] = headers[i].replaceAll("\"", "").trim();	1✔
238	}
239	} else {	1✔
240	String[] headers = csvHeader.toLowerCase().split(CarbonCommonConstants.COMMA);	1✔
241	csvColumns = new String[headers.length];	1✔
242	for (int i = 0; i < csvColumns.length; i++) {	1✔
243	csvColumns[i] = headers[i].trim();	1✔
244	}
245	}
246
247	// In SDK flow, hadoopConf will always be null,
248	// hence FileHeader check is not required for nontransactional table
249	if (hadoopConf != null && !CarbonDataProcessorUtil	1✔
250	.isHeaderValid(carbonLoadModel.getTableName(), csvColumns,	1✔
251	carbonLoadModel.getCarbonDataLoadSchema(), staticPartitionCols)) {	1✔
252	if (csvFile == null) {	1✔
253	LOG.error("CSV header in DDL is not proper."	1✔
254	+ " Column names in schema and CSV header are not the same.");
255	throw new CarbonDataLoadingException(	1✔
256	"CSV header in DDL is not proper. Column names in schema and CSV header are "
257	+ "not the same.");
258	} else {
259	LOG.error(	1✔
260	"CSV header in input file is not proper. Column names in schema and csv header are not "
261	+ "the same. Input file : " + CarbonUtil.removeAKSK(csvFile));	1✔
262	throw new CarbonDataLoadingException(	1✔
263	"CSV header in input file is not proper. Column names in schema and csv header are not "
264	+ "the same. Input file : " + CarbonUtil.removeAKSK(csvFile));	1✔
265	}
266	}
267
268	// In case of static partition columns just change the name of header if already exists as
269	// we should not take the column from csv file and add them as new columns at the end.
270	if (staticPartitionCols.size() > 0) {	1✔
271	List<String> updatedColumns = new ArrayList<>();	1✔
272	for (int i = 0; i < csvColumns.length; i++) {	1✔
273	if (staticPartitionCols.contains(csvColumns[i])) {	1✔
274	updatedColumns.add(csvColumns[i] + "1");	1✔
275	} else {
276	updatedColumns.add(csvColumns[i]);	1✔
277	}
278	}
279	updatedColumns.addAll(staticPartitionCols);	1✔
280	return updatedColumns.toArray(new String[updatedColumns.size()]);	1✔
281	} else {
282	return csvColumns;	1✔
283	}
284	}
285
286	}

apache / carbondata / 2436

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous