apache / carbondata / 336

Build # 336

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/17d1ae21fa5f464fd3ef6a70a6d1ffbf96a0ffdc">17d1ae21f<a href="https://github.com/apache/carbondata/commit/17d1ae21fa5f464fd3ef6a70a6d1ffbf96a0ffdc">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/06845961e3d9346343591b77a62a7c3018428a45">06845961e</a>">06845961e</a><a href="https://github.com/apache/carbondata/commit/17d1ae21fa5f464fd3ef6a70a6d1ffbf96a0ffdc"> into f5c7a19b8">f5c7a19b8</a>

Run Details

56 of 56 new or added lines in 5 files covered. (100.0%)

58090 of 72069 relevant lines covered (80.6%)

1.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.31

/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/JsonRowParser.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.processing.loading.parser.impl;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.complexobjects.ArrayObject;
import org.apache.carbondata.processing.loading.complexobjects.StructObject;
import org.apache.carbondata.processing.loading.parser.RowParser;

import org.apache.htrace.fasterxml.jackson.core.type.TypeReference;
import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;

public class JsonRowParser implements RowParser {

  private DataField[] dataFields;

  public JsonRowParser(DataField[] dataFields) {
    this.dataFields = dataFields;
  }

  @Override public Object[] parseRow(Object[] row) {
    try {
      return convertJsonToNoDictionaryToBytes((String) row[0]);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private Object[] convertJsonToNoDictionaryToBytes(String jsonString)
      throws IOException {
    ObjectMapper objectMapper = new ObjectMapper();
    try {
      Map<String, Object> jsonNodeMap =
          objectMapper.readValue(jsonString, new TypeReference<Map<String, Object>>() {
          });
      if (jsonNodeMap == null) {
        return null;
      }
      Map<String, Object> jsonNodeMapCaseInsensitive = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
      jsonNodeMapCaseInsensitive.putAll(jsonNodeMap);
      return jsonToCarbonRecord(jsonNodeMapCaseInsensitive, dataFields);
    } catch (IOException e) {
      throw new IOException("Failed to parse Json String: " + e.getMessage());
    }
  }

  private Object[] jsonToCarbonRecord(Map<String, Object> jsonNodeMap, DataField[] dataFields) {
    List<Object> fields = new ArrayList<>();
    for (DataField dataField : dataFields) {
      Object field = jsonToCarbonObject(jsonNodeMap, dataField.getColumn());
      fields.add(field);
    }
    // use this array object to form carbonRow
    return fields.toArray();
  }

  private Object jsonToCarbonObject(Map<String, Object> jsonNodeMap, CarbonColumn column) {
    DataType type = column.getDataType();
    if (DataTypes.isArrayType(type)) {
      CarbonDimension carbonDimension = (CarbonDimension) column;
      ArrayList array = (ArrayList) jsonNodeMap.get(extractChildColumnName(column));
      if ((array == null) || (array.size() == 0)) {
        return null;
      }
      // stored as array in carbonObject
      Object[] arrayChildObjects = new Object[array.size()];
      for (int i = 0; i < array.size(); i++) {
        // array column will have only one child, hence get(0).
        // But data can have n elements, hence the loop.
        CarbonDimension childCol = carbonDimension.getListOfChildDimensions().get(0);
        arrayChildObjects[i] = jsonChildElementToCarbonChildElement(array.get(i), childCol);
      }
      return new ArrayObject(arrayChildObjects);
    } else if (DataTypes.isStructType(type)) {
      CarbonDimension carbonDimension = (CarbonDimension) column;
      int size = carbonDimension.getNumberOfChild();
      Map<String, Object> jsonMap =
          (Map<String, Object>) jsonNodeMap.get(extractChildColumnName(column));
      if (jsonMap == null) {
        return null;
      }
      Object[] structChildObjects = new Object[size];
      for (int i = 0; i < size; i++) {
        CarbonDimension childCol = carbonDimension.getListOfChildDimensions().get(i);
        Object childObject =
            jsonChildElementToCarbonChildElement(jsonMap.get(extractChildColumnName(childCol)),
                childCol);
        structChildObjects[i] = childObject;
      }
      return new StructObject(structChildObjects);
    } else {
      // primitive type
      if (jsonNodeMap.get(extractChildColumnName(column)) == null) {
        return null;
      }
      return jsonNodeMap.get(extractChildColumnName(column)).toString();
    }
  }

  private Object jsonChildElementToCarbonChildElement(Object childObject,
      CarbonDimension column) {
    if (childObject == null) {
      return null;
    }
    DataType type = column.getDataType();
    if (DataTypes.isArrayType(type)) {
      ArrayList array = (ArrayList) childObject;
      if (array.size() == 0) {
        // handling empty array
        return null;
      }
      // stored as array in carbonObject
      Object[] arrayChildObjects = new Object[array.size()];
      for (int i = 0; i < array.size(); i++) {
        // array column will have only one child, hence get(0).
        // But data can have n elements, hence the loop.
        CarbonDimension childCol = column.getListOfChildDimensions().get(0);
        arrayChildObjects[i] = jsonChildElementToCarbonChildElement(array.get(i), childCol);
      }
      return new ArrayObject(arrayChildObjects);
    } else if (DataTypes.isStructType(type)) {
      Map<String, Object> childFieldsMap = (Map<String, Object>) childObject;
      int size = column.getNumberOfChild();
      Object[] structChildObjects = new Object[size];
      for (int i = 0; i < size; i++) {
        CarbonDimension childCol = column.getListOfChildDimensions().get(i);
        Object child = jsonChildElementToCarbonChildElement(
            childFieldsMap.get(extractChildColumnName(childCol)), childCol);
        structChildObjects[i] = child;
      }
      return new StructObject(structChildObjects);
    } else {
      // primitive type
      return childObject.toString();
    }
  }

  private static String extractChildColumnName(CarbonColumn column) {
    String columnName = column.getColName();
    if (columnName.contains(".")) {
      // complex type child column names can be like following
      // a) struct type --> parent.child
      // b) array type --> parent.val.val...child [If create table flow]
      // c) array type --> parent.val0.val1...child [If SDK flow]
      // But json data's key is only child column name. So, extracting below
      String[] splits = columnName.split("\\.");
      columnName = splits[splits.length - 1];
    }
    return columnName;
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.processing.loading.parser.impl;
18
19	import java.io.IOException;
20	import java.util.ArrayList;
21	import java.util.List;
22	import java.util.Map;
23	import java.util.TreeMap;
24
25	import org.apache.carbondata.core.metadata.datatype.DataType;
26	import org.apache.carbondata.core.metadata.datatype.DataTypes;
27	import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
28	import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
29	import org.apache.carbondata.processing.loading.DataField;
30	import org.apache.carbondata.processing.loading.complexobjects.ArrayObject;
31	import org.apache.carbondata.processing.loading.complexobjects.StructObject;
32	import org.apache.carbondata.processing.loading.parser.RowParser;
33
34	import org.apache.htrace.fasterxml.jackson.core.type.TypeReference;
35	import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper;
36
37	public class JsonRowParser implements RowParser {
38
39	private DataField[] dataFields;
40
41	public JsonRowParser(DataField[] dataFields) {	1✔
42	this.dataFields = dataFields;	1✔
43	}	1✔
44
45	@Override public Object[] parseRow(Object[] row) {
46	try {
47	return convertJsonToNoDictionaryToBytes((String) row[0]);	1✔
48	} catch (IOException e) {	×
49	throw new RuntimeException(e);	×
50	}
51	}
52
53	private Object[] convertJsonToNoDictionaryToBytes(String jsonString)
54	throws IOException {
55	ObjectMapper objectMapper = new ObjectMapper();	1✔
56	try {
57	Map<String, Object> jsonNodeMap =	1✔
58	objectMapper.readValue(jsonString, new TypeReference<Map<String, Object>>() {	1✔
59	});
60	if (jsonNodeMap == null) {	1✔
61	return null;	×
62	}
63	Map<String, Object> jsonNodeMapCaseInsensitive = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);	1✔
64	jsonNodeMapCaseInsensitive.putAll(jsonNodeMap);	1✔
65	return jsonToCarbonRecord(jsonNodeMapCaseInsensitive, dataFields);	1✔
66	} catch (IOException e) {	×
67	throw new IOException("Failed to parse Json String: " + e.getMessage());	×
68	}
69	}
70
71	private Object[] jsonToCarbonRecord(Map<String, Object> jsonNodeMap, DataField[] dataFields) {
72	List<Object> fields = new ArrayList<>();	1✔
73	for (DataField dataField : dataFields) {	1✔
74	Object field = jsonToCarbonObject(jsonNodeMap, dataField.getColumn());	1✔
75	fields.add(field);	1✔
76	}
77	// use this array object to form carbonRow
78	return fields.toArray();	1✔
79	}
80
81	private Object jsonToCarbonObject(Map<String, Object> jsonNodeMap, CarbonColumn column) {
82	DataType type = column.getDataType();	1✔
83	if (DataTypes.isArrayType(type)) {	1✔
84	CarbonDimension carbonDimension = (CarbonDimension) column;	1✔
85	ArrayList array = (ArrayList) jsonNodeMap.get(extractChildColumnName(column));	1✔
86	if ((array == null) \|\| (array.size() == 0)) {	1✔
87	return null;	×
88	}
89	// stored as array in carbonObject
90	Object[] arrayChildObjects = new Object[array.size()];	1✔
91	for (int i = 0; i < array.size(); i++) {	1✔
92	// array column will have only one child, hence get(0).
93	// But data can have n elements, hence the loop.
94	CarbonDimension childCol = carbonDimension.getListOfChildDimensions().get(0);	1✔
95	arrayChildObjects[i] = jsonChildElementToCarbonChildElement(array.get(i), childCol);	1✔
96	}
97	return new ArrayObject(arrayChildObjects);	1✔
98	} else if (DataTypes.isStructType(type)) {	1✔
99	CarbonDimension carbonDimension = (CarbonDimension) column;	1✔
100	int size = carbonDimension.getNumberOfChild();	1✔
101	Map<String, Object> jsonMap =	1✔
102	(Map<String, Object>) jsonNodeMap.get(extractChildColumnName(column));	1✔
103	if (jsonMap == null) {	1✔
104	return null;	×
105	}
106	Object[] structChildObjects = new Object[size];	1✔
107	for (int i = 0; i < size; i++) {	1✔
108	CarbonDimension childCol = carbonDimension.getListOfChildDimensions().get(i);	1✔
109	Object childObject =	1✔
110	jsonChildElementToCarbonChildElement(jsonMap.get(extractChildColumnName(childCol)),	1✔
111	childCol);
112	structChildObjects[i] = childObject;	1✔
113	}
114	return new StructObject(structChildObjects);	1✔
115	} else {
116	// primitive type
117	if (jsonNodeMap.get(extractChildColumnName(column)) == null) {	1✔
118	return null;	1✔
119	}
120	return jsonNodeMap.get(extractChildColumnName(column)).toString();	1✔
121	}
122	}
123
124	private Object jsonChildElementToCarbonChildElement(Object childObject,
125	CarbonDimension column) {
126	if (childObject == null) {	1✔
127	return null;	×
128	}
129	DataType type = column.getDataType();	1✔
130	if (DataTypes.isArrayType(type)) {	1✔
131	ArrayList array = (ArrayList) childObject;	1✔
132	if (array.size() == 0) {	1✔
133	// handling empty array
134	return null;	×
135	}
136	// stored as array in carbonObject
137	Object[] arrayChildObjects = new Object[array.size()];	1✔
138	for (int i = 0; i < array.size(); i++) {	1✔
139	// array column will have only one child, hence get(0).
140	// But data can have n elements, hence the loop.
141	CarbonDimension childCol = column.getListOfChildDimensions().get(0);	1✔
142	arrayChildObjects[i] = jsonChildElementToCarbonChildElement(array.get(i), childCol);	1✔
143	}
144	return new ArrayObject(arrayChildObjects);	1✔
145	} else if (DataTypes.isStructType(type)) {	1✔
146	Map<String, Object> childFieldsMap = (Map<String, Object>) childObject;	1✔
147	int size = column.getNumberOfChild();	1✔
148	Object[] structChildObjects = new Object[size];	1✔
149	for (int i = 0; i < size; i++) {	1✔
150	CarbonDimension childCol = column.getListOfChildDimensions().get(i);	1✔
151	Object child = jsonChildElementToCarbonChildElement(	1✔
152	childFieldsMap.get(extractChildColumnName(childCol)), childCol);	1✔
153	structChildObjects[i] = child;	1✔
154	}
155	return new StructObject(structChildObjects);	1✔
156	} else {
157	// primitive type
158	return childObject.toString();	1✔
159	}
160	}
161
162	private static String extractChildColumnName(CarbonColumn column) {
163	String columnName = column.getColName();	1✔
164	if (columnName.contains(".")) {	1✔
165	// complex type child column names can be like following
166	// a) struct type --> parent.child
167	// b) array type --> parent.val.val...child [If create table flow]
168	// c) array type --> parent.val0.val1...child [If SDK flow]
169	// But json data's key is only child column name. So, extracting below
170	String[] splits = columnName.split("\\.");	1✔
171	columnName = splits[splits.length - 1];	1✔
172	}
173	return columnName;	1✔
174	}
175	}

apache / carbondata / 336

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous