2681

Build # 2681

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/f422d1398ae78ad1fec4cb97b21a65dae6ea4ea0">f422d1398<a href="https://github.com/apache/carbondata/commit/f422d1398ae78ad1fec4cb97b21a65dae6ea4ea0">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/c25f24187db7f086ef159bc5aecabe2237c6c9f7">c25f24187</a>">c25f24187</a><a href="https://github.com/apache/carbondata/commit/f422d1398ae78ad1fec4cb97b21a65dae6ea4ea0"> into 8e39ee113">8e39ee113</a>

Run Details

160 of 160 new or added lines in 22 files covered. (100.0%)

62526 of 79013 relevant lines covered (79.13%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

13.7

/integration/spark-datasource/src/main/scala/org/apache/carbondata/converter/SparkDataTypeConverterImpl.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.carbondata.converter;

import java.io.Serializable;
import java.math.BigDecimal;

import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.util.DataTypeConverter;

import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
import org.apache.spark.sql.catalyst.util.ArrayBasedMapData;
import org.apache.spark.sql.catalyst.util.GenericArrayData;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.DecimalType;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.unsafe.types.UTF8String;

/**
 * Convert java data type to spark data type
 */
public final class SparkDataTypeConverterImpl implements DataTypeConverter, Serializable {

  private static final long serialVersionUID = -4379212832935070583L;

  @Override
  public Object convertFromStringToDecimal(Object data) {
    BigDecimal javaDecVal = new BigDecimal(data.toString());
    return org.apache.spark.sql.types.Decimal.apply(javaDecVal);
  }

  @Override
  public Object convertFromBigDecimalToDecimal(Object data) {
    if (null == data) {
      return null;
    }
    return org.apache.spark.sql.types.Decimal.apply((BigDecimal)data);
  }

  @Override
  public Object convertFromDecimalToBigDecimal(Object data) {
    return ((org.apache.spark.sql.types.Decimal) data).toJavaBigDecimal();
  }

  @Override
  public byte[] convertFromStringToByte(Object data) {
    if (null == data) {
      return null;
    }
    return UTF8String.fromString((String) data).getBytes();
  }

  @Override
  public Object convertFromByteToUTF8String(byte[] data) {
    if (null == data) {
      return null;
    }
    return UTF8String.fromBytes(data);
  }

  @Override
  public byte[] convertFromByteToUTF8Bytes(byte[] data) {
    return UTF8String.fromBytes(data).getBytes();
  }

  @Override
  public Object convertFromStringToUTF8String(Object data) {
    if (null == data) {
      return null;
    }
    return UTF8String.fromString((String) data);
  }

  @Override
  public Object wrapWithGenericArrayData(Object data) {
    return new GenericArrayData(data);
  }

  @Override
  public Object wrapWithGenericRow(Object[] fields) {
    return new GenericInternalRow(fields);
  }

  @Override
  public Object wrapWithArrayBasedMapData(Object[] keyArray, Object[] valueArray) {
    return new ArrayBasedMapData(new GenericArrayData(keyArray), new GenericArrayData(valueArray));
  }

  @Override
  public Object[] unwrapGenericRowToObject(Object data) {
    GenericInternalRow row = (GenericInternalRow) data;
    return row.values();
  }

  private static org.apache.spark.sql.types.DataType convertCarbonToSparkDataType(
      DataType carbonDataType) {
    if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.STRING) {
      return DataTypes.StringType;
    } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.SHORT) {
      return DataTypes.ShortType;
    } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.INT) {
      return DataTypes.IntegerType;
    } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.LONG) {
      return DataTypes.LongType;
    } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.DOUBLE) {
      return DataTypes.DoubleType;
    } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BOOLEAN) {
      return DataTypes.BooleanType;
    } else if (org.apache.carbondata.core.metadata.datatype.DataTypes.isDecimal(carbonDataType)) {
      return DataTypes.createDecimalType();
    } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.TIMESTAMP) {
      return DataTypes.TimestampType;
    } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.DATE) {
      return DataTypes.DateType;
    } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BINARY) {
      return DataTypes.BinaryType;
    } else {
      return null;
    }
  }

  /**
   * convert from CarbonColumn array to Spark's StructField array
   */
  @Override
  public Object[] convertCarbonSchemaToSparkSchema(CarbonColumn[] carbonColumns) {
    StructField[] fields = new StructField[carbonColumns.length];
    for (int i = 0; i < carbonColumns.length; i++) {
      CarbonColumn carbonColumn = carbonColumns[i];
      if (carbonColumn.isDimension()) {
        if (carbonColumn.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
          DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
              .getDirectDictionaryGenerator(carbonColumn.getDataType());
          fields[i] = new StructField(carbonColumn.getColName(),
              convertCarbonToSparkDataType(generator.getReturnType()), true, null);
        } else if (!carbonColumn.hasEncoding(Encoding.DICTIONARY)) {
          fields[i] = new StructField(carbonColumn.getColName(),
              convertCarbonToSparkDataType(carbonColumn.getDataType()), true, null);
        } else if (carbonColumn.isComplex()) {
          fields[i] = new StructField(carbonColumn.getColName(),
              convertCarbonToSparkDataType(carbonColumn.getDataType()), true, null);
        } else {
          fields[i] = new StructField(carbonColumn.getColName(),
              convertCarbonToSparkDataType(
                  org.apache.carbondata.core.metadata.datatype.DataTypes.INT), true, null);
        }
      } else if (carbonColumn.isMeasure()) {
        DataType dataType = carbonColumn.getDataType();
        if (dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BOOLEAN
            || dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.SHORT
            || dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.INT
            || dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.LONG
            || dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BINARY) {
          fields[i] = new StructField(carbonColumn.getColName(),
              convertCarbonToSparkDataType(dataType), true, null);
        } else if (org.apache.carbondata.core.metadata.datatype.DataTypes.isDecimal(dataType)) {
          CarbonMeasure measure = (CarbonMeasure) carbonColumn;
          fields[i] = new StructField(carbonColumn.getColName(),
              new DecimalType(measure.getPrecision(), measure.getScale()), true, null);
        } else {
          fields[i] = new StructField(carbonColumn.getColName(),
              convertCarbonToSparkDataType(
                  org.apache.carbondata.core.metadata.datatype.DataTypes.DOUBLE), true, null);
        }
      }
    }
    return fields;
  }

}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17
18	package org.apache.carbondata.converter;
19
20	import java.io.Serializable;
21	import java.math.BigDecimal;
22
23	import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
24	import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
25	import org.apache.carbondata.core.metadata.datatype.DataType;
26	import org.apache.carbondata.core.metadata.encoder.Encoding;
27	import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
28	import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
29	import org.apache.carbondata.core.util.DataTypeConverter;
30
31	import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
32	import org.apache.spark.sql.catalyst.util.ArrayBasedMapData;
33	import org.apache.spark.sql.catalyst.util.GenericArrayData;
34	import org.apache.spark.sql.types.DataTypes;
35	import org.apache.spark.sql.types.DecimalType;
36	import org.apache.spark.sql.types.StructField;
37	import org.apache.spark.unsafe.types.UTF8String;
38
39	/**
40	* Convert java data type to spark data type
41	*/
42	public final class SparkDataTypeConverterImpl implements DataTypeConverter, Serializable {	1✔
43
44	private static final long serialVersionUID = -4379212832935070583L;
45
46	@Override
47	public Object convertFromStringToDecimal(Object data) {
48	BigDecimal javaDecVal = new BigDecimal(data.toString());	×
49	return org.apache.spark.sql.types.Decimal.apply(javaDecVal);	×
50	}
51
52	@Override
53	public Object convertFromBigDecimalToDecimal(Object data) {
54	if (null == data) {	1✔
55	return null;	×
56	}
57	return org.apache.spark.sql.types.Decimal.apply((BigDecimal)data);	1✔
58	}
59
60	@Override
61	public Object convertFromDecimalToBigDecimal(Object data) {
62	return ((org.apache.spark.sql.types.Decimal) data).toJavaBigDecimal();	×
63	}
64
65	@Override
66	public byte[] convertFromStringToByte(Object data) {
67	if (null == data) {	×
68	return null;	×
69	}
70	return UTF8String.fromString((String) data).getBytes();	×
71	}
72
73	@Override
74	public Object convertFromByteToUTF8String(byte[] data) {
75	if (null == data) {	1✔
76	return null;	×
77	}
78	return UTF8String.fromBytes(data);	1✔
79	}
80
81	@Override
82	public byte[] convertFromByteToUTF8Bytes(byte[] data) {
83	return UTF8String.fromBytes(data).getBytes();	×
84	}
85
86	@Override
87	public Object convertFromStringToUTF8String(Object data) {
88	if (null == data) {	×
89	return null;	×
90	}
91	return UTF8String.fromString((String) data);	×
92	}
93
94	@Override
95	public Object wrapWithGenericArrayData(Object data) {
96	return new GenericArrayData(data);	1✔
97	}
98
99	@Override
100	public Object wrapWithGenericRow(Object[] fields) {
101	return new GenericInternalRow(fields);	1✔
102	}
103
104	@Override
105	public Object wrapWithArrayBasedMapData(Object[] keyArray, Object[] valueArray) {
106	return new ArrayBasedMapData(new GenericArrayData(keyArray), new GenericArrayData(valueArray));	1✔
107	}
108
109	@Override
110	public Object[] unwrapGenericRowToObject(Object data) {
111	GenericInternalRow row = (GenericInternalRow) data;	1✔
112	return row.values();	1✔
113	}
114
115	private static org.apache.spark.sql.types.DataType convertCarbonToSparkDataType(
116	DataType carbonDataType) {
117	if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.STRING) {	×
118	return DataTypes.StringType;	×
119	} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.SHORT) {	×
120	return DataTypes.ShortType;	×
121	} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.INT) {	×
122	return DataTypes.IntegerType;	×
123	} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.LONG) {	×
124	return DataTypes.LongType;	×
125	} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.DOUBLE) {	×
126	return DataTypes.DoubleType;	×
127	} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BOOLEAN) {	×
128	return DataTypes.BooleanType;	×
129	} else if (org.apache.carbondata.core.metadata.datatype.DataTypes.isDecimal(carbonDataType)) {	×
130	return DataTypes.createDecimalType();	×
131	} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.TIMESTAMP) {	×
132	return DataTypes.TimestampType;	×
133	} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.DATE) {	×
134	return DataTypes.DateType;	×
135	} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BINARY) {	×
136	return DataTypes.BinaryType;	×
137	} else {
138	return null;	×
139	}
140	}
141
142	/**
143	* convert from CarbonColumn array to Spark's StructField array
144	*/
145	@Override
146	public Object[] convertCarbonSchemaToSparkSchema(CarbonColumn[] carbonColumns) {
147	StructField[] fields = new StructField[carbonColumns.length];	×
148	for (int i = 0; i < carbonColumns.length; i++) {	×
149	CarbonColumn carbonColumn = carbonColumns[i];	×
150	if (carbonColumn.isDimension()) {	×
151	if (carbonColumn.hasEncoding(Encoding.DIRECT_DICTIONARY)) {	×
152	DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory	×
153	.getDirectDictionaryGenerator(carbonColumn.getDataType());	×
154	fields[i] = new StructField(carbonColumn.getColName(),	×
155	convertCarbonToSparkDataType(generator.getReturnType()), true, null);	×
156	} else if (!carbonColumn.hasEncoding(Encoding.DICTIONARY)) {	×
157	fields[i] = new StructField(carbonColumn.getColName(),	×
158	convertCarbonToSparkDataType(carbonColumn.getDataType()), true, null);	×
159	} else if (carbonColumn.isComplex()) {	×
160	fields[i] = new StructField(carbonColumn.getColName(),	×
161	convertCarbonToSparkDataType(carbonColumn.getDataType()), true, null);	×
162	} else {
163	fields[i] = new StructField(carbonColumn.getColName(),	×
164	convertCarbonToSparkDataType(	×
165	org.apache.carbondata.core.metadata.datatype.DataTypes.INT), true, null);
166	}
167	} else if (carbonColumn.isMeasure()) {	×
168	DataType dataType = carbonColumn.getDataType();	×
169	if (dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BOOLEAN	×
170	\|\| dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.SHORT
171	\|\| dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.INT
172	\|\| dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.LONG
173	\|\| dataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BINARY) {
174	fields[i] = new StructField(carbonColumn.getColName(),	×
175	convertCarbonToSparkDataType(dataType), true, null);	×
176	} else if (org.apache.carbondata.core.metadata.datatype.DataTypes.isDecimal(dataType)) {	×
177	CarbonMeasure measure = (CarbonMeasure) carbonColumn;	×
178	fields[i] = new StructField(carbonColumn.getColName(),	×
179	new DecimalType(measure.getPrecision(), measure.getScale()), true, null);	×
180	} else {	×
181	fields[i] = new StructField(carbonColumn.getColName(),	×
182	convertCarbonToSparkDataType(	×
183	org.apache.carbondata.core.metadata.datatype.DataTypes.DOUBLE), true, null);
184	}
185	}
186	}
187	return fields;	×
188	}
189
190	}

apache / carbondata / 2681

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous