3944

Build # 3944

Build Type

push

jenkins

Committed by GitHub

Commit Message

<a href="https://github.com/apache/carbondata/commit/<a class=hub.com/apache/carbondata/commit/d164e4585814eac437edfdb245cbe757a00c4035">d164e4585<a href="https://github.com/apache/carbondata/commit/d164e4585814eac437edfdb245cbe757a00c4035">">Merge </a><a class="double-link" href="https://github.com/apache/carbondata/commit/<a class="double-link" href="https://github.com/apache/carbondata/commit/b690cccc3239ed29e5ebd36467470ddcc8130b4a">b690cccc3</a>">b690cccc3</a><a href="https://github.com/apache/carbondata/commit/d164e4585814eac437edfdb245cbe757a00c4035"> into 188e7e4d4">188e7e4d4</a>

Run Details

183 of 183 new or added lines in 13 files covered. (100.0%)

64985 of 83572 relevant lines covered (77.76%)

1.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.39

/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.core.datamap;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

import org.apache.carbondata.core.metadata.schema.table.Writable;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.readcommitter.ReadCommittedScope;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentRefreshInfo;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.path.CarbonTablePath;

import org.apache.hadoop.conf.Configuration;

/**
 * Represents one load of carbondata
 */
public class Segment implements Serializable, Writable {

  private static final long serialVersionUID = 7044555408162234064L;

  private String segmentNo;

  private String segmentFileName;

  /**
   * List of index shards which are already got filtered through CG index operation.
   */
  private Set<String> filteredIndexShardNames = new HashSet<>();

  /**
   * Points to the Read Committed Scope of the segment. This is a flavor of
   * transactional isolation level which only allows snapshot read of the
   * data and make non committed data invisible to the reader.
   */
  private transient ReadCommittedScope readCommittedScope;

  /**
   * keeps all the details about segments
   */
  private transient LoadMetadataDetails loadMetadataDetails;

  private String segmentString;

  private long indexSize = 0;

  public Segment() {

  }

  public Segment(String segmentNo) {
    this.segmentNo = segmentNo;
  }

  public Segment(String segmentNo, ReadCommittedScope readCommittedScope) {
    this.segmentNo = segmentNo;
    this.readCommittedScope = readCommittedScope;
    segmentString = segmentNo;
  }

  /**
   * ReadCommittedScope will be null. So getCommittedIndexFile will not work and will throw
   * a NullPointerException. In case getCommittedIndexFile is need to be accessed then
   * use the other constructor and pass proper ReadCommittedScope.
   * @param segmentNo
   * @param segmentFileName
   */
  public Segment(String segmentNo, String segmentFileName) {
    this.segmentNo = segmentNo;
    this.segmentFileName = segmentFileName;
    this.readCommittedScope = null;
    if (segmentFileName != null) {
      segmentString = segmentNo + "#" + segmentFileName;
    } else {
      segmentString = segmentNo;
    }
  }

  /**
   *
   * @param segmentNo
   * @param segmentFileName
   * @param readCommittedScope
   */
  public Segment(String segmentNo, String segmentFileName, ReadCommittedScope readCommittedScope) {
    this.segmentNo = segmentNo;
    this.segmentFileName = segmentFileName;
    this.readCommittedScope = readCommittedScope;
    if (segmentFileName != null) {
      segmentString = segmentNo + "#" + segmentFileName;
    } else {
      segmentString = segmentNo;
    }
  }

  /**
   * @param segmentNo
   * @param segmentFileName
   * @param readCommittedScope
   */
  public Segment(String segmentNo, String segmentFileName, ReadCommittedScope readCommittedScope,
      LoadMetadataDetails loadMetadataDetails) {
    this.segmentNo = segmentNo;
    this.segmentFileName = segmentFileName;
    this.readCommittedScope = readCommittedScope;
    this.loadMetadataDetails = loadMetadataDetails;
    if (loadMetadataDetails.getIndexSize() != null) {
      this.indexSize = Long.parseLong(loadMetadataDetails.getIndexSize());
    }
    if (segmentFileName != null) {
      segmentString = segmentNo + "#" + segmentFileName;
    } else {
      segmentString = segmentNo;
    }
  }

  /**
   *
   * @return map of Absolute path of index file as key and null as value -- without mergeIndex
   * map of AbsolutePath with fileName of MergeIndex parent file as key and mergeIndexFileName
   *                                                             as value -- with mergeIndex
   * @throws IOException
   */
  public Map<String, String> getCommittedIndexFile() throws IOException {
    return readCommittedScope.getCommittedIndexFile(this);
  }

  public SegmentRefreshInfo getSegmentRefreshInfo(UpdateVO updateVo)
      throws IOException {
    return readCommittedScope.getCommittedSegmentRefreshInfo(this, updateVo);
  }

  public String getSegmentNo() {
    return segmentNo;
  }

  public String getSegmentFileName() {
    return segmentFileName;
  }

  public void setReadCommittedScope(ReadCommittedScope readCommittedScope) {
    this.readCommittedScope = readCommittedScope;
  }

  public ReadCommittedScope getReadCommittedScope() {
    return readCommittedScope;
  }

  public static List<Segment> toSegmentList(String[] segmentIds,
      ReadCommittedScope readCommittedScope) {
    List<Segment> list = new ArrayList<>(segmentIds.length);
    for (String segmentId : segmentIds) {
      list.add(toSegment(segmentId, readCommittedScope));
    }
    return list;
  }

  public static List<Segment> toSegmentList(List<String> segmentIds,
      ReadCommittedScope readCommittedScope) {
    List<Segment> list = new ArrayList<>(segmentIds.size());
    for (String segmentId : segmentIds) {
      list.add(toSegment(segmentId, readCommittedScope));
    }
    return list;
  }

  /**
   * readCommittedScope provide Read Snapshot isolation.
   * @param segmentId
   * @param readCommittedScope
   * @return
   */
  public static Segment toSegment(String segmentId, ReadCommittedScope readCommittedScope) {
    // SegmentId can be combination of segmentNo and segmentFileName.
    String[] split = segmentId.split("#");
    if (split.length > 1) {
      return new Segment(split[0], split[1], readCommittedScope);
    } else if (split.length > 0) {
      return new Segment(split[0], null, readCommittedScope);
    }
    return new Segment(segmentId, null, readCommittedScope);
  }

  /**
   * Converts to segment object
   * @param segmentId
   * @return
   */
  public static Segment toSegment(String segmentId) {
    // SegmentId can be combination of segmentNo and segmentFileName.
    return toSegment(segmentId, null);
  }

  /**
   * Read the table status and get the segment corresponding to segmentNo
   * @param segmentNo
   * @param tablePath
   * @return
   */
  public static Segment getSegment(String segmentNo, String tablePath) {
    LoadMetadataDetails[] loadMetadataDetails =
        SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath));
    return getSegment(segmentNo, loadMetadataDetails);
  }

  /**
   * Get the segment object corresponding to segmentNo
   * @param segmentNo
   * @param loadMetadataDetails
   * @return
   */
  public static Segment getSegment(String segmentNo, LoadMetadataDetails[] loadMetadataDetails) {
    for (LoadMetadataDetails details: loadMetadataDetails) {
      if (details.getLoadName().equals(segmentNo)) {
        return new Segment(details.getLoadName(), details.getSegmentFile());
      }
    }
    return null;
  }

  public Configuration getConfiguration() {
    return readCommittedScope.getConfiguration();
  }

  public Set<String> getFilteredIndexShardNames() {
    return filteredIndexShardNames;
  }

  public void setFilteredIndexShardName(String filteredIndexShardName) {
    this.filteredIndexShardNames.add(filteredIndexShardName);
  }

  @Override public boolean equals(Object o) {
    if (this == o) return true;
    if (o == null || getClass() != o.getClass()) return false;
    Segment segment = (Segment) o;
    return Objects.equals(segmentNo, segment.segmentNo);
  }

  @Override public int hashCode() {
    return Objects.hash(segmentNo);
  }

  @Override public String toString() {
    return segmentString;
  }

  public LoadMetadataDetails getLoadMetadataDetails() {
    return loadMetadataDetails;
  }

  public long getIndexSize() {
    return indexSize;
  }

  public void setIndexSize(long indexSize) {
    this.indexSize = indexSize;
  }

  @Override public void write(DataOutput out) throws IOException {
    out.writeUTF(segmentNo);
    boolean writeSegmentFileName = segmentFileName != null;
    out.writeBoolean(writeSegmentFileName);
    if (writeSegmentFileName) {
      out.writeUTF(segmentFileName);
    }
    out.writeInt(filteredIndexShardNames.size());
    for (String name: filteredIndexShardNames) {
      out.writeUTF(name);
    }
    if (segmentString == null) {
      out.writeBoolean(false);
    } else {
      out.writeBoolean(true);
      out.writeUTF(segmentString);
    }
    out.writeLong(indexSize);
  }

  @Override public void readFields(DataInput in) throws IOException {
    this.segmentNo = in.readUTF();
    if (in.readBoolean()) {
      this.segmentFileName = in.readUTF();
    }
    filteredIndexShardNames = new HashSet<>();
    int indexShardNameSize = in.readInt();
    for (int i = 0; i < indexShardNameSize; i++) {
      filteredIndexShardNames.add(in.readUTF());
    }
    if (in.readBoolean()) {
      this.segmentString = in.readUTF();
    }
    this.indexSize = in.readLong();
  }
}

1	/*
2	* Licensed to the Apache Software Foundation (ASF) under one or more
3	* contributor license agreements. See the NOTICE file distributed with
4	* this work for additional information regarding copyright ownership.
5	* The ASF licenses this file to You under the Apache License, Version 2.0
6	* (the "License"); you may not use this file except in compliance with
7	* the License. You may obtain a copy of the License at
8	*
9	* http://www.apache.org/licenses/LICENSE-2.0
10	*
11	* Unless required by applicable law or agreed to in writing, software
12	* distributed under the License is distributed on an "AS IS" BASIS,
13	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	* See the License for the specific language governing permissions and
15	* limitations under the License.
16	*/
17	package org.apache.carbondata.core.datamap;
18
19	import java.io.DataInput;
20	import java.io.DataOutput;
21	import java.io.IOException;
22	import java.io.Serializable;
23	import java.util.ArrayList;
24	import java.util.HashSet;
25	import java.util.List;
26	import java.util.Map;
27	import java.util.Objects;
28	import java.util.Set;
29
30	import org.apache.carbondata.core.metadata.schema.table.Writable;
31	import org.apache.carbondata.core.mutate.UpdateVO;
32	import org.apache.carbondata.core.readcommitter.ReadCommittedScope;
33	import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
34	import org.apache.carbondata.core.statusmanager.SegmentRefreshInfo;
35	import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
36	import org.apache.carbondata.core.util.path.CarbonTablePath;
37
38	import org.apache.hadoop.conf.Configuration;
39
40	/**
41	* Represents one load of carbondata
42	*/
43	public class Segment implements Serializable, Writable {
44
45	private static final long serialVersionUID = 7044555408162234064L;
46
47	private String segmentNo;
48
49	private String segmentFileName;
50
51	/**
52	* List of index shards which are already got filtered through CG index operation.
53	*/
54	private Set<String> filteredIndexShardNames = new HashSet<>();	2✔
55
56	/**
57	* Points to the Read Committed Scope of the segment. This is a flavor of
58	* transactional isolation level which only allows snapshot read of the
59	* data and make non committed data invisible to the reader.
60	*/
61	private transient ReadCommittedScope readCommittedScope;
62
63	/**
64	* keeps all the details about segments
65	*/
66	private transient LoadMetadataDetails loadMetadataDetails;
67
68	private String segmentString;
69
70	private long indexSize = 0;	2✔
71
72	public Segment() {	×
73
74	}	×
75
76	public Segment(String segmentNo) {	1✔
77	this.segmentNo = segmentNo;	1✔
78	}	1✔
79
80	public Segment(String segmentNo, ReadCommittedScope readCommittedScope) {	1✔
81	this.segmentNo = segmentNo;	1✔
82	this.readCommittedScope = readCommittedScope;	1✔
83	segmentString = segmentNo;	1✔
84	}	1✔
85
86	/**
87	* ReadCommittedScope will be null. So getCommittedIndexFile will not work and will throw
88	* a NullPointerException. In case getCommittedIndexFile is need to be accessed then
89	* use the other constructor and pass proper ReadCommittedScope.
90	* @param segmentNo
91	* @param segmentFileName
92	*/
93	public Segment(String segmentNo, String segmentFileName) {	1✔
94	this.segmentNo = segmentNo;	1✔
95	this.segmentFileName = segmentFileName;	1✔
96	this.readCommittedScope = null;	1✔
97	if (segmentFileName != null) {	1✔
98	segmentString = segmentNo + "#" + segmentFileName;	1✔
99	} else {
100	segmentString = segmentNo;	1✔
101	}
102	}	1✔
103
104	/**
105	*
106	* @param segmentNo
107	* @param segmentFileName
108	* @param readCommittedScope
109	*/
110	public Segment(String segmentNo, String segmentFileName, ReadCommittedScope readCommittedScope) {	2✔
111	this.segmentNo = segmentNo;	2✔
112	this.segmentFileName = segmentFileName;	2✔
113	this.readCommittedScope = readCommittedScope;	2✔
114	if (segmentFileName != null) {	2✔
115	segmentString = segmentNo + "#" + segmentFileName;	1✔
116	} else {
117	segmentString = segmentNo;	2✔
118	}
119	}	2✔
120
121	/**
122	* @param segmentNo
123	* @param segmentFileName
124	* @param readCommittedScope
125	*/
126	public Segment(String segmentNo, String segmentFileName, ReadCommittedScope readCommittedScope,
127	LoadMetadataDetails loadMetadataDetails) {	1✔
128	this.segmentNo = segmentNo;	1✔
129	this.segmentFileName = segmentFileName;	1✔
130	this.readCommittedScope = readCommittedScope;	1✔
131	this.loadMetadataDetails = loadMetadataDetails;	1✔
132	if (loadMetadataDetails.getIndexSize() != null) {	1✔
133	this.indexSize = Long.parseLong(loadMetadataDetails.getIndexSize());	1✔
134	}
135	if (segmentFileName != null) {	1✔
136	segmentString = segmentNo + "#" + segmentFileName;	1✔
137	} else {
138	segmentString = segmentNo;	1✔
139	}
140	}	1✔
141
142	/**
143	*
144	* @return map of Absolute path of index file as key and null as value -- without mergeIndex
145	* map of AbsolutePath with fileName of MergeIndex parent file as key and mergeIndexFileName
146	* as value -- with mergeIndex
147	* @throws IOException
148	*/
149	public Map<String, String> getCommittedIndexFile() throws IOException {
150	return readCommittedScope.getCommittedIndexFile(this);	1✔
151	}
152
153	public SegmentRefreshInfo getSegmentRefreshInfo(UpdateVO updateVo)
154	throws IOException {
155	return readCommittedScope.getCommittedSegmentRefreshInfo(this, updateVo);	1✔
156	}
157
158	public String getSegmentNo() {
159	return segmentNo;	2✔
160	}
161
162	public String getSegmentFileName() {
163	return segmentFileName;	1✔
164	}
165
166	public void setReadCommittedScope(ReadCommittedScope readCommittedScope) {
167	this.readCommittedScope = readCommittedScope;	1✔
168	}	1✔
169
170	public ReadCommittedScope getReadCommittedScope() {
171	return readCommittedScope;	1✔
172	}
173
174	public static List<Segment> toSegmentList(String[] segmentIds,
175	ReadCommittedScope readCommittedScope) {
176	List<Segment> list = new ArrayList<>(segmentIds.length);	1✔
177	for (String segmentId : segmentIds) {	1✔
178	list.add(toSegment(segmentId, readCommittedScope));	1✔
179	}
180	return list;	1✔
181	}
182
183	public static List<Segment> toSegmentList(List<String> segmentIds,
184	ReadCommittedScope readCommittedScope) {
185	List<Segment> list = new ArrayList<>(segmentIds.size());	1✔
186	for (String segmentId : segmentIds) {	1✔
187	list.add(toSegment(segmentId, readCommittedScope));	1✔
188	}	1✔
189	return list;	1✔
190	}
191
192	/**
193	* readCommittedScope provide Read Snapshot isolation.
194	* @param segmentId
195	* @param readCommittedScope
196	* @return
197	*/
198	public static Segment toSegment(String segmentId, ReadCommittedScope readCommittedScope) {
199	// SegmentId can be combination of segmentNo and segmentFileName.
200	String[] split = segmentId.split("#");	2✔
201	if (split.length > 1) {	2✔
202	return new Segment(split[0], split[1], readCommittedScope);	1✔
203	} else if (split.length > 0) {	2✔
204	return new Segment(split[0], null, readCommittedScope);	2✔
205	}
206	return new Segment(segmentId, null, readCommittedScope);	×
207	}
208
209	/**
210	* Converts to segment object
211	* @param segmentId
212	* @return
213	*/
214	public static Segment toSegment(String segmentId) {
215	// SegmentId can be combination of segmentNo and segmentFileName.
216	return toSegment(segmentId, null);	2✔
217	}
218
219	/**
220	* Read the table status and get the segment corresponding to segmentNo
221	* @param segmentNo
222	* @param tablePath
223	* @return
224	*/
225	public static Segment getSegment(String segmentNo, String tablePath) {
226	LoadMetadataDetails[] loadMetadataDetails =	1✔
227	SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath));	1✔
228	return getSegment(segmentNo, loadMetadataDetails);	1✔
229	}
230
231	/**
232	* Get the segment object corresponding to segmentNo
233	* @param segmentNo
234	* @param loadMetadataDetails
235	* @return
236	*/
237	public static Segment getSegment(String segmentNo, LoadMetadataDetails[] loadMetadataDetails) {
238	for (LoadMetadataDetails details: loadMetadataDetails) {	1✔
239	if (details.getLoadName().equals(segmentNo)) {	1✔
240	return new Segment(details.getLoadName(), details.getSegmentFile());	1✔
241	}
242	}
243	return null;	1✔
244	}
245
246	public Configuration getConfiguration() {
247	return readCommittedScope.getConfiguration();	1✔
248	}
249
250	public Set<String> getFilteredIndexShardNames() {
251	return filteredIndexShardNames;	1✔
252	}
253
254	public void setFilteredIndexShardName(String filteredIndexShardName) {
255	this.filteredIndexShardNames.add(filteredIndexShardName);	1✔
256	}	1✔
257
258	@Override public boolean equals(Object o) {
259	if (this == o) return true;	2✔
260	if (o == null \|\| getClass() != o.getClass()) return false;	2✔
261	Segment segment = (Segment) o;	2✔
262	return Objects.equals(segmentNo, segment.segmentNo);	2✔
263	}
264
265	@Override public int hashCode() {
266	return Objects.hash(segmentNo);	2✔
267	}
268
269	@Override public String toString() {
270	return segmentString;	2✔
271	}
272
273	public LoadMetadataDetails getLoadMetadataDetails() {
274	return loadMetadataDetails;	1✔
275	}
276
277	public long getIndexSize() {
278	return indexSize;	1✔
279	}
280
281	public void setIndexSize(long indexSize) {
282	this.indexSize = indexSize;	1✔
283	}	1✔
284
285	@Override public void write(DataOutput out) throws IOException {
286	out.writeUTF(segmentNo);	×
287	boolean writeSegmentFileName = segmentFileName != null;	×
288	out.writeBoolean(writeSegmentFileName);	×
289	if (writeSegmentFileName) {	×
290	out.writeUTF(segmentFileName);	×
291	}
292	out.writeInt(filteredIndexShardNames.size());	×
293	for (String name: filteredIndexShardNames) {	×
294	out.writeUTF(name);	×
295	}	×
296	if (segmentString == null) {	×
297	out.writeBoolean(false);	×
298	} else {
299	out.writeBoolean(true);	×
300	out.writeUTF(segmentString);	×
301	}
302	out.writeLong(indexSize);	×
303	}	×
304
305	@Override public void readFields(DataInput in) throws IOException {
306	this.segmentNo = in.readUTF();	×
307	if (in.readBoolean()) {	×
308	this.segmentFileName = in.readUTF();	×
309	}
310	filteredIndexShardNames = new HashSet<>();	×
311	int indexShardNameSize = in.readInt();	×
312	for (int i = 0; i < indexShardNameSize; i++) {	×
313	filteredIndexShardNames.add(in.readUTF());	×
314	}
315	if (in.readBoolean()) {	×
316	this.segmentString = in.readUTF();	×
317	}
318	this.indexSize = in.readLong();	×
319	}	×
320	}

apache / carbondata / 3944

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous