#318

Build # #318

Build Type

push

Committed by

Commit Message

Feature/upgrade springboot (#268)

* update springboot to 2.1.18

* update springboot to 2.2.13

* Upgrade to srping boot 2.7.11

* Update changelog

* Update changelog

* fix version

Run Details

3 of 3 new or added lines in 2 files covered. (100.0%)

132 of 3058 relevant lines covered (4.32%)

0.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMapping.java

/**
 * Copyright (C) 2016-2021 Expedia, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hotels.bdp.waggledance.mapping.model;

import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.unescapeIdentifier;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.SortedSet;
import java.util.Stack;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.antlr.runtime.CommonToken;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.parse.ParseUtils;

import com.hotels.bdp.waggledance.api.WaggleDanceException;

public enum ASTQueryMapping implements QueryMapping {

  INSTANCE;

  private static final String PRESTO_VIEW_MARKER = "/* Presto View";
  private final static String RE_WORD_BOUNDARY = "\\b";
  private final static Comparator<CommonToken> ON_START_INDEX = Comparator.comparingInt(CommonToken::getStartIndex);

  @Override
  public String transformOutboundDatabaseName(MetaStoreMapping metaStoreMapping, String query) {
    if (hasNonHiveViewMarker(query)) {
      // skipping queries that are not "Hive" view queries. We can't parse those.
      return query;
    }
    ASTNode root;
    try {
      root = ParseUtils.parse(query);
    } catch (ParseException e) {
      throw new WaggleDanceException("Can't parse query: '" + query + "'", e);
    }

    StringBuilder result = transformDatabaseTableTokens(metaStoreMapping, root, query);
    transformFunctionTokens(metaStoreMapping, root, result);
    return result.toString();
  }

  private boolean hasNonHiveViewMarker(String query) {
    if (query != null && query.trim().startsWith(PRESTO_VIEW_MARKER)) {
      return true;
    }
    return false;
  }

  private StringBuilder transformDatabaseTableTokens(MetaStoreMapping metaStoreMapping, ASTNode root, String query) {
    StringBuilder result = new StringBuilder();
    SortedSet<CommonToken> dbNameTokens = extractDbNameTokens(root);
    int startIndex = 0;
    for (CommonToken dbNameNode : dbNameTokens) {
      final String dbName = dbNameNode.getText();
      final boolean escaped = dbName.startsWith("`") && dbName.endsWith("`");
      String transformedDbName = metaStoreMapping.transformOutboundDatabaseName(unescapeIdentifier(dbName));
      if (escaped) {
        transformedDbName = "`" + transformedDbName + "`";
      }
      result.append(query, startIndex, dbNameNode.getStartIndex());
      result.append(transformedDbName);
      startIndex = dbNameNode.getStopIndex() + 1;
    }
    result.append(query.substring(startIndex));
    return result;
  }

  private void transformFunctionTokens(MetaStoreMapping metaStoreMapping, ASTNode root, StringBuilder result) {
    // Done differently from the extractDbNameTokens as the Function tokens do not contain a correct start index. We'll
    // have to fall back to search and replace.
    List<CommonToken> functionTokens = extractFunctionTokens(root);
    for (CommonToken functionNode : functionTokens) {
      final String functionName = functionNode.getText();
      Pattern pattern = Pattern.compile(RE_WORD_BOUNDARY + functionName + RE_WORD_BOUNDARY);
      Matcher matcher = pattern.matcher(result);
      if (matcher.find()) {
        int index = matcher.start();
        String prefix = metaStoreMapping.getDatabasePrefix();
        result.replace(index, index, prefix);
      }
    }
  }

  private SortedSet<CommonToken> extractDbNameTokens(ASTNode root) {
    SortedSet<CommonToken> dbNameTokens = new TreeSet<>(ON_START_INDEX);
    Stack<ASTNode> stack = new Stack<>();
    stack.push(root);
    while (!stack.isEmpty()) {
      ASTNode current = stack.pop();
      for (ASTNode child : getChildren(current)) {
        stack.push(child);
      }
      if (current.getType() == HiveParser.TOK_TABNAME) {
        if (current.getChildCount() == 2 && childrenAreIdentifiers(current)) {
          // First child of TOK_TABNAME node is the database name node
          CommonToken dbNameNode = (CommonToken) ((ASTNode) current.getChild(0)).getToken();
          dbNameTokens.add(dbNameNode);
        }
        // Otherwise TOK_TABNAME node only has one child which contains just the table name
      }
    }
    return dbNameTokens;
  }

  private List<CommonToken> extractFunctionTokens(ASTNode root) {
    List<CommonToken> tokens = new ArrayList<>();
    Stack<ASTNode> stack = new Stack<>();
    stack.push(root);

    while (!stack.isEmpty()) {
      ASTNode current = stack.pop();
      for (ASTNode child : getChildren(current)) {
        stack.push(child);
      }
      if (current.getType() == HiveParser.TOK_FUNCTION) {
        if (current.getChildCount() == 1 && childrenAreIdentifiers(current)) {
          // TOK_FUNCTION has one child, <dbName.functionName>.
          CommonToken dbNameDotFunctionNameNode = (CommonToken) ((ASTNode) current.getChild(0)).getToken();
          tokens.add(dbNameDotFunctionNameNode);
        }
      }
    }
    return tokens;
  }

  private boolean childrenAreIdentifiers(ASTNode current) {
    for (ASTNode child : getChildren(current)) {
      if (child.getType() != HiveParser.Identifier) {
        return false;
      }
    }
    return true;
  }

  private static List<ASTNode> getChildren(ASTNode pt) {
    List<ASTNode> rt = new ArrayList<>();
    List<Node> children = pt.getChildren();
    if (children != null) {
      for (Node child : pt.getChildren()) {
        rt.add((ASTNode) child);
      }
    }
    return rt;
  }

}

1	/**
2	* Copyright (C) 2016-2021 Expedia, Inc.
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16	package com.hotels.bdp.waggledance.mapping.model;
17
18	import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.unescapeIdentifier;
19
20	import java.util.ArrayList;
21	import java.util.Comparator;
22	import java.util.List;
23	import java.util.SortedSet;
24	import java.util.Stack;
25	import java.util.TreeSet;
26	import java.util.regex.Matcher;
27	import java.util.regex.Pattern;
28
29	import org.antlr.runtime.CommonToken;
30	import org.apache.hadoop.hive.ql.lib.Node;
31	import org.apache.hadoop.hive.ql.parse.ASTNode;
32	import org.apache.hadoop.hive.ql.parse.HiveParser;
33	import org.apache.hadoop.hive.ql.parse.ParseException;
34	import org.apache.hadoop.hive.ql.parse.ParseUtils;
35
36	import com.hotels.bdp.waggledance.api.WaggleDanceException;
37
38	public enum ASTQueryMapping implements QueryMapping {
39
40	INSTANCE;	×
41
42	private static final String PRESTO_VIEW_MARKER = "/* Presto View";
43	private final static String RE_WORD_BOUNDARY = "\\b";
44	private final static Comparator<CommonToken> ON_START_INDEX = Comparator.comparingInt(CommonToken::getStartIndex);	×
45
46	@Override
47	public String transformOutboundDatabaseName(MetaStoreMapping metaStoreMapping, String query) {
48	if (hasNonHiveViewMarker(query)) {	×
49	// skipping queries that are not "Hive" view queries. We can't parse those.
50	return query;	×
51	}
52	ASTNode root;
53	try {
54	root = ParseUtils.parse(query);	×
55	} catch (ParseException e) {	×
56	throw new WaggleDanceException("Can't parse query: '" + query + "'", e);	×
57	}
58
59	StringBuilder result = transformDatabaseTableTokens(metaStoreMapping, root, query);	×
60	transformFunctionTokens(metaStoreMapping, root, result);	×
61	return result.toString();	×
62	}
63
64	private boolean hasNonHiveViewMarker(String query) {
65	if (query != null && query.trim().startsWith(PRESTO_VIEW_MARKER)) {	×
66	return true;	×
67	}
68	return false;	×
69	}
70
71	private StringBuilder transformDatabaseTableTokens(MetaStoreMapping metaStoreMapping, ASTNode root, String query) {
72	StringBuilder result = new StringBuilder();	×
73	SortedSet<CommonToken> dbNameTokens = extractDbNameTokens(root);	×
74	int startIndex = 0;	×
75	for (CommonToken dbNameNode : dbNameTokens) {	×
76	final String dbName = dbNameNode.getText();	×
77	final boolean escaped = dbName.startsWith("`") && dbName.endsWith("`");	×
78	String transformedDbName = metaStoreMapping.transformOutboundDatabaseName(unescapeIdentifier(dbName));	×
79	if (escaped) {	×
80	transformedDbName = "`" + transformedDbName + "`";	×
81	}
82	result.append(query, startIndex, dbNameNode.getStartIndex());	×
83	result.append(transformedDbName);	×
84	startIndex = dbNameNode.getStopIndex() + 1;	×
85	}
86	result.append(query.substring(startIndex));	×
87	return result;	×
88	}
89
90	private void transformFunctionTokens(MetaStoreMapping metaStoreMapping, ASTNode root, StringBuilder result) {
91	// Done differently from the extractDbNameTokens as the Function tokens do not contain a correct start index. We'll
92	// have to fall back to search and replace.
93	List<CommonToken> functionTokens = extractFunctionTokens(root);	×
94	for (CommonToken functionNode : functionTokens) {	×
95	final String functionName = functionNode.getText();	×
96	Pattern pattern = Pattern.compile(RE_WORD_BOUNDARY + functionName + RE_WORD_BOUNDARY);	×
97	Matcher matcher = pattern.matcher(result);	×
98	if (matcher.find()) {	×
99	int index = matcher.start();	×
100	String prefix = metaStoreMapping.getDatabasePrefix();	×
101	result.replace(index, index, prefix);	×
102	}
103	}
104	}	×
105
106	private SortedSet<CommonToken> extractDbNameTokens(ASTNode root) {
107	SortedSet<CommonToken> dbNameTokens = new TreeSet<>(ON_START_INDEX);	×
108	Stack<ASTNode> stack = new Stack<>();	×
109	stack.push(root);	×
110	while (!stack.isEmpty()) {	×
111	ASTNode current = stack.pop();	×
112	for (ASTNode child : getChildren(current)) {	×
113	stack.push(child);	×
114	}
115	if (current.getType() == HiveParser.TOK_TABNAME) {	×
116	if (current.getChildCount() == 2 && childrenAreIdentifiers(current)) {	×
117	// First child of TOK_TABNAME node is the database name node
118	CommonToken dbNameNode = (CommonToken) ((ASTNode) current.getChild(0)).getToken();	×
119	dbNameTokens.add(dbNameNode);	×
120	}
121	// Otherwise TOK_TABNAME node only has one child which contains just the table name
122	}
123	}
124	return dbNameTokens;	×
125	}
126
127	private List<CommonToken> extractFunctionTokens(ASTNode root) {
128	List<CommonToken> tokens = new ArrayList<>();	×
129	Stack<ASTNode> stack = new Stack<>();	×
130	stack.push(root);	×
131
132	while (!stack.isEmpty()) {	×
133	ASTNode current = stack.pop();	×
134	for (ASTNode child : getChildren(current)) {	×
135	stack.push(child);	×
136	}
137	if (current.getType() == HiveParser.TOK_FUNCTION) {	×
138	if (current.getChildCount() == 1 && childrenAreIdentifiers(current)) {	×
139	// TOK_FUNCTION has one child, <dbName.functionName>.
140	CommonToken dbNameDotFunctionNameNode = (CommonToken) ((ASTNode) current.getChild(0)).getToken();	×
141	tokens.add(dbNameDotFunctionNameNode);	×
142	}
143	}
144	}
145	return tokens;	×
146	}
147
148	private boolean childrenAreIdentifiers(ASTNode current) {
149	for (ASTNode child : getChildren(current)) {	×
150	if (child.getType() != HiveParser.Identifier) {	×
151	return false;	×
152	}
153	}
154	return true;	×
155	}
156
157	private static List<ASTNode> getChildren(ASTNode pt) {
158	List<ASTNode> rt = new ArrayList<>();	×
159	List<Node> children = pt.getChildren();	×
160	if (children != null) {	×
161	for (Node child : pt.getChildren()) {	×
162	rt.add((ASTNode) child);	×
163	}
164	}
165	return rt;	×
166	}
167
168	}

HotelsDotCom / waggle-dance / #318

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous