#361

Committed 20 Oct 2023 03:42PM UTC coverage: 73.43% (+0.01%) from 73.418%

Build # #361

Build Type

push

Committed by

Commit Message

Added timeout on isAvailable calls (#300)

* Added timeout

* Update CHANGELOG.md

Run Details

16 of 16 new or added lines in 1 file covered. (100.0%)

2385 of 3248 relevant lines covered (73.43%)

0.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.83

/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java

/**
 * Copyright (C) 2016-2023 Expedia, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hotels.bdp.waggledance.mapping.model;

import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import org.apache.hadoop.hive.metastore.MetaStoreFilterHook;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.hotels.bdp.waggledance.api.model.ConnectionType;
import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIface;
import com.hotels.bdp.waggledance.server.security.AccessControlHandler;
import com.hotels.bdp.waggledance.server.security.NotAllowedException;

class MetaStoreMappingImpl implements MetaStoreMapping {

  private final static Logger log = LoggerFactory.getLogger(MetaStoreMappingImpl.class);

  // MilliSeconds
  static final long DEFAULT_AVAILABILITY_TIMEOUT = 500;

  private final String databasePrefix;
  private final CloseableThriftHiveMetastoreIface client;
  private final AccessControlHandler accessControlHandler;
  private final String name;
  private final long latency;
  private final MetaStoreFilterHook metastoreFilter;

  private final ConnectionType connectionType;

  MetaStoreMappingImpl(
      String databasePrefix,
      String name,
      CloseableThriftHiveMetastoreIface client,
      AccessControlHandler accessControlHandler,
      ConnectionType connectionType,
      long latency,
      MetaStoreFilterHook metastoreFilter) {
    this.databasePrefix = databasePrefix;
    this.name = name;
    this.client = client;
    this.accessControlHandler = accessControlHandler;
    this.connectionType = connectionType;
    this.latency = latency;
    this.metastoreFilter = metastoreFilter;
  }

  @Override
  public String transformOutboundDatabaseName(String databaseName) {
    return databaseName.toLowerCase(Locale.ROOT);
  }

  @Override
  public List<String> transformOutboundDatabaseNameMultiple(String databaseName) {
    return Collections.singletonList(transformInboundDatabaseName(databaseName));
  }

  @Override
  public ThriftHiveMetastore.Iface getClient() {
    return client;
  }

  @Override
  public MetaStoreFilterHook getMetastoreFilter() {
    return metastoreFilter;
  }

  @Override
  public Database transformOutboundDatabase(Database database) {
    database.setName(transformOutboundDatabaseName(database.getName()));
    return database;
  }

  @Override
  public String transformInboundDatabaseName(String databaseName) {
    return databaseName.toLowerCase(Locale.ROOT);
  }

  @Override
  public String getDatabasePrefix() {
    return databasePrefix;
  }

  @Override
  public void close() throws IOException {
    client.close();
  }

  /**
   * This is potentially slow so a best effort is made and false is returned after a timeout.
   */
  @Override
  public boolean isAvailable() {
    Future<Boolean> future = CompletableFuture.supplyAsync(() -> {
      try {
        boolean isOpen = client.isOpen();
        if (isOpen && connectionType == ConnectionType.TUNNELED) {
          client.getStatus();
        }
        return isOpen;
      } catch (Exception e) {
        log.error("Metastore Mapping {} unavailable", name, e);
        return false;
      }
    });
    try {
      return future.get(DEFAULT_AVAILABILITY_TIMEOUT + getLatency(), TimeUnit.MILLISECONDS);
    } catch (TimeoutException e) {
      log.info("Took too long to check availability of '" + name + "', assuming unavailable");
      future.cancel(true);
    } catch (InterruptedException | ExecutionException e) {
      log.error("Error while checking availability '" + name + "', assuming unavailable");
    }
    return false;
  }

  @Override
  public MetaStoreMapping checkWritePermissions(String databaseName) {
    if (!accessControlHandler.hasWritePermission(databaseName)) {
      throw new NotAllowedException(
          "You cannot perform this operation on the virtual database '" + databaseName + "'.");
    }
    return this;
  }

  @Override
  public void createDatabase(Database database)
    throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
    if (accessControlHandler.hasCreatePermission()) {
      getClient().create_database(database);
      accessControlHandler.databaseCreatedNotification(database.getName());
    } else {
      throw new NotAllowedException("You cannot create the database '" + database.getName() + "'.");
    }
  }

  @Override
  public String getMetastoreMappingName() {
    return name;
  }

  @Override
  public long getLatency() {
    return latency;
  }

}

1	/**
2	* Copyright (C) 2016-2023 Expedia, Inc.
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16	package com.hotels.bdp.waggledance.mapping.model;
17
18	import java.io.IOException;
19	import java.util.Collections;
20	import java.util.List;
21	import java.util.Locale;
22	import java.util.concurrent.CompletableFuture;
23	import java.util.concurrent.ExecutionException;
24	import java.util.concurrent.Future;
25	import java.util.concurrent.TimeUnit;
26	import java.util.concurrent.TimeoutException;
27
28	import org.apache.hadoop.hive.metastore.MetaStoreFilterHook;
29	import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
30	import org.apache.hadoop.hive.metastore.api.Database;
31	import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
32	import org.apache.hadoop.hive.metastore.api.MetaException;
33	import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
34	import org.apache.thrift.TException;
35	import org.slf4j.Logger;
36	import org.slf4j.LoggerFactory;
37
38	import com.hotels.bdp.waggledance.api.model.ConnectionType;
39	import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIface;
40	import com.hotels.bdp.waggledance.server.security.AccessControlHandler;
41	import com.hotels.bdp.waggledance.server.security.NotAllowedException;
42
43	class MetaStoreMappingImpl implements MetaStoreMapping {
44
45	private final static Logger log = LoggerFactory.getLogger(MetaStoreMappingImpl.class);	1✔
46
47	// MilliSeconds
48	static final long DEFAULT_AVAILABILITY_TIMEOUT = 500;	1✔
49
50	private final String databasePrefix;
51	private final CloseableThriftHiveMetastoreIface client;
52	private final AccessControlHandler accessControlHandler;
53	private final String name;
54	private final long latency;
55	private final MetaStoreFilterHook metastoreFilter;
56
57	private final ConnectionType connectionType;
58
59	MetaStoreMappingImpl(	1✔
60	String databasePrefix,
61	String name,
62	CloseableThriftHiveMetastoreIface client,
63	AccessControlHandler accessControlHandler,
64	ConnectionType connectionType,
65	long latency,
66	MetaStoreFilterHook metastoreFilter) {
67	this.databasePrefix = databasePrefix;	1✔
68	this.name = name;	1✔
69	this.client = client;	1✔
70	this.accessControlHandler = accessControlHandler;	1✔
71	this.connectionType = connectionType;	1✔
72	this.latency = latency;	1✔
73	this.metastoreFilter = metastoreFilter;	1✔
74	}	1✔
75
76	@Override
77	public String transformOutboundDatabaseName(String databaseName) {
78	return databaseName.toLowerCase(Locale.ROOT);	1✔
79	}
80
81	@Override
82	public List<String> transformOutboundDatabaseNameMultiple(String databaseName) {
83	return Collections.singletonList(transformInboundDatabaseName(databaseName));	1✔
84	}
85
86	@Override
87	public ThriftHiveMetastore.Iface getClient() {
88	return client;	1✔
89	}
90
91	@Override
92	public MetaStoreFilterHook getMetastoreFilter() {
93	return metastoreFilter;	1✔
94	}
95
96	@Override
97	public Database transformOutboundDatabase(Database database) {
98	database.setName(transformOutboundDatabaseName(database.getName()));	1✔
99	return database;	1✔
100	}
101
102	@Override
103	public String transformInboundDatabaseName(String databaseName) {
104	return databaseName.toLowerCase(Locale.ROOT);	1✔
105	}
106
107	@Override
108	public String getDatabasePrefix() {
109	return databasePrefix;	1✔
110	}
111
112	@Override
113	public void close() throws IOException {
114	client.close();	1✔
115	}	1✔
116
117	/**
118	* This is potentially slow so a best effort is made and false is returned after a timeout.
119	*/
120	@Override
121	public boolean isAvailable() {
122	Future<Boolean> future = CompletableFuture.supplyAsync(() -> {	1✔
123	try {
124	boolean isOpen = client.isOpen();	1✔
125	if (isOpen && connectionType == ConnectionType.TUNNELED) {	1✔
126	client.getStatus();	1✔
127	}
128	return isOpen;	1✔
129	} catch (Exception e) {	1✔
130	log.error("Metastore Mapping {} unavailable", name, e);	1✔
131	return false;	1✔
132	}
133	});
134	try {
135	return future.get(DEFAULT_AVAILABILITY_TIMEOUT + getLatency(), TimeUnit.MILLISECONDS);	1✔
136	} catch (TimeoutException e) {	1✔
137	log.info("Took too long to check availability of '" + name + "', assuming unavailable");	1✔
138	future.cancel(true);	1✔
139	} catch (InterruptedException \| ExecutionException e) {	×
140	log.error("Error while checking availability '" + name + "', assuming unavailable");	×
141	}
142	return false;	1✔
143	}
144
145	@Override
146	public MetaStoreMapping checkWritePermissions(String databaseName) {
147	if (!accessControlHandler.hasWritePermission(databaseName)) {	1✔
148	throw new NotAllowedException(	1✔
149	"You cannot perform this operation on the virtual database '" + databaseName + "'.");	1✔
150	}
151	return this;	1✔
152	}
153
154	@Override
155	public void createDatabase(Database database)
156	throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
157	if (accessControlHandler.hasCreatePermission()) {	1✔
158	getClient().create_database(database);	1✔
159	accessControlHandler.databaseCreatedNotification(database.getName());	1✔
160	} else {	1✔
161	throw new NotAllowedException("You cannot create the database '" + database.getName() + "'.");	1✔
162	}
163	}	1✔
164
165	@Override
166	public String getMetastoreMappingName() {
167	return name;	1✔
168	}
169
170	@Override
171	public long getLatency() {
172	return latency;	1✔
173	}
174
175	}

HotelsDotCom / waggle-dance / #361

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous