galaxyproject / cloudman / 428

pending completion

Build # 428

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #140 from galaxyproject/add_rke_support

Added rke support

Run Details

380 of 484 branches covered (78.51%)

Branch coverage included in aggregate %.

42 of 42 new or added lines in 5 files covered. (100.0%)

27 existing lines in 4 files now uncovered.

3655 of 3844 relevant lines covered (95.08%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.92

/cloudman/clusterman/cluster_templates.py

import abc
from rest_framework.exceptions import ValidationError
from cloudlaunch import models as cl_models


class CMClusterTemplate(object):

    def __init__(self, context, cluster):
        self.context = context
        self.cluster = cluster

    @property
    def connection_settings(self):
        return self.cluster.connection_settings

    @abc.abstractmethod
    def add_node(self, name, size):
        pass

    @abc.abstractmethod
    def remove_node(self):
        pass

    @abc.abstractmethod
    def activate_autoscaling(self, min_nodes=0, max_nodes=None, size=None):
        pass

    @abc.abstractmethod
    def deactivate_autoscaling(self):
        pass

    @staticmethod
    def get_template_for(context, cluster):
        if cluster.cluster_type == "KUBE_RKE":
            return CMRKETemplate(context, cluster)
        else:
            raise KeyError("Cannon get cluster template for unknown cluster "
                           "type: %s" % cluster.cluster_type)


class CMRKETemplate(CMClusterTemplate):

    def __init__(self, context, cluster):
        super(CMRKETemplate, self).__init__(context, cluster)
        settings = cluster.connection_settings.get('rke_config')
        self._rke_registration_server = settings.get('rke_registration_server')
        self._rke_registration_token = settings.get('rke_registration_token')
        self._rke_cluster_id = settings.get('rke_cluster_id')

    @property
    def rke_registration_server(self):
        return self._rke_registration_server

    @property
    def rke_registration_token(self):
        return self._rke_registration_token

    @property
    def rke_cluster_id(self):
        return self._rke_cluster_id

    def _find_matching_vm_type(self, zone_model=None, default_vm_type=None,
                               min_vcpus=0, min_ram=0, vm_family=""):
        """
        Finds the vm_type that best matches the given criteria. If no criteria
        is specified, will return the default vm type.

        :param zone_model:
        :param default_vm_type:
        :param min_vcpus:
        :param min_ram:
        :param vm_family:
        :return:
        """
        vm_type = default_vm_type or self.cluster.default_vm_type
        if min_vcpus > 0 or min_ram > 0 or not vm_type.startswith(vm_family):
            # Add some accommodation for rancher and k8s reserved resources
            # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
            min_vcpus += 1.0
            min_ram *= 1.1

            cloud = self.context.cloudlaunch_client.infrastructure.clouds.get(
                zone_model.region.cloud.id)
            region = cloud.regions.get(zone_model.region.region_id)
            zone = region.zones.get(zone_model.zone_id)
            default_matches = zone.vm_types.list(vm_type_prefix=vm_type)
            if default_matches:
                default_match = default_matches[0]
                min_vcpus = min_vcpus if min_vcpus > float(default_match.vcpus) else default_match.vcpus
                min_ram = min_ram if min_ram > float(default_match.ram) else default_match.ram
            candidates = zone.vm_types.list(min_vcpus=min_vcpus, min_ram=min_ram,
                                            vm_type_prefix=vm_family)
            if candidates:
                candidate_type = sorted(candidates, key=lambda x: float(x.vcpus) * float(x.ram))[0]
                return candidate_type.name
        return vm_type

    def add_node(self, name, vm_type=None, zone=None, min_vcpus=0, min_ram=0, vm_family=""):
        settings = self.cluster.connection_settings
        zone = zone or self.cluster.default_zone
        deployment_target = cl_models.CloudDeploymentTarget.objects.get(
            target_zone=zone)
        params = {
            'name': name,
            'application': 'cm_rke_kubernetes_plugin',
            'deployment_target_id': deployment_target.id,
            'application_version': '0.1.0',
            'config_app': {
                'action': 'add_node',
                'config_kube_rke': {
                    'rke_registration_server': self.rke_registration_server,
                    'rke_registration_token': self.rke_registration_token,
                    'rke_cluster_id': self.rke_cluster_id
                },
                "config_appliance": {
                    "sshUser": "ubuntu",
                    "runner": "ansible",
                    "repository": "https://github.com/CloudVE/cloudman-boot",
                    "inventoryTemplate":
                        "[controllers]\n\n"
                        "[agents]\n"
                        "${host}\n\n"
                        "[rke_cluster:children]\n"
                        "controllers\n"
                        "agents\n\n"
                        "[all:vars]\n"
                        "ansible_ssh_port=22\n"
                        "ansible_user='${user}'\n"
                        "ansible_ssh_private_key_file=pk\n"
                        "ansible_ssh_extra_args='-o StrictHostKeyChecking=no"
                        " -o ControlMaster=no'\n"
                },
                'config_cloudlaunch': (settings.get('app_config', {})
                                       .get('config_cloudlaunch', {})),
                'config_cloudman': {
                    'cluster_name': self.cluster.name
                }
            }
        }

        params['config_app']['config_cloudlaunch']['vmType'] = \
            self._find_matching_vm_type(
                zone_model=zone, default_vm_type=vm_type, min_vcpus=min_vcpus,
                min_ram=min_ram, vm_family=vm_family)

        print("Adding node: {0} of type: {1}".format(
            name, params['config_app']['config_cloudlaunch']['vmType']))

        # Don't use hostname config
        params['config_app']['config_cloudlaunch'].pop('hostnameConfig', None)
        try:
            print("Launching node with settings: {0}".format(params))
            return self.context.cloudlaunch_client.deployments.create(**params)
        except Exception as e:
            raise ValidationError("Could not launch node: " + str(e))

    def remove_node(self, node):
        return self.context.cloudlaunch_client.deployments.tasks.create(
            action='DELETE', deployment_pk=node.deployment.pk)

1	import abc	1✔
2	from rest_framework.exceptions import ValidationError	1✔
3	from cloudlaunch import models as cl_models	1✔
4
5
6	class CMClusterTemplate(object):	1✔
7
8	def __init__(self, context, cluster):	1✔
9	self.context = context	1✔
10	self.cluster = cluster	1✔
11
12	@property	1✔
13	def connection_settings(self):
14	return self.cluster.connection_settings	×
15
16	@abc.abstractmethod	1✔
17	def add_node(self, name, size):
18	pass	×
19
20	@abc.abstractmethod	1✔
21	def remove_node(self):
22	pass	×
23
24	@abc.abstractmethod	1✔
25	def activate_autoscaling(self, min_nodes=0, max_nodes=None, size=None):	1✔
26	pass	×
27
28	@abc.abstractmethod	1✔
29	def deactivate_autoscaling(self):
30	pass	×
31
32	@staticmethod	1✔
33	def get_template_for(context, cluster):
34	if cluster.cluster_type == "KUBE_RKE":	1!
35	return CMRKETemplate(context, cluster)	1✔
36	else:
37	raise KeyError("Cannon get cluster template for unknown cluster "	×
38	"type: %s" % cluster.cluster_type)
39
40
41	class CMRKETemplate(CMClusterTemplate):	1✔
42
43	def __init__(self, context, cluster):	1✔
44	super(CMRKETemplate, self).__init__(context, cluster)	1✔
45	settings = cluster.connection_settings.get('rke_config')	1✔
46	self._rke_registration_server = settings.get('rke_registration_server')	1✔
47	self._rke_registration_token = settings.get('rke_registration_token')	1✔
48	self._rke_cluster_id = settings.get('rke_cluster_id')	1✔
49
50	@property	1✔
51	def rke_registration_server(self):
52	return self._rke_registration_server	1✔
53
54	@property	1✔
55	def rke_registration_token(self):
56	return self._rke_registration_token	1✔
57
58	@property	1✔
59	def rke_cluster_id(self):
60	return self._rke_cluster_id	1✔
61
62	def _find_matching_vm_type(self, zone_model=None, default_vm_type=None,	1✔
63	min_vcpus=0, min_ram=0, vm_family=""):
64	"""
65	Finds the vm_type that best matches the given criteria. If no criteria
66	is specified, will return the default vm type.
67
68	:param zone_model:
69	:param default_vm_type:
70	:param min_vcpus:
71	:param min_ram:
72	:param vm_family:
73	:return:
74	"""
75	vm_type = default_vm_type or self.cluster.default_vm_type	1✔
76	if min_vcpus > 0 or min_ram > 0 or not vm_type.startswith(vm_family):	1✔
77	# Add some accommodation for rancher and k8s reserved resources
78	# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
79	min_vcpus += 1.0	1✔
80	min_ram *= 1.1	1✔
81
82	cloud = self.context.cloudlaunch_client.infrastructure.clouds.get(	1✔
83	zone_model.region.cloud.id)
84	region = cloud.regions.get(zone_model.region.region_id)	1✔
85	zone = region.zones.get(zone_model.zone_id)	1✔
86	default_matches = zone.vm_types.list(vm_type_prefix=vm_type)	1✔
87	if default_matches:	1!
UNCOV 88	default_match = default_matches[0]	×
UNCOV 89	min_vcpus = min_vcpus if min_vcpus > float(default_match.vcpus) else default_match.vcpus	×
UNCOV 90	min_ram = min_ram if min_ram > float(default_match.ram) else default_match.ram	×
91	candidates = zone.vm_types.list(min_vcpus=min_vcpus, min_ram=min_ram,	1✔
92	vm_type_prefix=vm_family)
93	if candidates:	1!
UNCOV 94	candidate_type = sorted(candidates, key=lambda x: float(x.vcpus) * float(x.ram))[0]	×
UNCOV 95	return candidate_type.name	×
96	return vm_type	1✔
97
98	def add_node(self, name, vm_type=None, zone=None, min_vcpus=0, min_ram=0, vm_family=""):	1✔
99	settings = self.cluster.connection_settings	1✔
100	zone = zone or self.cluster.default_zone	1✔
101	deployment_target = cl_models.CloudDeploymentTarget.objects.get(	1✔
102	target_zone=zone)
103	params = {	1✔
104	'name': name,
105	'application': 'cm_rke_kubernetes_plugin',
106	'deployment_target_id': deployment_target.id,
107	'application_version': '0.1.0',
108	'config_app': {
109	'action': 'add_node',
110	'config_kube_rke': {
111	'rke_registration_server': self.rke_registration_server,
112	'rke_registration_token': self.rke_registration_token,
113	'rke_cluster_id': self.rke_cluster_id
114	},
115	"config_appliance": {
116	"sshUser": "ubuntu",
117	"runner": "ansible",
118	"repository": "https://github.com/CloudVE/cloudman-boot",
119	"inventoryTemplate":
120	"[controllers]\n\n"
121	"[agents]\n"
122	"${host}\n\n"
123	"[rke_cluster:children]\n"
124	"controllers\n"
125	"agents\n\n"
126	"[all:vars]\n"
127	"ansible_ssh_port=22\n"
128	"ansible_user='${user}'\n"
129	"ansible_ssh_private_key_file=pk\n"
130	"ansible_ssh_extra_args='-o StrictHostKeyChecking=no"
131	" -o ControlMaster=no'\n"
132	},
133	'config_cloudlaunch': (settings.get('app_config', {})
134	.get('config_cloudlaunch', {})),
135	'config_cloudman': {
136	'cluster_name': self.cluster.name
137	}
138	}
139	}
140
141	params['config_app']['config_cloudlaunch']['vmType'] = \	1✔
142	self._find_matching_vm_type(
143	zone_model=zone, default_vm_type=vm_type, min_vcpus=min_vcpus,
144	min_ram=min_ram, vm_family=vm_family)
145
146	print("Adding node: {0} of type: {1}".format(	1✔
147	name, params['config_app']['config_cloudlaunch']['vmType']))
148
149	# Don't use hostname config
150	params['config_app']['config_cloudlaunch'].pop('hostnameConfig', None)	1✔
151	try:	1✔
152	print("Launching node with settings: {0}".format(params))	1✔
153	return self.context.cloudlaunch_client.deployments.create(**params)	1✔
154	except Exception as e:	×
155	raise ValidationError("Could not launch node: " + str(e))	×
156
157	def remove_node(self, node):	1✔
158	return self.context.cloudlaunch_client.deployments.tasks.create(	1✔
159	action='DELETE', deployment_pk=node.deployment.pk)

galaxyproject / cloudman / 428

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous