5048326742

Build Type

push

github

Committed by GitHub

Commit Message

Merge pull request #2796 from misl6/release-2023.05.21

Run Details

909 of 2275 branches covered (39.96%)

Branch coverage included in aggregate %.

27 of 39 new or added lines in 9 files covered. (69.23%)

5 existing lines in 2 files now uncovered.

4690 of 7432 relevant lines covered (63.11%)

2.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.39

/pythonforandroid/distribution.py

from os.path import exists, join
import glob
import json

from pythonforandroid.logger import (debug, info, info_notify, warning, Err_Style, Err_Fore)
from pythonforandroid.util import current_directory, BuildInterruptingException
from shutil import rmtree


class Distribution:
    '''State container for information about a distribution (i.e. an
    Android project).

    This is separate from a Bootstrap because the Bootstrap is
    concerned with building and populating the dist directory, whereas
    the dist itself could also come from e.g. a binary download.
    '''
    ctx = None

    name = None  # A name identifying the dist. May not be None.
    needs_build = False  # Whether the dist needs compiling
    url = None
    dist_dir = None  # Where the dist dir ultimately is. Should not be None.
    ndk_api = None

    archs = []
    '''The names of the arch targets that the dist is built for.'''

    recipes = []

    description = ''  # A long description

    def __init__(self, ctx):
        self.ctx = ctx

    def __str__(self):
        return '<Distribution: name {} with recipes ({})>'.format(
            # self.name, ', '.join([recipe.name for recipe in self.recipes]))
            self.name, ', '.join(self.recipes))

    def __repr__(self):
        return str(self)

    @classmethod
    def get_distribution(
            cls,
            ctx,
            *,
            archs,  # required keyword argument: there is no sensible default
            name=None,
            recipes=[],
            ndk_api=None,
            force_build=False,
            extra_dist_dirs=[],
            require_perfect_match=False,
            allow_replace_dist=True
    ):
        '''Takes information about the distribution, and decides what kind of
        distribution it will be.

        If parameters conflict (e.g. a dist with that name already
        exists, but doesn't have the right set of recipes),
        an error is thrown.

        Parameters
        ----------
        name : str
            The name of the distribution. If a dist with this name already '
            exists, it will be used.
        ndk_api : int
            The NDK API to compile against, included in the dist because it cannot
            be changed later during APK packaging.
        archs : list
            The target architectures list to compile against, included in the dist because
            it cannot be changed later during APK packaging.
        recipes : list
            The recipes that the distribution must contain.
        force_download: bool
            If True, only downloaded dists are considered.
        force_build : bool
            If True, the dist is forced to be built locally.
        extra_dist_dirs : list
            Any extra directories in which to search for dists.
        require_perfect_match : bool
            If True, will only match distributions with precisely the
            correct set of recipes.
        allow_replace_dist : bool
            If True, will allow an existing dist with the specified
            name but incompatible requirements to be overwritten by
            a new one with the current requirements.
        '''

        possible_dists = Distribution.get_distributions(ctx)
        debug(f"All possible dists: {possible_dists}")

        # Will hold dists that would be built in the same folder as an existing dist
        folder_match_dist = None

        # 0) Check if a dist with that name and architecture already exists
        if name is not None and name:
            possible_dists = [
                d for d in possible_dists if
                (d.name == name) and all(arch_name in d.archs for arch_name in archs)]
            debug(f"Dist matching name and arch: {possible_dists}")

            if possible_dists:
                # There should only be one folder with a given dist name *and* arch.
                # We could check that here, but for compatibility let's let it slide
                # and just record the details of one of them. We only use this data to
                # possibly fail the build later, so it doesn't really matter if there
                # was more than one clash.
                folder_match_dist = possible_dists[0]

        # 1) Check if any existing dists meet the requirements
        _possible_dists = []
        for dist in possible_dists:
            if (
                ndk_api is not None and dist.ndk_api != ndk_api
            ) or dist.ndk_api is None:
                debug(
                    f"dist {dist} failed to match ndk_api, target api {ndk_api}, dist api {dist.ndk_api}"
                )
                continue
            for recipe in recipes:
                if recipe not in dist.recipes:
                    debug(f"dist {dist} missing recipe {recipe}")
                    break
            else:
                _possible_dists.append(dist)
        possible_dists = _possible_dists
        debug(f"Dist matching ndk_api and recipe: {possible_dists}")

        if possible_dists:
            info('Of the existing distributions, the following meet '
                 'the given requirements:')
            pretty_log_dists(possible_dists)
        else:
            info('No existing dists meet the given requirements!')

        # If any dist has perfect recipes, arch and NDK API, return it
        for dist in possible_dists:
            if force_build:
                debug("Skipping dist due to forced build")
                continue
            if ndk_api is not None and dist.ndk_api != ndk_api:
                debug("Skipping dist due to ndk_api mismatch")
                continue
            if not all(arch_name in dist.archs for arch_name in archs):
                debug("Skipping dist due to arch mismatch")
                continue
            if (set(dist.recipes) == set(recipes) or
                (set(recipes).issubset(set(dist.recipes)) and
                 not require_perfect_match)):
                info_notify('{} has compatible recipes, using this one'
                            .format(dist.name))
                return dist
            else:
                debug(
                    f"Skipping dist due to recipes mismatch, expected {set(recipes)}, actual {set(dist.recipes)}"
                )

        # If there was a name match but we didn't already choose it,
        # then the existing dist is incompatible with the requested
        # configuration and the build cannot continue
        if folder_match_dist is not None and not allow_replace_dist:
            raise BuildInterruptingException(
                'Asked for dist with name {name} with recipes ({req_recipes}) and '
                'NDK API {req_ndk_api}, but a dist '
                'with this name already exists and has either incompatible recipes '
                '({dist_recipes}) or NDK API {dist_ndk_api}'.format(
                    name=name,
                    req_ndk_api=ndk_api,
                    dist_ndk_api=folder_match_dist.ndk_api,
                    req_recipes=', '.join(recipes),
                    dist_recipes=', '.join(folder_match_dist.recipes)))

        assert len(possible_dists) < 2

        # If we got this far, we need to build a new dist
        dist = Distribution(ctx)
        dist.needs_build = True

        if not name:
            filen = 'unnamed_dist_{}'
            i = 1
            while exists(join(ctx.dist_dir, filen.format(i))):
                i += 1
            name = filen.format(i)

        dist.name = name
        dist.dist_dir = join(
            ctx.dist_dir,
            name)
        dist.recipes = recipes
        dist.ndk_api = ctx.ndk_api
        dist.archs = archs

        return dist

    def folder_exists(self):
        return exists(self.dist_dir)

    def delete(self):
        rmtree(self.dist_dir)

    @classmethod
    def get_distributions(cls, ctx, extra_dist_dirs=[]):
        '''Returns all the distributions found locally.'''
        if extra_dist_dirs:
            raise BuildInterruptingException(
                'extra_dist_dirs argument to get_distributions '
                'is not yet implemented')
        dist_dir = ctx.dist_dir
        folders = glob.glob(join(dist_dir, '*'))
        for dir in extra_dist_dirs:
            folders.extend(glob.glob(join(dir, '*')))

        dists = []
        for folder in folders:
            if exists(join(folder, 'dist_info.json')):
                with open(join(folder, 'dist_info.json')) as fileh:
                    dist_info = json.load(fileh)
                dist = cls(ctx)
                dist.name = dist_info['dist_name']
                dist.dist_dir = folder
                dist.needs_build = False
                dist.recipes = dist_info['recipes']
                if 'archs' in dist_info:
                    dist.archs = dist_info['archs']
                if 'ndk_api' in dist_info:
                    dist.ndk_api = dist_info['ndk_api']
                else:
                    dist.ndk_api = None
                    warning(
                        "Distribution {distname}: ({distdir}) has been "
                        "built with an unknown api target, ignoring it, "
                        "you might want to delete it".format(
                            distname=dist.name,
                            distdir=dist.dist_dir
                        )
                    )
                dists.append(dist)
        return dists

    def save_info(self, dirn):
        '''
        Save information about the distribution in its dist_dir.
        '''
        with current_directory(dirn):
            info('Saving distribution info')
            with open('dist_info.json', 'w') as fileh:
                json.dump({'dist_name': self.name,
                           'bootstrap': self.ctx.bootstrap.name,
                           'archs': [arch.arch for arch in self.ctx.archs],
                           'ndk_api': self.ctx.ndk_api,
                           'use_setup_py': self.ctx.use_setup_py,
                           'recipes': self.ctx.recipe_build_order + self.ctx.python_modules,
                           'hostpython': self.ctx.hostpython,
                           'python_version': self.ctx.python_recipe.major_minor_version_string},
                          fileh)


def pretty_log_dists(dists, log_func=info):
    infos = []
    for dist in dists:
        ndk_api = 'unknown' if dist.ndk_api is None else dist.ndk_api
        infos.append('{Fore.GREEN}{Style.BRIGHT}{name}{Style.RESET_ALL}: min API {ndk_api}, '
                     'includes recipes ({Fore.GREEN}{recipes}'
                     '{Style.RESET_ALL}), built for archs ({Fore.BLUE}'
                     '{archs}{Style.RESET_ALL})'.format(
                         ndk_api=ndk_api,
                         name=dist.name, recipes=', '.join(dist.recipes),
                         archs=', '.join(dist.archs) if dist.archs else 'UNKNOWN',
                         Fore=Err_Fore, Style=Err_Style))

    for line in infos:
        log_func('\t' + line)

1	from os.path import exists, join	4✔
2	import glob	4✔
3	import json	4✔
4
5	from pythonforandroid.logger import (debug, info, info_notify, warning, Err_Style, Err_Fore)	4✔
6	from pythonforandroid.util import current_directory, BuildInterruptingException	4✔
7	from shutil import rmtree	4✔
8
9
10	class Distribution:	4✔
11	'''State container for information about a distribution (i.e. an
12	Android project).
13
14	This is separate from a Bootstrap because the Bootstrap is
15	concerned with building and populating the dist directory, whereas
16	the dist itself could also come from e.g. a binary download.
17	'''
18	ctx = None	4✔
19
20	name = None # A name identifying the dist. May not be None.	4✔
21	needs_build = False # Whether the dist needs compiling	4✔
22	url = None	4✔
23	dist_dir = None # Where the dist dir ultimately is. Should not be None.	4✔
24	ndk_api = None	4✔
25
26	archs = []	4✔
27	'''The names of the arch targets that the dist is built for.'''	1✔
28
29	recipes = []	4✔
30
31	description = '' # A long description	4✔
32
33	def __init__(self, ctx):	4✔
34	self.ctx = ctx	4✔
35
36	def __str__(self):	4✔
37	return '<Distribution: name {} with recipes ({})>'.format(	4✔
38	# self.name, ', '.join([recipe.name for recipe in self.recipes]))
39	self.name, ', '.join(self.recipes))
40
41	def __repr__(self):
42	return str(self)
43
44	@classmethod	4✔
45	def get_distribution(	4✔
46	cls,
47	ctx,
48	*,
49	archs, # required keyword argument: there is no sensible default
50	name=None,
51	recipes=[],
52	ndk_api=None,
53	force_build=False,
54	extra_dist_dirs=[],
55	require_perfect_match=False,
56	allow_replace_dist=True
57	):
58	'''Takes information about the distribution, and decides what kind of
59	distribution it will be.
60
61	If parameters conflict (e.g. a dist with that name already
62	exists, but doesn't have the right set of recipes),
63	an error is thrown.
64
65	Parameters
66	----------
67	name : str
68	The name of the distribution. If a dist with this name already '
69	exists, it will be used.
70	ndk_api : int
71	The NDK API to compile against, included in the dist because it cannot
72	be changed later during APK packaging.
73	archs : list
74	The target architectures list to compile against, included in the dist because
75	it cannot be changed later during APK packaging.
76	recipes : list
77	The recipes that the distribution must contain.
78	force_download: bool
79	If True, only downloaded dists are considered.
80	force_build : bool
81	If True, the dist is forced to be built locally.
82	extra_dist_dirs : list
83	Any extra directories in which to search for dists.
84	require_perfect_match : bool
85	If True, will only match distributions with precisely the
86	correct set of recipes.
87	allow_replace_dist : bool
88	If True, will allow an existing dist with the specified
89	name but incompatible requirements to be overwritten by
90	a new one with the current requirements.
91	'''
92
93	possible_dists = Distribution.get_distributions(ctx)	4✔
94	debug(f"All possible dists: {possible_dists}")	4✔
95
96	# Will hold dists that would be built in the same folder as an existing dist
97	folder_match_dist = None	4✔
98
99	# 0) Check if a dist with that name and architecture already exists
100	if name is not None and name:	4✔
101	possible_dists = [	4✔
102	d for d in possible_dists if
103	(d.name == name) and all(arch_name in d.archs for arch_name in archs)]
104	debug(f"Dist matching name and arch: {possible_dists}")	4✔
105
106	if possible_dists:	4✔
107	# There should only be one folder with a given dist name and arch.
108	# We could check that here, but for compatibility let's let it slide
109	# and just record the details of one of them. We only use this data to
110	# possibly fail the build later, so it doesn't really matter if there
111	# was more than one clash.
112	folder_match_dist = possible_dists[0]	4✔
113
114	# 1) Check if any existing dists meet the requirements
115	_possible_dists = []	4✔
116	for dist in possible_dists:	4✔
117	if (	4✔
118	ndk_api is not None and dist.ndk_api != ndk_api
119	) or dist.ndk_api is None:
120	debug(	4✔
121	f"dist {dist} failed to match ndk_api, target api {ndk_api}, dist api {dist.ndk_api}"
122	)
123	continue	4✔
124	for recipe in recipes:	4✔
125	if recipe not in dist.recipes:	4!
NEW 126	debug(f"dist {dist} missing recipe {recipe}")	×
UNCOV 127	break	×
128	else:
129	_possible_dists.append(dist)	4✔
130	possible_dists = _possible_dists	4✔
131	debug(f"Dist matching ndk_api and recipe: {possible_dists}")	4✔
132
133	if possible_dists:	4✔
134	info('Of the existing distributions, the following meet '	4✔
135	'the given requirements:')
136	pretty_log_dists(possible_dists)	4✔
137	else:
138	info('No existing dists meet the given requirements!')	4✔
139
140	# If any dist has perfect recipes, arch and NDK API, return it
141	for dist in possible_dists:	4✔
142	if force_build:	4!
NEW 143	debug("Skipping dist due to forced build")	×
UNCOV 144	continue	×
145	if ndk_api is not None and dist.ndk_api != ndk_api:	4!
NEW 146	debug("Skipping dist due to ndk_api mismatch")	×
UNCOV 147	continue	×
148	if not all(arch_name in dist.archs for arch_name in archs):	4!
NEW 149	debug("Skipping dist due to arch mismatch")	×
UNCOV 150	continue	×
151	if (set(dist.recipes) == set(recipes) or	4!
152	(set(recipes).issubset(set(dist.recipes)) and
153	not require_perfect_match)):
154	info_notify('{} has compatible recipes, using this one'	4✔
155	.format(dist.name))
156	return dist	4✔
157	else:
NEW 158	debug(	×
159	f"Skipping dist due to recipes mismatch, expected {set(recipes)}, actual {set(dist.recipes)}"
160	)
161
162	# If there was a name match but we didn't already choose it,
163	# then the existing dist is incompatible with the requested
164	# configuration and the build cannot continue
165	if folder_match_dist is not None and not allow_replace_dist:	4✔
166	raise BuildInterruptingException(	4✔
167	'Asked for dist with name {name} with recipes ({req_recipes}) and '
168	'NDK API {req_ndk_api}, but a dist '
169	'with this name already exists and has either incompatible recipes '
170	'({dist_recipes}) or NDK API {dist_ndk_api}'.format(
171	name=name,
172	req_ndk_api=ndk_api,
173	dist_ndk_api=folder_match_dist.ndk_api,
174	req_recipes=', '.join(recipes),
175	dist_recipes=', '.join(folder_match_dist.recipes)))
176
177	assert len(possible_dists) < 2	4✔
178
179	# If we got this far, we need to build a new dist
180	dist = Distribution(ctx)	4✔
181	dist.needs_build = True	4✔
182
183	if not name:	4✔
184	filen = 'unnamed_dist_{}'	4✔
185	i = 1	4✔
186	while exists(join(ctx.dist_dir, filen.format(i))):	4!
187	i += 1	×
188	name = filen.format(i)	4✔
189
190	dist.name = name	4✔
191	dist.dist_dir = join(	4✔
192	ctx.dist_dir,
193	name)
194	dist.recipes = recipes	4✔
195	dist.ndk_api = ctx.ndk_api	4✔
196	dist.archs = archs	4✔
197
198	return dist	4✔
199
200	def folder_exists(self):	4✔
201	return exists(self.dist_dir)	4✔
202
203	def delete(self):	4✔
204	rmtree(self.dist_dir)	4✔
205
206	@classmethod	4✔
207	def get_distributions(cls, ctx, extra_dist_dirs=[]):	4✔
208	'''Returns all the distributions found locally.'''
209	if extra_dist_dirs:	4✔
210	raise BuildInterruptingException(	4✔
211	'extra_dist_dirs argument to get_distributions '
212	'is not yet implemented')
213	dist_dir = ctx.dist_dir	4✔
214	folders = glob.glob(join(dist_dir, '*'))	4✔
215	for dir in extra_dist_dirs:	4!
216	folders.extend(glob.glob(join(dir, '*')))	×
217
218	dists = []	4✔
219	for folder in folders:	4✔
220	if exists(join(folder, 'dist_info.json')):	4✔
221	with open(join(folder, 'dist_info.json')) as fileh:	4✔
222	dist_info = json.load(fileh)	4✔
223	dist = cls(ctx)	4✔
224	dist.name = dist_info['dist_name']	4✔
225	dist.dist_dir = folder	4✔
226	dist.needs_build = False	4✔
227	dist.recipes = dist_info['recipes']	4✔
228	if 'archs' in dist_info:	4!
229	dist.archs = dist_info['archs']	4✔
230	if 'ndk_api' in dist_info:	4✔
231	dist.ndk_api = dist_info['ndk_api']	4✔
232	else:
233	dist.ndk_api = None	4✔
234	warning(	4✔
235	"Distribution {distname}: ({distdir}) has been "
236	"built with an unknown api target, ignoring it, "
237	"you might want to delete it".format(
238	distname=dist.name,
239	distdir=dist.dist_dir
240	)
241	)
242	dists.append(dist)	4✔
243	return dists	4✔
244
245	def save_info(self, dirn):	4✔
246	'''
247	Save information about the distribution in its dist_dir.
248	'''
249	with current_directory(dirn):	4✔
250	info('Saving distribution info')	4✔
251	with open('dist_info.json', 'w') as fileh:	4✔
252	json.dump({'dist_name': self.name,	4✔
253	'bootstrap': self.ctx.bootstrap.name,
254	'archs': [arch.arch for arch in self.ctx.archs],
255	'ndk_api': self.ctx.ndk_api,
256	'use_setup_py': self.ctx.use_setup_py,
257	'recipes': self.ctx.recipe_build_order + self.ctx.python_modules,
258	'hostpython': self.ctx.hostpython,
259	'python_version': self.ctx.python_recipe.major_minor_version_string},
260	fileh)
261
262
263	def pretty_log_dists(dists, log_func=info):	4✔
264	infos = []	4✔
265	for dist in dists:	4✔
266	ndk_api = 'unknown' if dist.ndk_api is None else dist.ndk_api	4✔
267	infos.append('{Fore.GREEN}{Style.BRIGHT}{name}{Style.RESET_ALL}: min API {ndk_api}, '	4✔
268	'includes recipes ({Fore.GREEN}{recipes}'
269	'{Style.RESET_ALL}), built for archs ({Fore.BLUE}'
270	'{archs}{Style.RESET_ALL})'.format(
271	ndk_api=ndk_api,
272	name=dist.name, recipes=', '.join(dist.recipes),
273	archs=', '.join(dist.archs) if dist.archs else 'UNKNOWN',
274	Fore=Err_Fore, Style=Err_Style))
275
276	for line in infos:	4✔
277	log_func('\t' + line)	4✔

kivy / python-for-android / 5048326742

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous