• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

kivy / python-for-android / 10427251988

16 Aug 2024 10:54PM UTC coverage: 59.035%. Remained the same
10427251988

push

github

web-flow
Merge pull request #3049 from kivy/feature/bump_sh_dep

:arrow_up: Bump sh module to v2, refs #2746

1048 of 2363 branches covered (44.35%)

Branch coverage included in aggregate %.

0 of 4 new or added lines in 2 files covered. (0.0%)

1 existing line in 1 file now uncovered.

4862 of 7648 relevant lines covered (63.57%)

2.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

45.05
/pythonforandroid/recipe.py
1
from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split
4✔
2
import glob
4✔
3

4
import hashlib
4✔
5
from re import match
4✔
6

7
import sh
4✔
8
import shutil
4✔
9
import fnmatch
4✔
10
import zipfile
4✔
11
import urllib.request
4✔
12
from urllib.request import urlretrieve
4✔
13
from os import listdir, unlink, environ, curdir, walk
4✔
14
from sys import stdout
4✔
15
import time
4✔
16
try:
4✔
17
    from urlparse import urlparse
4✔
18
except ImportError:
4✔
19
    from urllib.parse import urlparse
4✔
20

21
import packaging.version
4✔
22

23
from pythonforandroid.logger import (
4✔
24
    logger, info, warning, debug, shprint, info_main, error)
25
from pythonforandroid.util import (
4✔
26
    current_directory, ensure_dir, BuildInterruptingException, rmdir, move,
27
    touch, patch_wheel_setuptools_logging)
28
from pythonforandroid.util import load_source as import_recipe
4✔
29

30

31
url_opener = urllib.request.build_opener()
4✔
32
url_orig_headers = url_opener.addheaders
4✔
33
urllib.request.install_opener(url_opener)
4✔
34

35

36
class RecipeMeta(type):
4✔
37
    def __new__(cls, name, bases, dct):
4✔
38
        if name != 'Recipe':
4✔
39
            if 'url' in dct:
4✔
40
                dct['_url'] = dct.pop('url')
4✔
41
            if 'version' in dct:
4✔
42
                dct['_version'] = dct.pop('version')
4✔
43

44
        return super().__new__(cls, name, bases, dct)
4✔
45

46

47
class Recipe(metaclass=RecipeMeta):
4✔
48
    _url = None
4✔
49
    '''The address from which the recipe may be downloaded. This is not
2✔
50
    essential, it may be omitted if the source is available some other
51
    way, such as via the :class:`IncludedFilesBehaviour` mixin.
52

53
    If the url includes the version, you may (and probably should)
54
    replace this with ``{version}``, which will automatically be
55
    replaced by the :attr:`version` string during download.
56

57
    .. note:: Methods marked (internal) are used internally and you
58
              probably don't need to call them, but they are available
59
              if you want.
60
    '''
61

62
    _version = None
4✔
63
    '''A string giving the version of the software the recipe describes,
2✔
64
    e.g. ``2.0.3`` or ``master``.'''
65

66
    md5sum = None
4✔
67
    '''The md5sum of the source from the :attr:`url`. Non-essential, but
2✔
68
    you should try to include this, it is used to check that the download
69
    finished correctly.
70
    '''
71

72
    sha512sum = None
4✔
73
    '''The sha512sum of the source from the :attr:`url`. Non-essential, but
2✔
74
    you should try to include this, it is used to check that the download
75
    finished correctly.
76
    '''
77

78
    blake2bsum = None
4✔
79
    '''The blake2bsum of the source from the :attr:`url`. Non-essential, but
2✔
80
    you should try to include this, it is used to check that the download
81
    finished correctly.
82
    '''
83

84
    depends = []
4✔
85
    '''A list containing the names of any recipes that this recipe depends on.
2✔
86
    '''
87

88
    conflicts = []
4✔
89
    '''A list containing the names of any recipes that are known to be
2✔
90
    incompatible with this one.'''
91

92
    opt_depends = []
4✔
93
    '''A list of optional dependencies, that must be built before this
2✔
94
    recipe if they are built at all, but whose presence is not essential.'''
95

96
    patches = []
4✔
97
    '''A list of patches to apply to the source. Values can be either a string
2✔
98
    referring to the patch file relative to the recipe dir, or a tuple of the
99
    string patch file and a callable, which will receive the kwargs `arch` and
100
    `recipe`, which should return True if the patch should be applied.'''
101

102
    python_depends = []
4✔
103
    '''A list of pure-Python packages that this package requires. These
2✔
104
    packages will NOT be available at build time, but will be added to the
105
    list of pure-Python packages to install via pip. If you need these packages
106
    at build time, you must create a recipe.'''
107

108
    archs = ['armeabi']  # Not currently implemented properly
4✔
109

110
    built_libraries = {}
4✔
111
    """Each recipe that builds a system library (e.g.:libffi, openssl, etc...)
2✔
112
    should contain a dict holding the relevant information of the library. The
113
    keys should be the generated libraries and the values the relative path of
114
    the library inside his build folder. This dict will be used to perform
115
    different operations:
116
        - copy the library into the right location, depending on if it's shared
117
          or static)
118
        - check if we have to rebuild the library
119

120
    Here an example of how it would look like for `libffi` recipe:
121

122
        - `built_libraries = {'libffi.so': '.libs'}`
123

124
    .. note:: in case that the built library resides in recipe's build
125
              directory, you can set the following values for the relative
126
              path: `'.', None or ''`
127
    """
128

129
    need_stl_shared = False
4✔
130
    '''Some libraries or python packages may need the c++_shared in APK.
2✔
131
    We can automatically do this for any recipe if we set this property to
132
    `True`'''
133

134
    stl_lib_name = 'c++_shared'
4✔
135
    '''
2✔
136
    The default STL shared lib to use: `c++_shared`.
137

138
    .. note:: Android NDK version > 17 only supports 'c++_shared', because
139
        starting from NDK r18 the `gnustl_shared` lib has been deprecated.
140
    '''
141

142
    def get_stl_library(self, arch):
4✔
143
        return join(
4✔
144
            arch.ndk_lib_dir,
145
            'lib{name}.so'.format(name=self.stl_lib_name),
146
        )
147

148
    def install_stl_lib(self, arch):
4✔
149
        if not self.ctx.has_lib(
4!
150
            arch.arch, 'lib{name}.so'.format(name=self.stl_lib_name)
151
        ):
152
            self.install_libs(arch, self.get_stl_library(arch))
4✔
153

154
    @property
4✔
155
    def version(self):
4✔
156
        key = 'VERSION_' + self.name
4✔
157
        return environ.get(key, self._version)
4✔
158

159
    @property
4✔
160
    def url(self):
4✔
161
        key = 'URL_' + self.name
4✔
162
        return environ.get(key, self._url)
4✔
163

164
    @property
4✔
165
    def versioned_url(self):
4✔
166
        '''A property returning the url of the recipe with ``{version}``
167
        replaced by the :attr:`url`. If accessing the url, you should use this
168
        property, *not* access the url directly.'''
169
        if self.url is None:
4!
170
            return None
×
171
        return self.url.format(version=self.version)
4✔
172

173
    def download_file(self, url, target, cwd=None):
4✔
174
        """
175
        (internal) Download an ``url`` to a ``target``.
176
        """
177
        if not url:
4!
178
            return
×
179

180
        info('Downloading {} from {}'.format(self.name, url))
4✔
181

182
        if cwd:
4!
183
            target = join(cwd, target)
×
184

185
        parsed_url = urlparse(url)
4✔
186
        if parsed_url.scheme in ('http', 'https'):
4!
187
            def report_hook(index, blksize, size):
4✔
188
                if size <= 0:
×
189
                    progression = '{0} bytes'.format(index * blksize)
×
190
                else:
191
                    progression = '{0:.2f}%'.format(
×
192
                        index * blksize * 100. / float(size))
193
                if "CI" not in environ:
×
194
                    stdout.write('- Download {}\r'.format(progression))
×
195
                    stdout.flush()
×
196

197
            if exists(target):
4!
198
                unlink(target)
×
199

200
            # Download item with multiple attempts (for bad connections):
201
            attempts = 0
4✔
202
            seconds = 1
4✔
203
            while True:
2✔
204
                try:
4✔
205
                    # jqueryui.com returns a 403 w/ the default user agent
206
                    # Mozilla/5.0 doesnt handle redirection for liblzma
207
                    url_opener.addheaders = [('User-agent', 'Wget/1.0')]
4✔
208
                    urlretrieve(url, target, report_hook)
4✔
209
                except OSError as e:
4✔
210
                    attempts += 1
4✔
211
                    if attempts >= 5:
4✔
212
                        raise
4✔
213
                    stdout.write('Download failed: {}; retrying in {} second(s)...'.format(e, seconds))
4✔
214
                    time.sleep(seconds)
4✔
215
                    seconds *= 2
4✔
216
                    continue
4✔
217
                finally:
218
                    url_opener.addheaders = url_orig_headers
4✔
219
                break
3✔
220
            return target
4✔
221
        elif parsed_url.scheme in ('git', 'git+file', 'git+ssh', 'git+http', 'git+https'):
×
222
            if not isdir(target):
×
223
                if url.startswith('git+'):
×
224
                    url = url[4:]
×
225
                # if 'version' is specified, do a shallow clone
226
                if self.version:
×
227
                    ensure_dir(target)
×
228
                    with current_directory(target):
×
229
                        shprint(sh.git, 'init')
×
230
                        shprint(sh.git, 'remote', 'add', 'origin', url)
×
231
                else:
232
                    shprint(sh.git, 'clone', '--recursive', url, target)
×
233
            with current_directory(target):
×
234
                if self.version:
×
235
                    shprint(sh.git, 'fetch', '--tags', '--depth', '1')
×
236
                    shprint(sh.git, 'checkout', self.version)
×
237
                branch = sh.git('branch', '--show-current')
×
238
                if branch:
×
239
                    shprint(sh.git, 'pull')
×
240
                    shprint(sh.git, 'pull', '--recurse-submodules')
×
241
                shprint(sh.git, 'submodule', 'update', '--recursive', '--init', '--depth', '1')
×
242
            return target
×
243

244
    def apply_patch(self, filename, arch, build_dir=None):
4✔
245
        """
246
        Apply a patch from the current recipe directory into the current
247
        build directory.
248

249
        .. versionchanged:: 0.6.0
250
            Add ability to apply patch from any dir via kwarg `build_dir`'''
251
        """
252
        info("Applying patch {}".format(filename))
4✔
253
        build_dir = build_dir if build_dir else self.get_build_dir(arch)
4✔
254
        filename = join(self.get_recipe_dir(), filename)
4✔
255
        shprint(sh.patch, "-t", "-d", build_dir, "-p1",
4✔
256
                "-i", filename, _tail=10)
257

258
    def copy_file(self, filename, dest):
4✔
259
        info("Copy {} to {}".format(filename, dest))
×
260
        filename = join(self.get_recipe_dir(), filename)
×
261
        dest = join(self.build_dir, dest)
×
262
        shutil.copy(filename, dest)
×
263

264
    def append_file(self, filename, dest):
4✔
265
        info("Append {} to {}".format(filename, dest))
×
266
        filename = join(self.get_recipe_dir(), filename)
×
267
        dest = join(self.build_dir, dest)
×
268
        with open(filename, "rb") as fd:
×
269
            data = fd.read()
×
270
        with open(dest, "ab") as fd:
×
271
            fd.write(data)
×
272

273
    @property
4✔
274
    def name(self):
4✔
275
        '''The name of the recipe, the same as the folder containing it.'''
276
        modname = self.__class__.__module__
4✔
277
        return modname.split(".", 2)[-1]
4✔
278

279
    @property
4✔
280
    def filtered_archs(self):
4✔
281
        '''Return archs of self.ctx that are valid build archs
282
        for the Recipe.'''
283
        result = []
×
284
        for arch in self.ctx.archs:
×
285
            if not self.archs or (arch.arch in self.archs):
×
286
                result.append(arch)
×
287
        return result
×
288

289
    def check_recipe_choices(self):
4✔
290
        '''Checks what recipes are being built to see which of the alternative
291
        and optional dependencies are being used,
292
        and returns a list of these.'''
293
        recipes = []
4✔
294
        built_recipes = self.ctx.recipe_build_order
4✔
295
        for recipe in self.depends:
4✔
296
            if isinstance(recipe, (tuple, list)):
4!
297
                for alternative in recipe:
×
298
                    if alternative in built_recipes:
×
299
                        recipes.append(alternative)
×
300
                        break
×
301
        for recipe in self.opt_depends:
4✔
302
            if recipe in built_recipes:
4!
303
                recipes.append(recipe)
×
304
        return sorted(recipes)
4✔
305

306
    def get_opt_depends_in_list(self, recipes):
4✔
307
        '''Given a list of recipe names, returns those that are also in
308
        self.opt_depends.
309
        '''
310
        return [recipe for recipe in recipes if recipe in self.opt_depends]
4✔
311

312
    def get_build_container_dir(self, arch):
4✔
313
        '''Given the arch name, returns the directory where it will be
314
        built.
315

316
        This returns a different directory depending on what
317
        alternative or optional dependencies are being built.
318
        '''
319
        dir_name = self.get_dir_name()
4✔
320
        return join(self.ctx.build_dir, 'other_builds',
4✔
321
                    dir_name, '{}__ndk_target_{}'.format(arch, self.ctx.ndk_api))
322

323
    def get_dir_name(self):
4✔
324
        choices = self.check_recipe_choices()
4✔
325
        dir_name = '-'.join([self.name] + choices)
4✔
326
        return dir_name
4✔
327

328
    def get_build_dir(self, arch):
4✔
329
        '''Given the arch name, returns the directory where the
330
        downloaded/copied package will be built.'''
331

332
        return join(self.get_build_container_dir(arch), self.name)
4✔
333

334
    def get_recipe_dir(self):
4✔
335
        """
336
        Returns the local recipe directory or defaults to the core recipe
337
        directory.
338
        """
339
        if self.ctx.local_recipes is not None:
4!
340
            local_recipe_dir = join(self.ctx.local_recipes, self.name)
×
341
            if exists(local_recipe_dir):
×
342
                return local_recipe_dir
×
343
        return join(self.ctx.root_dir, 'recipes', self.name)
4✔
344

345
    # Public Recipe API to be subclassed if needed
346

347
    def download_if_necessary(self):
4✔
348
        info_main('Downloading {}'.format(self.name))
4✔
349
        user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
4✔
350
        if user_dir is not None:
4✔
351
            info('P4A_{}_DIR is set, skipping download for {}'.format(
4✔
352
                self.name, self.name))
353
            return
4✔
354
        self.download()
4✔
355

356
    def download(self):
4✔
357
        if self.url is None:
4✔
358
            info('Skipping {} download as no URL is set'.format(self.name))
4✔
359
            return
4✔
360

361
        url = self.versioned_url
4✔
362
        expected_digests = {}
4✔
363
        for alg in set(hashlib.algorithms_guaranteed) | set(('md5', 'sha512', 'blake2b')):
4✔
364
            expected_digest = getattr(self, alg + 'sum') if hasattr(self, alg + 'sum') else None
4✔
365
            ma = match(u'^(.+)#' + alg + u'=([0-9a-f]{32,})$', url)
4✔
366
            if ma:                # fragmented URL?
4!
367
                if expected_digest:
×
368
                    raise ValueError(
×
369
                        ('Received {}sum from both the {} recipe '
370
                         'and its url').format(alg, self.name))
371
                url = ma.group(1)
×
372
                expected_digest = ma.group(2)
×
373
            if expected_digest:
4!
374
                expected_digests[alg] = expected_digest
×
375

376
        ensure_dir(join(self.ctx.packages_path, self.name))
4✔
377

378
        with current_directory(join(self.ctx.packages_path, self.name)):
4✔
379
            filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8')
4✔
380

381
            do_download = True
4✔
382
            marker_filename = '.mark-{}'.format(filename)
4✔
383
            if exists(filename) and isfile(filename):
4!
384
                if not exists(marker_filename):
×
385
                    shprint(sh.rm, filename)
×
386
                else:
387
                    for alg, expected_digest in expected_digests.items():
×
388
                        current_digest = algsum(alg, filename)
×
389
                        if current_digest != expected_digest:
×
390
                            debug('* Generated {}sum: {}'.format(alg,
×
391
                                                                 current_digest))
392
                            debug('* Expected {}sum: {}'.format(alg,
×
393
                                                                expected_digest))
394
                            raise ValueError(
×
395
                                ('Generated {0}sum does not match expected {0}sum '
396
                                 'for {1} recipe').format(alg, self.name))
397
                    do_download = False
×
398

399
            # If we got this far, we will download
400
            if do_download:
4!
401
                debug('Downloading {} from {}'.format(self.name, url))
4✔
402

403
                shprint(sh.rm, '-f', marker_filename)
4✔
404
                self.download_file(self.versioned_url, filename)
4✔
405
                touch(marker_filename)
4✔
406

407
                if exists(filename) and isfile(filename):
4!
408
                    for alg, expected_digest in expected_digests.items():
×
409
                        current_digest = algsum(alg, filename)
×
410
                        if current_digest != expected_digest:
×
411
                            debug('* Generated {}sum: {}'.format(alg,
×
412
                                                                 current_digest))
413
                            debug('* Expected {}sum: {}'.format(alg,
×
414
                                                                expected_digest))
415
                            raise ValueError(
×
416
                                ('Generated {0}sum does not match expected {0}sum '
417
                                 'for {1} recipe').format(alg, self.name))
418
            else:
419
                info('{} download already cached, skipping'.format(self.name))
×
420

421
    def unpack(self, arch):
4✔
422
        info_main('Unpacking {} for {}'.format(self.name, arch))
×
423

424
        build_dir = self.get_build_container_dir(arch)
×
425

426
        user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
×
427
        if user_dir is not None:
×
428
            info('P4A_{}_DIR exists, symlinking instead'.format(
×
429
                self.name.lower()))
430
            if exists(self.get_build_dir(arch)):
×
431
                return
×
432
            rmdir(build_dir)
×
433
            ensure_dir(build_dir)
×
434
            shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
×
435
            return
×
436

437
        if self.url is None:
×
438
            info('Skipping {} unpack as no URL is set'.format(self.name))
×
439
            return
×
440

441
        filename = shprint(
×
442
            sh.basename, self.versioned_url).stdout[:-1].decode('utf-8')
443
        ma = match(u'^(.+)#[a-z0-9_]{3,}=([0-9a-f]{32,})$', filename)
×
444
        if ma:                  # fragmented URL?
×
445
            filename = ma.group(1)
×
446

447
        with current_directory(build_dir):
×
448
            directory_name = self.get_build_dir(arch)
×
449

450
            if not exists(directory_name) or not isdir(directory_name):
×
451
                extraction_filename = join(
×
452
                    self.ctx.packages_path, self.name, filename)
453
                if isfile(extraction_filename):
×
454
                    if extraction_filename.endswith(('.zip', '.whl')):
×
455
                        try:
×
456
                            sh.unzip(extraction_filename)
×
457
                        except (sh.ErrorReturnCode_1, sh.ErrorReturnCode_2):
×
458
                            # return code 1 means unzipping had
459
                            # warnings but did complete,
460
                            # apparently happens sometimes with
461
                            # github zips
462
                            pass
×
463
                        fileh = zipfile.ZipFile(extraction_filename, 'r')
×
464
                        root_directory = fileh.filelist[0].filename.split('/')[0]
×
465
                        if root_directory != basename(directory_name):
×
466
                            move(root_directory, directory_name)
×
467
                    elif extraction_filename.endswith(
×
468
                            ('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz', '.txz')):
469
                        sh.tar('xf', extraction_filename)
×
NEW
470
                        root_directory = sh.tar('tf', extraction_filename).split('\n')[0].split('/')[0]
×
UNCOV
471
                        if root_directory != basename(directory_name):
×
472
                            move(root_directory, directory_name)
×
473
                    else:
474
                        raise Exception(
×
475
                            'Could not extract {} download, it must be .zip, '
476
                            '.tar.gz or .tar.bz2 or .tar.xz'.format(extraction_filename))
477
                elif isdir(extraction_filename):
×
478
                    ensure_dir(directory_name)
×
479
                    for entry in listdir(extraction_filename):
×
480
                        # Previously we filtered out the .git folder, but during the build process for some recipes
481
                        # (e.g. when version is parsed by `setuptools_scm`) that may be needed.
482
                        shprint(sh.cp, '-Rv',
×
483
                                join(extraction_filename, entry),
484
                                directory_name)
485
                else:
486
                    raise Exception(
×
487
                        'Given path is neither a file nor a directory: {}'
488
                        .format(extraction_filename))
489

490
            else:
491
                info('{} is already unpacked, skipping'.format(self.name))
×
492

493
    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
4✔
494
        """Return the env specialized for the recipe
495
        """
496
        if arch is None:
4!
497
            arch = self.filtered_archs[0]
×
498
        env = arch.get_env(with_flags_in_cc=with_flags_in_cc)
4✔
499
        return env
4✔
500

501
    def prebuild_arch(self, arch):
4✔
502
        '''Run any pre-build tasks for the Recipe. By default, this checks if
503
        any prebuild_archname methods exist for the archname of the current
504
        architecture, and runs them if so.'''
505
        prebuild = "prebuild_{}".format(arch.arch.replace('-', '_'))
4✔
506
        if hasattr(self, prebuild):
4!
507
            getattr(self, prebuild)()
×
508
        else:
509
            info('{} has no {}, skipping'.format(self.name, prebuild))
4✔
510

511
    def is_patched(self, arch):
4✔
512
        build_dir = self.get_build_dir(arch.arch)
4✔
513
        return exists(join(build_dir, '.patched'))
4✔
514

515
    def apply_patches(self, arch, build_dir=None):
4✔
516
        '''Apply any patches for the Recipe.
517

518
        .. versionchanged:: 0.6.0
519
            Add ability to apply patches from any dir via kwarg `build_dir`'''
520
        if self.patches:
×
521
            info_main('Applying patches for {}[{}]'
×
522
                      .format(self.name, arch.arch))
523

524
            if self.is_patched(arch):
×
525
                info_main('{} already patched, skipping'.format(self.name))
×
526
                return
×
527

528
            build_dir = build_dir if build_dir else self.get_build_dir(arch.arch)
×
529
            for patch in self.patches:
×
530
                if isinstance(patch, (tuple, list)):
×
531
                    patch, patch_check = patch
×
532
                    if not patch_check(arch=arch, recipe=self):
×
533
                        continue
×
534

535
                self.apply_patch(
×
536
                        patch.format(version=self.version, arch=arch.arch),
537
                        arch.arch, build_dir=build_dir)
538

539
            touch(join(build_dir, '.patched'))
×
540

541
    def should_build(self, arch):
4✔
542
        '''Should perform any necessary test and return True only if it needs
543
        building again. Per default we implement a library test, in case that
544
        we detect so.
545

546
        '''
547
        if self.built_libraries:
4!
548
            return not all(
4✔
549
                exists(lib) for lib in self.get_libraries(arch.arch)
550
            )
551
        return True
×
552

553
    def build_arch(self, arch):
4✔
554
        '''Run any build tasks for the Recipe. By default, this checks if
555
        any build_archname methods exist for the archname of the current
556
        architecture, and runs them if so.'''
557
        build = "build_{}".format(arch.arch)
×
558
        if hasattr(self, build):
×
559
            getattr(self, build)()
×
560

561
    def install_libraries(self, arch):
4✔
562
        '''This method is always called after `build_arch`. In case that we
563
        detect a library recipe, defined by the class attribute
564
        `built_libraries`, we will copy all defined libraries into the
565
         right location.
566
        '''
567
        if not self.built_libraries:
4!
568
            return
×
569
        shared_libs = [
4✔
570
            lib for lib in self.get_libraries(arch) if lib.endswith(".so")
571
        ]
572
        self.install_libs(arch, *shared_libs)
4✔
573

574
    def postbuild_arch(self, arch):
4✔
575
        '''Run any post-build tasks for the Recipe. By default, this checks if
576
        any postbuild_archname methods exist for the archname of the
577
        current architecture, and runs them if so.
578
        '''
579
        postbuild = "postbuild_{}".format(arch.arch)
4✔
580
        if hasattr(self, postbuild):
4!
581
            getattr(self, postbuild)()
×
582

583
        if self.need_stl_shared:
4!
584
            self.install_stl_lib(arch)
4✔
585

586
    def prepare_build_dir(self, arch):
4✔
587
        '''Copies the recipe data into a build dir for the given arch. By
588
        default, this unpacks a downloaded recipe. You should override
589
        it (or use a Recipe subclass with different behaviour) if you
590
        want to do something else.
591
        '''
592
        self.unpack(arch)
×
593

594
    def clean_build(self, arch=None):
4✔
595
        '''Deletes all the build information of the recipe.
596

597
        If arch is not None, only this arch dir is deleted. Otherwise
598
        (the default) all builds for all archs are deleted.
599

600
        By default, this just deletes the main build dir. If the
601
        recipe has e.g. object files biglinked, or .so files stored
602
        elsewhere, you should override this method.
603

604
        This method is intended for testing purposes, it may have
605
        strange results. Rebuild everything if this seems to happen.
606

607
        '''
608
        if arch is None:
×
609
            base_dir = join(self.ctx.build_dir, 'other_builds', self.name)
×
610
        else:
611
            base_dir = self.get_build_container_dir(arch)
×
612
        dirs = glob.glob(base_dir + '-*')
×
613
        if exists(base_dir):
×
614
            dirs.append(base_dir)
×
615
        if not dirs:
×
616
            warning('Attempted to clean build for {} but found no existing '
×
617
                    'build dirs'.format(self.name))
618

619
        for directory in dirs:
×
620
            rmdir(directory)
×
621

622
        # Delete any Python distributions to ensure the recipe build
623
        # doesn't persist in site-packages
624
        rmdir(self.ctx.python_installs_dir)
×
625

626
    def install_libs(self, arch, *libs):
4✔
627
        libs_dir = self.ctx.get_libs_dir(arch.arch)
4✔
628
        if not libs:
4!
629
            warning('install_libs called with no libraries to install!')
×
630
            return
×
631
        args = libs + (libs_dir,)
4✔
632
        shprint(sh.cp, *args)
4✔
633

634
    def has_libs(self, arch, *libs):
4✔
635
        return all(map(lambda lib: self.ctx.has_lib(arch.arch, lib), libs))
×
636

637
    def get_libraries(self, arch_name, in_context=False):
4✔
638
        """Return the full path of the library depending on the architecture.
639
        Per default, the build library path it will be returned, unless
640
        `get_libraries` has been called with kwarg `in_context` set to
641
        True.
642

643
        .. note:: this method should be used for library recipes only
644
        """
645
        recipe_libs = set()
4✔
646
        if not self.built_libraries:
4!
647
            return recipe_libs
×
648
        for lib, rel_path in self.built_libraries.items():
4✔
649
            if not in_context:
4!
650
                abs_path = join(self.get_build_dir(arch_name), rel_path, lib)
4✔
651
                if rel_path in {".", "", None}:
4✔
652
                    abs_path = join(self.get_build_dir(arch_name), lib)
4✔
653
            else:
654
                abs_path = join(self.ctx.get_libs_dir(arch_name), lib)
×
655
            recipe_libs.add(abs_path)
4✔
656
        return recipe_libs
4✔
657

658
    @classmethod
4✔
659
    def recipe_dirs(cls, ctx):
4✔
660
        recipe_dirs = []
4✔
661
        if ctx.local_recipes is not None:
4✔
662
            recipe_dirs.append(realpath(ctx.local_recipes))
4✔
663
        if ctx.storage_dir:
4✔
664
            recipe_dirs.append(join(ctx.storage_dir, 'recipes'))
4✔
665
        recipe_dirs.append(join(ctx.root_dir, "recipes"))
4✔
666
        return recipe_dirs
4✔
667

668
    @classmethod
4✔
669
    def list_recipes(cls, ctx):
4✔
670
        forbidden_dirs = ('__pycache__', )
4✔
671
        for recipes_dir in cls.recipe_dirs(ctx):
4✔
672
            if recipes_dir and exists(recipes_dir):
4✔
673
                for name in listdir(recipes_dir):
4✔
674
                    if name in forbidden_dirs:
4✔
675
                        continue
4✔
676
                    fn = join(recipes_dir, name)
4✔
677
                    if isdir(fn):
4✔
678
                        yield name
4✔
679

680
    @classmethod
4✔
681
    def get_recipe(cls, name, ctx):
4✔
682
        '''Returns the Recipe with the given name, if it exists.'''
683
        name = name.lower()
4✔
684
        if not hasattr(cls, "recipes"):
4✔
685
            cls.recipes = {}
4✔
686
        if name in cls.recipes:
4✔
687
            return cls.recipes[name]
4✔
688

689
        recipe_file = None
4✔
690
        for recipes_dir in cls.recipe_dirs(ctx):
4✔
691
            if not exists(recipes_dir):
4✔
692
                continue
4✔
693
            # Find matching folder (may differ in case):
694
            for subfolder in listdir(recipes_dir):
4✔
695
                if subfolder.lower() == name:
4✔
696
                    recipe_file = join(recipes_dir, subfolder, '__init__.py')
4✔
697
                    if exists(recipe_file):
4!
698
                        name = subfolder  # adapt to actual spelling
4✔
699
                        break
4✔
700
                    recipe_file = None
×
701
            if recipe_file is not None:
4✔
702
                break
4✔
703

704
        else:
705
            raise ValueError('Recipe does not exist: {}'.format(name))
4✔
706

707
        mod = import_recipe('pythonforandroid.recipes.{}'.format(name), recipe_file)
4✔
708
        if len(logger.handlers) > 1:
4!
709
            logger.removeHandler(logger.handlers[1])
×
710
        recipe = mod.recipe
4✔
711
        recipe.ctx = ctx
4✔
712
        cls.recipes[name.lower()] = recipe
4✔
713
        return recipe
4✔
714

715

716
class IncludedFilesBehaviour(object):
4✔
717
    '''Recipe mixin class that will automatically unpack files included in
718
    the recipe directory.'''
719
    src_filename = None
4✔
720

721
    def prepare_build_dir(self, arch):
4✔
722
        if self.src_filename is None:
×
723
            raise BuildInterruptingException(
×
724
                'IncludedFilesBehaviour failed: no src_filename specified')
725
        rmdir(self.get_build_dir(arch))
×
726
        shprint(sh.cp, '-a', join(self.get_recipe_dir(), self.src_filename),
×
727
                self.get_build_dir(arch))
728

729

730
class BootstrapNDKRecipe(Recipe):
4✔
731
    '''A recipe class for recipes built in an Android project jni dir with
732
    an Android.mk. These are not cached separatly, but built in the
733
    bootstrap's own building directory.
734

735
    To build an NDK project which is not part of the bootstrap, see
736
    :class:`~pythonforandroid.recipe.NDKRecipe`.
737

738
    To link with python, call the method :meth:`get_recipe_env`
739
    with the kwarg *with_python=True*.
740
    '''
741

742
    dir_name = None  # The name of the recipe build folder in the jni dir
4✔
743

744
    def get_build_container_dir(self, arch):
4✔
745
        return self.get_jni_dir()
×
746

747
    def get_build_dir(self, arch):
4✔
748
        if self.dir_name is None:
×
749
            raise ValueError('{} recipe doesn\'t define a dir_name, but '
×
750
                             'this is necessary'.format(self.name))
751
        return join(self.get_build_container_dir(arch), self.dir_name)
×
752

753
    def get_jni_dir(self):
4✔
754
        return join(self.ctx.bootstrap.build_dir, 'jni')
4✔
755

756
    def get_recipe_env(self, arch=None, with_flags_in_cc=True, with_python=False):
4✔
757
        env = super().get_recipe_env(arch, with_flags_in_cc)
×
758
        if not with_python:
×
759
            return env
×
760

761
        env['PYTHON_INCLUDE_ROOT'] = self.ctx.python_recipe.include_root(arch.arch)
×
762
        env['PYTHON_LINK_ROOT'] = self.ctx.python_recipe.link_root(arch.arch)
×
763
        env['EXTRA_LDLIBS'] = ' -lpython{}'.format(
×
764
            self.ctx.python_recipe.link_version)
765
        return env
×
766

767

768
class NDKRecipe(Recipe):
4✔
769
    '''A recipe class for any NDK project not included in the bootstrap.'''
770

771
    generated_libraries = []
4✔
772

773
    def should_build(self, arch):
4✔
774
        lib_dir = self.get_lib_dir(arch)
×
775

776
        for lib in self.generated_libraries:
×
777
            if not exists(join(lib_dir, lib)):
×
778
                return True
×
779

780
        return False
×
781

782
    def get_lib_dir(self, arch):
4✔
783
        return join(self.get_build_dir(arch.arch), 'obj', 'local', arch.arch)
×
784

785
    def get_jni_dir(self, arch):
4✔
786
        return join(self.get_build_dir(arch.arch), 'jni')
×
787

788
    def build_arch(self, arch, *extra_args):
4✔
789
        super().build_arch(arch)
×
790

791
        env = self.get_recipe_env(arch)
×
792
        with current_directory(self.get_build_dir(arch.arch)):
×
793
            shprint(
×
794
                sh.Command(join(self.ctx.ndk_dir, "ndk-build")),
795
                'V=1',
796
                'NDK_DEBUG=' + ("1" if self.ctx.build_as_debuggable else "0"),
797
                'APP_PLATFORM=android-' + str(self.ctx.ndk_api),
798
                'APP_ABI=' + arch.arch,
799
                *extra_args, _env=env
800
            )
801

802

803
class PythonRecipe(Recipe):
4✔
804
    site_packages_name = None
4✔
805
    '''The name of the module's folder when installed in the Python
2✔
806
    site-packages (e.g. for pyjnius it is 'jnius')'''
807

808
    call_hostpython_via_targetpython = True
4✔
809
    '''If True, tries to install the module using the hostpython binary
2✔
810
    copied to the target (normally arm) python build dir. However, this
811
    will fail if the module tries to import e.g. _io.so. Set this to False
812
    to call hostpython from its own build dir, installing the module in
813
    the right place via arguments to setup.py. However, this may not set
814
    the environment correctly and so False is not the default.'''
815

816
    install_in_hostpython = False
4✔
817
    '''If True, additionally installs the module in the hostpython build
2✔
818
    dir. This will make it available to other recipes if
819
    call_hostpython_via_targetpython is False.
820
    '''
821

822
    install_in_targetpython = True
4✔
823
    '''If True, installs the module in the targetpython installation dir.
2✔
824
    This is almost always what you want to do.'''
825

826
    setup_extra_args = []
4✔
827
    '''List of extra arguments to pass to setup.py'''
2✔
828

829
    depends = ['python3']
4✔
830
    '''
2✔
831
    .. note:: it's important to keep this depends as a class attribute outside
832
              `__init__` because sometimes we only initialize the class, so the
833
              `__init__` call won't be called and the deps would be missing
834
              (which breaks the dependency graph computation)
835

836
    .. warning:: don't forget to call `super().__init__()` in any recipe's
837
                 `__init__`, or otherwise it may not be ensured that it depends
838
                 on python2 or python3 which can break the dependency graph
839
    '''
840

841
    hostpython_prerequisites = []
4✔
842
    '''List of hostpython packages required to build a recipe'''
2✔
843

844
    def __init__(self, *args, **kwargs):
4✔
845
        super().__init__(*args, **kwargs)
4✔
846
        if 'python3' not in self.depends:
4✔
847
            # We ensure here that the recipe depends on python even it overrode
848
            # `depends`. We only do this if it doesn't already depend on any
849
            # python, since some recipes intentionally don't depend on/work
850
            # with all python variants
851
            depends = self.depends
4✔
852
            depends.append('python3')
4✔
853
            depends = list(set(depends))
4✔
854
            self.depends = depends
4✔
855

856
    def clean_build(self, arch=None):
4✔
857
        super().clean_build(arch=arch)
×
858
        name = self.folder_name
×
859
        python_install_dirs = glob.glob(join(self.ctx.python_installs_dir, '*'))
×
860
        for python_install in python_install_dirs:
×
861
            site_packages_dir = glob.glob(join(python_install, 'lib', 'python*',
×
862
                                               'site-packages'))
863
            if site_packages_dir:
×
864
                build_dir = join(site_packages_dir[0], name)
×
865
                if exists(build_dir):
×
866
                    info('Deleted {}'.format(build_dir))
×
867
                    rmdir(build_dir)
×
868

869
    @property
4✔
870
    def real_hostpython_location(self):
4✔
871
        host_name = 'host{}'.format(self.ctx.python_recipe.name)
4✔
872
        if host_name == 'hostpython3':
4!
873
            python_recipe = Recipe.get_recipe(host_name, self.ctx)
4✔
874
            return python_recipe.python_exe
4✔
875
        else:
876
            python_recipe = self.ctx.python_recipe
×
877
            return 'python{}'.format(python_recipe.version)
×
878

879
    @property
4✔
880
    def hostpython_location(self):
4✔
881
        if not self.call_hostpython_via_targetpython:
4!
882
            return self.real_hostpython_location
4✔
883
        return self.ctx.hostpython
×
884

885
    @property
4✔
886
    def folder_name(self):
4✔
887
        '''The name of the build folders containing this recipe.'''
888
        name = self.site_packages_name
×
889
        if name is None:
×
890
            name = self.name
×
891
        return name
×
892

893
    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
4✔
894
        env = super().get_recipe_env(arch, with_flags_in_cc)
4✔
895
        env['PYTHONNOUSERSITE'] = '1'
4✔
896
        # Set the LANG, this isn't usually important but is a better default
897
        # as it occasionally matters how Python e.g. reads files
898
        env['LANG'] = "en_GB.UTF-8"
4✔
899
        # Binaries made by packages installed by pip
900
        env["PATH"] = join(self.hostpython_site_dir, "bin") + ":" + env["PATH"]
4✔
901

902
        if not self.call_hostpython_via_targetpython:
4!
903
            env['CFLAGS'] += ' -I{}'.format(
4✔
904
                self.ctx.python_recipe.include_root(arch.arch)
905
            )
906
            env['LDFLAGS'] += ' -L{} -lpython{}'.format(
4✔
907
                self.ctx.python_recipe.link_root(arch.arch),
908
                self.ctx.python_recipe.link_version,
909
            )
910

911
            hppath = []
4✔
912
            hppath.append(join(dirname(self.hostpython_location), 'Lib'))
4✔
913
            hppath.append(join(hppath[0], 'site-packages'))
4✔
914
            builddir = join(dirname(self.hostpython_location), 'build')
4✔
915
            if exists(builddir):
4!
916
                hppath += [join(builddir, d) for d in listdir(builddir)
×
917
                           if isdir(join(builddir, d))]
918
            if len(hppath) > 0:
4!
919
                if 'PYTHONPATH' in env:
4!
920
                    env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']])
×
921
                else:
922
                    env['PYTHONPATH'] = ':'.join(hppath)
4✔
923
        return env
4✔
924

925
    def should_build(self, arch):
4✔
926
        name = self.folder_name
×
927
        if self.ctx.has_package(name, arch):
×
928
            info('Python package already exists in site-packages')
×
929
            return False
×
930
        info('{} apparently isn\'t already in site-packages'.format(name))
×
931
        return True
×
932

933
    def build_arch(self, arch):
4✔
934
        '''Install the Python module by calling setup.py install with
935
        the target Python dir.'''
936
        self.install_hostpython_prerequisites()
×
937
        super().build_arch(arch)
×
938
        self.install_python_package(arch)
×
939

940
    def install_python_package(self, arch, name=None, env=None, is_dir=True):
4✔
941
        '''Automate the installation of a Python package (or a cython
942
        package where the cython components are pre-built).'''
943
        # arch = self.filtered_archs[0]  # old kivy-ios way
944
        if name is None:
×
945
            name = self.name
×
946
        if env is None:
×
947
            env = self.get_recipe_env(arch)
×
948

949
        info('Installing {} into site-packages'.format(self.name))
×
950

951
        hostpython = sh.Command(self.hostpython_location)
×
952
        hpenv = env.copy()
×
953
        with current_directory(self.get_build_dir(arch.arch)):
×
954
            shprint(hostpython, 'setup.py', 'install', '-O2',
×
955
                    '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)),
956
                    '--install-lib=.',
957
                    _env=hpenv, *self.setup_extra_args)
958

959
            # If asked, also install in the hostpython build dir
960
            if self.install_in_hostpython:
×
961
                self.install_hostpython_package(arch)
×
962

963
    def get_hostrecipe_env(self, arch):
4✔
964
        env = environ.copy()
×
965
        env['PYTHONPATH'] = self.hostpython_site_dir
×
966
        return env
×
967

968
    @property
4✔
969
    def hostpython_site_dir(self):
4✔
970
        return join(dirname(self.real_hostpython_location), 'Lib', 'site-packages')
4✔
971

972
    def install_hostpython_package(self, arch):
4✔
973
        env = self.get_hostrecipe_env(arch)
×
974
        real_hostpython = sh.Command(self.real_hostpython_location)
×
975
        shprint(real_hostpython, 'setup.py', 'install', '-O2',
×
976
                '--root={}'.format(dirname(self.real_hostpython_location)),
977
                '--install-lib=Lib/site-packages',
978
                _env=env, *self.setup_extra_args)
979

980
    @property
4✔
981
    def python_major_minor_version(self):
4✔
982
        parsed_version = packaging.version.parse(self.ctx.python_recipe.version)
×
983
        return f"{parsed_version.major}.{parsed_version.minor}"
×
984

985
    def install_hostpython_prerequisites(self, packages=None, force_upgrade=True):
4✔
986
        if not packages:
×
987
            packages = self.hostpython_prerequisites
×
988

989
        if len(packages) == 0:
×
990
            return
×
991

992
        pip_options = [
×
993
            "install",
994
            *packages,
995
            "--target", self.hostpython_site_dir, "--python-version",
996
            self.ctx.python_recipe.version,
997
            # Don't use sources, instead wheels
998
            "--only-binary=:all:",
999
        ]
1000
        if force_upgrade:
×
1001
            pip_options.append("--upgrade")
×
1002
        # Use system's pip
1003
        shprint(sh.pip, *pip_options)
×
1004

1005
    def restore_hostpython_prerequisites(self, packages):
4✔
1006
        _packages = []
×
1007
        for package in packages:
×
1008
            original_version = Recipe.get_recipe(package, self.ctx).version
×
1009
            _packages.append(package + "==" + original_version)
×
1010
        self.install_hostpython_prerequisites(packages=_packages)
×
1011

1012

1013
class CompiledComponentsPythonRecipe(PythonRecipe):
4✔
1014
    pre_build_ext = False
4✔
1015

1016
    build_cmd = 'build_ext'
4✔
1017

1018
    def build_arch(self, arch):
4✔
1019
        '''Build any cython components, then install the Python module by
1020
        calling setup.py install with the target Python dir.
1021
        '''
1022
        Recipe.build_arch(self, arch)
×
1023
        self.install_hostpython_prerequisites()
×
1024
        self.build_compiled_components(arch)
×
1025
        self.install_python_package(arch)
×
1026

1027
    def build_compiled_components(self, arch):
4✔
1028
        info('Building compiled components in {}'.format(self.name))
×
1029

1030
        env = self.get_recipe_env(arch)
×
1031
        hostpython = sh.Command(self.hostpython_location)
×
1032
        with current_directory(self.get_build_dir(arch.arch)):
×
1033
            if self.install_in_hostpython:
×
1034
                shprint(hostpython, 'setup.py', 'clean', '--all', _env=env)
×
1035
            shprint(hostpython, 'setup.py', self.build_cmd, '-v',
×
1036
                    _env=env, *self.setup_extra_args)
1037
            build_dir = glob.glob('build/lib.*')[0]
×
1038
            shprint(sh.find, build_dir, '-name', '"*.o"', '-exec',
×
1039
                    env['STRIP'], '{}', ';', _env=env)
1040

1041
    def install_hostpython_package(self, arch):
4✔
1042
        env = self.get_hostrecipe_env(arch)
×
1043
        self.rebuild_compiled_components(arch, env)
×
1044
        super().install_hostpython_package(arch)
×
1045

1046
    def rebuild_compiled_components(self, arch, env):
4✔
1047
        info('Rebuilding compiled components in {}'.format(self.name))
×
1048

1049
        hostpython = sh.Command(self.real_hostpython_location)
×
1050
        shprint(hostpython, 'setup.py', 'clean', '--all', _env=env)
×
1051
        shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env,
×
1052
                *self.setup_extra_args)
1053

1054

1055
class CppCompiledComponentsPythonRecipe(CompiledComponentsPythonRecipe):
4✔
1056
    """ Extensions that require the cxx-stl """
1057
    call_hostpython_via_targetpython = False
4✔
1058
    need_stl_shared = True
4✔
1059

1060

1061
class CythonRecipe(PythonRecipe):
4✔
1062
    pre_build_ext = False
4✔
1063
    cythonize = True
4✔
1064
    cython_args = []
4✔
1065
    call_hostpython_via_targetpython = False
4✔
1066

1067
    def build_arch(self, arch):
4✔
1068
        '''Build any cython components, then install the Python module by
1069
        calling setup.py install with the target Python dir.
1070
        '''
1071
        Recipe.build_arch(self, arch)
×
1072
        self.build_cython_components(arch)
×
1073
        self.install_python_package(arch)
×
1074

1075
    def build_cython_components(self, arch):
4✔
1076
        info('Cythonizing anything necessary in {}'.format(self.name))
×
1077

1078
        env = self.get_recipe_env(arch)
×
1079

1080
        with current_directory(self.get_build_dir(arch.arch)):
×
1081
            hostpython = sh.Command(self.ctx.hostpython)
×
1082
            shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
×
1083
            debug('cwd is {}'.format(realpath(curdir)))
×
1084
            info('Trying first build of {} to get cython files: this is '
×
1085
                 'expected to fail'.format(self.name))
1086

1087
            manually_cythonise = False
×
1088
            try:
×
1089
                shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
×
1090
                        *self.setup_extra_args)
1091
            except sh.ErrorReturnCode_1:
×
1092
                print()
×
1093
                info('{} first build failed (as expected)'.format(self.name))
×
1094
                manually_cythonise = True
×
1095

1096
            if manually_cythonise:
×
1097
                self.cythonize_build(env=env)
×
1098
                shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
×
1099
                        _tail=20, _critical=True, *self.setup_extra_args)
1100
            else:
1101
                info('First build appeared to complete correctly, skipping manual'
×
1102
                     'cythonising.')
1103

1104
            if not self.ctx.with_debug_symbols:
×
1105
                self.strip_object_files(arch, env)
×
1106

1107
    def strip_object_files(self, arch, env, build_dir=None):
4✔
1108
        if build_dir is None:
×
1109
            build_dir = self.get_build_dir(arch.arch)
×
1110
        with current_directory(build_dir):
×
1111
            info('Stripping object files')
×
1112
            shprint(sh.find, '.', '-iname', '*.so', '-exec',
×
1113
                    '/usr/bin/echo', '{}', ';', _env=env)
1114
            shprint(sh.find, '.', '-iname', '*.so', '-exec',
×
1115
                    env['STRIP'].split(' ')[0], '--strip-unneeded',
1116
                    # '/usr/bin/strip', '--strip-unneeded',
1117
                    '{}', ';', _env=env)
1118

1119
    def cythonize_file(self, env, build_dir, filename):
4✔
1120
        short_filename = filename
×
1121
        if filename.startswith(build_dir):
×
1122
            short_filename = filename[len(build_dir) + 1:]
×
1123
        info(u"Cythonize {}".format(short_filename))
×
1124
        cyenv = env.copy()
×
1125
        if 'CYTHONPATH' in cyenv:
×
1126
            cyenv['PYTHONPATH'] = cyenv['CYTHONPATH']
×
1127
        elif 'PYTHONPATH' in cyenv:
×
1128
            del cyenv['PYTHONPATH']
×
1129
        if 'PYTHONNOUSERSITE' in cyenv:
×
1130
            cyenv.pop('PYTHONNOUSERSITE')
×
1131
        python_command = sh.Command("python{}".format(
×
1132
            self.ctx.python_recipe.major_minor_version_string.split(".")[0]
1133
        ))
1134
        shprint(python_command, "-c"
×
1135
                "import sys; from Cython.Compiler.Main import setuptools_main; sys.exit(setuptools_main());",
1136
                filename, *self.cython_args, _env=cyenv)
1137

1138
    def cythonize_build(self, env, build_dir="."):
4✔
1139
        if not self.cythonize:
×
1140
            info('Running cython cancelled per recipe setting')
×
1141
            return
×
1142
        info('Running cython where appropriate')
×
1143
        for root, dirnames, filenames in walk("."):
×
1144
            for filename in fnmatch.filter(filenames, "*.pyx"):
×
1145
                self.cythonize_file(env, build_dir, join(root, filename))
×
1146

1147
    def get_recipe_env(self, arch, with_flags_in_cc=True):
4✔
1148
        env = super().get_recipe_env(arch, with_flags_in_cc)
×
1149
        env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format(
×
1150
            self.ctx.get_libs_dir(arch.arch) +
1151
            ' -L{} '.format(self.ctx.libs_dir) +
1152
            ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local',
1153
                                arch.arch)))
1154

1155
        env['LDSHARED'] = env['CC'] + ' -shared'
×
1156
        # shprint(sh.whereis, env['LDSHARED'], _env=env)
1157
        env['LIBLINK'] = 'NOTNONE'
×
1158
        if self.ctx.copy_libs:
×
1159
            env['COPYLIBS'] = '1'
×
1160

1161
        # Every recipe uses its own liblink path, object files are
1162
        # collected and biglinked later
1163
        liblink_path = join(self.get_build_container_dir(arch.arch),
×
1164
                            'objects_{}'.format(self.name))
1165
        env['LIBLINK_PATH'] = liblink_path
×
1166
        ensure_dir(liblink_path)
×
1167

1168
        return env
×
1169

1170

1171
class PyProjectRecipe(PythonRecipe):
4✔
1172
    '''Recipe for projects which containes `pyproject.toml`'''
1173

1174
    # Extra args to pass to `python -m build ...`
1175
    extra_build_args = []
4✔
1176
    call_hostpython_via_targetpython = False
4✔
1177

1178
    def get_recipe_env(self, arch, **kwargs):
4✔
1179
        # Custom hostpython
1180
        self.ctx.python_recipe.python_exe = join(
4✔
1181
            self.ctx.python_recipe.get_build_dir(arch), "android-build", "python3")
1182
        env = super().get_recipe_env(arch, **kwargs)
4✔
1183
        build_dir = self.get_build_dir(arch)
4✔
1184
        ensure_dir(build_dir)
4✔
1185
        build_opts = join(build_dir, "build-opts.cfg")
4✔
1186

1187
        with open(build_opts, "w") as file:
4✔
1188
            file.write("[bdist_wheel]\nplat-name={}".format(
4✔
1189
                self.get_wheel_platform_tag(arch)
1190
            ))
1191
            file.close()
4✔
1192

1193
        env["DIST_EXTRA_CONFIG"] = build_opts
4✔
1194
        return env
4✔
1195

1196
    def get_wheel_platform_tag(self, arch):
4✔
1197
        return "android_" + {
4✔
1198
            "armeabi-v7a": "arm",
1199
            "arm64-v8a": "aarch64",
1200
            "x86_64": "x86_64",
1201
            "x86": "i686",
1202
        }[arch.arch]
1203

1204
    def install_wheel(self, arch, built_wheels):
4✔
1205
        with patch_wheel_setuptools_logging():
×
1206
            from wheel.cli.tags import tags as wheel_tags
×
1207
            from wheel.wheelfile import WheelFile
×
1208
        _wheel = built_wheels[0]
×
1209
        built_wheel_dir = dirname(_wheel)
×
1210
        # Fix wheel platform tag
1211
        wheel_tag = wheel_tags(
×
1212
            _wheel,
1213
            platform_tags=self.get_wheel_platform_tag(arch),
1214
            remove=True,
1215
        )
1216
        selected_wheel = join(built_wheel_dir, wheel_tag)
×
1217

1218
        _dev_wheel_dir = environ.get("P4A_WHEEL_DIR", False)
×
1219
        if _dev_wheel_dir:
×
1220
            ensure_dir(_dev_wheel_dir)
×
1221
            shprint(sh.cp, selected_wheel, _dev_wheel_dir)
×
1222

1223
        info(f"Installing built wheel: {wheel_tag}")
×
1224
        destination = self.ctx.get_python_install_dir(arch.arch)
×
1225
        with WheelFile(selected_wheel) as wf:
×
1226
            for zinfo in wf.filelist:
×
1227
                wf.extract(zinfo, destination)
×
1228
            wf.close()
×
1229

1230
    def build_arch(self, arch):
4✔
1231
        self.install_hostpython_prerequisites(
×
1232
            packages=["build[virtualenv]", "pip"] + self.hostpython_prerequisites
1233
        )
1234
        build_dir = self.get_build_dir(arch.arch)
×
1235
        env = self.get_recipe_env(arch, with_flags_in_cc=True)
×
1236
        # make build dir separatly
1237
        sub_build_dir = join(build_dir, "p4a_android_build")
×
1238
        ensure_dir(sub_build_dir)
×
1239
        # copy hostpython to built python to ensure correct selection of libs and includes
1240
        shprint(sh.cp, self.real_hostpython_location, self.ctx.python_recipe.python_exe)
×
1241

1242
        build_args = [
×
1243
            "-m",
1244
            "build",
1245
            "--wheel",
1246
            "--config-setting",
1247
            "builddir={}".format(sub_build_dir),
1248
        ] + self.extra_build_args
1249

1250
        built_wheels = []
×
1251
        with current_directory(build_dir):
×
1252
            shprint(
×
1253
                sh.Command(self.ctx.python_recipe.python_exe), *build_args, _env=env
1254
            )
1255
            built_wheels = [realpath(whl) for whl in glob.glob("dist/*.whl")]
×
1256
        self.install_wheel(arch, built_wheels)
×
1257

1258

1259
class MesonRecipe(PyProjectRecipe):
4✔
1260
    '''Recipe for projects which uses meson as build system'''
1261

1262
    meson_version = "1.4.0"
4✔
1263
    ninja_version = "1.11.1.1"
4✔
1264

1265
    def sanitize_flags(self, *flag_strings):
4✔
1266
        return " ".join(flag_strings).strip().split(" ")
×
1267

1268
    def get_recipe_meson_options(self, arch):
4✔
1269
        env = self.get_recipe_env(arch, with_flags_in_cc=True)
×
1270
        return {
×
1271
            "binaries": {
1272
                "c": arch.get_clang_exe(with_target=True),
1273
                "cpp": arch.get_clang_exe(with_target=True, plus_plus=True),
1274
                "ar": self.ctx.ndk.llvm_ar,
1275
                "strip": self.ctx.ndk.llvm_strip,
1276
            },
1277
            "built-in options": {
1278
                "c_args": self.sanitize_flags(env["CFLAGS"], env["CPPFLAGS"]),
1279
                "cpp_args": self.sanitize_flags(env["CXXFLAGS"], env["CPPFLAGS"]),
1280
                "c_link_args": self.sanitize_flags(env["LDFLAGS"]),
1281
                "cpp_link_args": self.sanitize_flags(env["LDFLAGS"]),
1282
            },
1283
            "properties": {
1284
                "needs_exe_wrapper": True,
1285
                "sys_root": self.ctx.ndk.sysroot
1286
            },
1287
            "host_machine": {
1288
                "cpu_family": {
1289
                    "arm64-v8a": "aarch64",
1290
                    "armeabi-v7a": "arm",
1291
                    "x86_64": "x86_64",
1292
                    "x86": "x86"
1293
                }[arch.arch],
1294
                "cpu": {
1295
                    "arm64-v8a": "aarch64",
1296
                    "armeabi-v7a": "armv7",
1297
                    "x86_64": "x86_64",
1298
                    "x86": "i686"
1299
                }[arch.arch],
1300
                "endian": "little",
1301
                "system": "android",
1302
            }
1303
        }
1304

1305
    def write_build_options(self, arch):
4✔
1306
        """Writes python dict to meson config file"""
1307
        option_data = ""
×
1308
        build_options = self.get_recipe_meson_options(arch)
×
1309
        for key in build_options.keys():
×
1310
            data_chunk = "[{}]".format(key)
×
1311
            for subkey in build_options[key].keys():
×
1312
                value = build_options[key][subkey]
×
1313
                if isinstance(value, int):
×
1314
                    value = str(value)
×
1315
                elif isinstance(value, str):
×
1316
                    value = "'{}'".format(value)
×
1317
                elif isinstance(value, bool):
×
1318
                    value = "true" if value else "false"
×
1319
                elif isinstance(value, list):
×
1320
                    value = "['" + "', '".join(value) + "']"
×
1321
                data_chunk += "\n" + subkey + " = " + value
×
1322
            option_data += data_chunk + "\n\n"
×
1323
        return option_data
×
1324

1325
    def ensure_args(self, *args):
4✔
1326
        for arg in args:
×
1327
            if arg not in self.extra_build_args:
×
1328
                self.extra_build_args.append(arg)
×
1329

1330
    def build_arch(self, arch):
4✔
1331
        cross_file = join("/tmp", "android.meson.cross")
×
1332
        info("Writing cross file at: {}".format(cross_file))
×
1333
        # write cross config file
1334
        with open(cross_file, "w") as file:
×
1335
            file.write(self.write_build_options(arch))
×
1336
            file.close()
×
1337
        # set cross file
1338
        self.ensure_args('-Csetup-args=--cross-file', '-Csetup-args={}'.format(cross_file))
×
1339
        # ensure ninja and meson
1340
        for dep in [
×
1341
            "ninja=={}".format(self.ninja_version),
1342
            "meson=={}".format(self.meson_version),
1343
        ]:
1344
            if dep not in self.hostpython_prerequisites:
×
1345
                self.hostpython_prerequisites.append(dep)
×
1346
        super().build_arch(arch)
×
1347

1348

1349
class RustCompiledComponentsRecipe(PyProjectRecipe):
4✔
1350
    # Rust toolchain codes
1351
    # https://doc.rust-lang.org/nightly/rustc/platform-support.html
1352
    RUST_ARCH_CODES = {
4✔
1353
        "arm64-v8a": "aarch64-linux-android",
1354
        "armeabi-v7a": "armv7-linux-androideabi",
1355
        "x86_64": "x86_64-linux-android",
1356
        "x86": "i686-linux-android",
1357
    }
1358

1359
    call_hostpython_via_targetpython = False
4✔
1360

1361
    def get_recipe_env(self, arch, **kwargs):
4✔
1362
        env = super().get_recipe_env(arch, **kwargs)
×
1363

1364
        # Set rust build target
1365
        build_target = self.RUST_ARCH_CODES[arch.arch]
×
1366
        cargo_linker_name = "CARGO_TARGET_{}_LINKER".format(
×
1367
            build_target.upper().replace("-", "_")
1368
        )
1369
        env["CARGO_BUILD_TARGET"] = build_target
×
1370
        env[cargo_linker_name] = join(
×
1371
            self.ctx.ndk.llvm_prebuilt_dir,
1372
            "bin",
1373
            "{}{}-clang".format(
1374
                # NDK's Clang format
1375
                build_target.replace("7", "7a")
1376
                if build_target.startswith("armv7")
1377
                else build_target,
1378
                self.ctx.ndk_api,
1379
            ),
1380
        )
1381
        realpython_dir = self.ctx.python_recipe.get_build_dir(arch.arch)
×
1382

1383
        env["RUSTFLAGS"] = "-Clink-args=-L{} -L{}".format(
×
1384
            self.ctx.get_libs_dir(arch.arch), join(realpython_dir, "android-build")
1385
        )
1386

1387
        env["PYO3_CROSS_LIB_DIR"] = realpath(glob.glob(join(
×
1388
            realpython_dir, "android-build", "build",
1389
            "lib.linux-*-{}/".format(self.python_major_minor_version),
1390
        ))[0])
1391

1392
        info_main("Ensuring rust build toolchain")
×
1393
        shprint(sh.rustup, "target", "add", build_target)
×
1394

1395
        # Add host python to PATH
1396
        env["PATH"] = ("{hostpython_dir}:{old_path}").format(
×
1397
            hostpython_dir=Recipe.get_recipe(
1398
                "hostpython3", self.ctx
1399
            ).get_path_to_python(),
1400
            old_path=env["PATH"],
1401
        )
1402
        return env
×
1403

1404
    def check_host_deps(self):
4✔
1405
        if not hasattr(sh, "rustup"):
×
1406
            error(
×
1407
                "`rustup` was not found on host system."
1408
                "Please install it using :"
1409
                "\n`curl https://sh.rustup.rs -sSf | sh`\n"
1410
            )
1411
            exit(1)
×
1412

1413
    def build_arch(self, arch):
4✔
1414
        self.check_host_deps()
×
1415
        super().build_arch(arch)
×
1416

1417

1418
class TargetPythonRecipe(Recipe):
4✔
1419
    '''Class for target python recipes. Sets ctx.python_recipe to point to
1420
    itself, so as to know later what kind of Python was built or used.'''
1421

1422
    def __init__(self, *args, **kwargs):
4✔
1423
        self._ctx = None
4✔
1424
        super().__init__(*args, **kwargs)
4✔
1425

1426
    def prebuild_arch(self, arch):
4✔
1427
        super().prebuild_arch(arch)
×
1428
        self.ctx.python_recipe = self
×
1429

1430
    def include_root(self, arch):
4✔
1431
        '''The root directory from which to include headers.'''
1432
        raise NotImplementedError('Not implemented in TargetPythonRecipe')
1433

1434
    def link_root(self):
4✔
1435
        raise NotImplementedError('Not implemented in TargetPythonRecipe')
1436

1437
    @property
4✔
1438
    def major_minor_version_string(self):
4✔
1439
        parsed_version = packaging.version.parse(self.version)
4✔
1440
        return f"{parsed_version.major}.{parsed_version.minor}"
4✔
1441

1442
    def create_python_bundle(self, dirn, arch):
4✔
1443
        """
1444
        Create a packaged python bundle in the target directory, by
1445
        copying all the modules and standard library to the right
1446
        place.
1447
        """
1448
        raise NotImplementedError('{} does not implement create_python_bundle'.format(self))
1449

1450
    def reduce_object_file_names(self, dirn):
4✔
1451
        """Recursively renames all files named XXX.cpython-...-linux-gnu.so"
1452
        to "XXX.so", i.e. removing the erroneous architecture name
1453
        coming from the local system.
1454
        """
1455
        py_so_files = shprint(sh.find, dirn, '-iname', '*.so')
4✔
1456
        filens = py_so_files.stdout.decode('utf-8').split('\n')[:-1]
4✔
1457
        for filen in filens:
4!
1458
            file_dirname, file_basename = split(filen)
×
1459
            parts = file_basename.split('.')
×
1460
            if len(parts) <= 2:
×
1461
                continue
×
1462
            # PySide6 libraries end with .abi3.so
1463
            if parts[1] == "abi3":
×
1464
                continue
×
1465
            move(filen, join(file_dirname, parts[0] + '.so'))
×
1466

1467

1468
def algsum(alg, filen):
4✔
1469
    '''Calculate the digest of a file.
1470
    '''
1471
    with open(filen, 'rb') as fileh:
×
1472
        digest = getattr(hashlib, alg)(fileh.read())
×
1473

1474
    return digest.hexdigest()
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc