• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

kivy / python-for-android / 10342648059

11 Aug 2024 08:44PM UTC coverage: 59.07% (-0.07%) from 59.14%
10342648059

push

github

AndreMiras
:arrow_up: Bump to sh 2

1048 of 2361 branches covered (44.39%)

Branch coverage included in aggregate %.

1 of 5 new or added lines in 2 files covered. (20.0%)

7 existing lines in 3 files now uncovered.

4862 of 7644 relevant lines covered (63.61%)

2.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

45.33
/pythonforandroid/recipe.py
1
from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split
4✔
2
import glob
4✔
3

4
import hashlib
4✔
5
from re import match
4✔
6

7
import sh
4✔
8
import shutil
4✔
9
import fnmatch
4✔
10
import zipfile
4✔
11
import urllib.request
4✔
12
from urllib.request import urlretrieve
4✔
13
from os import listdir, unlink, environ, curdir, walk
4✔
14
from sys import stdout
4✔
15
from wheel.wheelfile import WheelFile
4✔
16
from wheel.cli.tags import tags as wheel_tags
4✔
17
import time
4✔
18
try:
4✔
19
    from urlparse import urlparse
4✔
20
except ImportError:
4✔
21
    from urllib.parse import urlparse
4✔
22

23
import packaging.version
4✔
24

25
from pythonforandroid.logger import (
4✔
26
    logger, info, warning, debug, shprint, info_main, error)
27
from pythonforandroid.util import (
4✔
28
    current_directory, ensure_dir, BuildInterruptingException, rmdir, move,
29
    touch)
30
from pythonforandroid.util import load_source as import_recipe
4✔
31

32

33
url_opener = urllib.request.build_opener()
4✔
34
url_orig_headers = url_opener.addheaders
4✔
35
urllib.request.install_opener(url_opener)
4✔
36

37

38
class RecipeMeta(type):
4✔
39
    def __new__(cls, name, bases, dct):
4✔
40
        if name != 'Recipe':
4✔
41
            if 'url' in dct:
4✔
42
                dct['_url'] = dct.pop('url')
4✔
43
            if 'version' in dct:
4✔
44
                dct['_version'] = dct.pop('version')
4✔
45

46
        return super().__new__(cls, name, bases, dct)
4✔
47

48

49
class Recipe(metaclass=RecipeMeta):
4✔
50
    _url = None
4✔
51
    '''The address from which the recipe may be downloaded. This is not
2✔
52
    essential, it may be omitted if the source is available some other
53
    way, such as via the :class:`IncludedFilesBehaviour` mixin.
54

55
    If the url includes the version, you may (and probably should)
56
    replace this with ``{version}``, which will automatically be
57
    replaced by the :attr:`version` string during download.
58

59
    .. note:: Methods marked (internal) are used internally and you
60
              probably don't need to call them, but they are available
61
              if you want.
62
    '''
63

64
    _version = None
4✔
65
    '''A string giving the version of the software the recipe describes,
2✔
66
    e.g. ``2.0.3`` or ``master``.'''
67

68
    md5sum = None
4✔
69
    '''The md5sum of the source from the :attr:`url`. Non-essential, but
2✔
70
    you should try to include this, it is used to check that the download
71
    finished correctly.
72
    '''
73

74
    sha512sum = None
4✔
75
    '''The sha512sum of the source from the :attr:`url`. Non-essential, but
2✔
76
    you should try to include this, it is used to check that the download
77
    finished correctly.
78
    '''
79

80
    blake2bsum = None
4✔
81
    '''The blake2bsum of the source from the :attr:`url`. Non-essential, but
2✔
82
    you should try to include this, it is used to check that the download
83
    finished correctly.
84
    '''
85

86
    depends = []
4✔
87
    '''A list containing the names of any recipes that this recipe depends on.
2✔
88
    '''
89

90
    conflicts = []
4✔
91
    '''A list containing the names of any recipes that are known to be
2✔
92
    incompatible with this one.'''
93

94
    opt_depends = []
4✔
95
    '''A list of optional dependencies, that must be built before this
2✔
96
    recipe if they are built at all, but whose presence is not essential.'''
97

98
    patches = []
4✔
99
    '''A list of patches to apply to the source. Values can be either a string
2✔
100
    referring to the patch file relative to the recipe dir, or a tuple of the
101
    string patch file and a callable, which will receive the kwargs `arch` and
102
    `recipe`, which should return True if the patch should be applied.'''
103

104
    python_depends = []
4✔
105
    '''A list of pure-Python packages that this package requires. These
2✔
106
    packages will NOT be available at build time, but will be added to the
107
    list of pure-Python packages to install via pip. If you need these packages
108
    at build time, you must create a recipe.'''
109

110
    archs = ['armeabi']  # Not currently implemented properly
4✔
111

112
    built_libraries = {}
4✔
113
    """Each recipe that builds a system library (e.g.:libffi, openssl, etc...)
2✔
114
    should contain a dict holding the relevant information of the library. The
115
    keys should be the generated libraries and the values the relative path of
116
    the library inside his build folder. This dict will be used to perform
117
    different operations:
118
        - copy the library into the right location, depending on if it's shared
119
          or static)
120
        - check if we have to rebuild the library
121

122
    Here an example of how it would look like for `libffi` recipe:
123

124
        - `built_libraries = {'libffi.so': '.libs'}`
125

126
    .. note:: in case that the built library resides in recipe's build
127
              directory, you can set the following values for the relative
128
              path: `'.', None or ''`
129
    """
130

131
    need_stl_shared = False
4✔
132
    '''Some libraries or python packages may need the c++_shared in APK.
2✔
133
    We can automatically do this for any recipe if we set this property to
134
    `True`'''
135

136
    stl_lib_name = 'c++_shared'
4✔
137
    '''
2✔
138
    The default STL shared lib to use: `c++_shared`.
139

140
    .. note:: Android NDK version > 17 only supports 'c++_shared', because
141
        starting from NDK r18 the `gnustl_shared` lib has been deprecated.
142
    '''
143

144
    def get_stl_library(self, arch):
4✔
145
        return join(
4✔
146
            arch.ndk_lib_dir,
147
            'lib{name}.so'.format(name=self.stl_lib_name),
148
        )
149

150
    def install_stl_lib(self, arch):
4✔
151
        if not self.ctx.has_lib(
4!
152
            arch.arch, 'lib{name}.so'.format(name=self.stl_lib_name)
153
        ):
154
            self.install_libs(arch, self.get_stl_library(arch))
4✔
155

156
    @property
4✔
157
    def version(self):
4✔
158
        key = 'VERSION_' + self.name
4✔
159
        return environ.get(key, self._version)
4✔
160

161
    @property
4✔
162
    def url(self):
4✔
163
        key = 'URL_' + self.name
4✔
164
        return environ.get(key, self._url)
4✔
165

166
    @property
4✔
167
    def versioned_url(self):
4✔
168
        '''A property returning the url of the recipe with ``{version}``
169
        replaced by the :attr:`url`. If accessing the url, you should use this
170
        property, *not* access the url directly.'''
171
        if self.url is None:
4!
172
            return None
×
173
        return self.url.format(version=self.version)
4✔
174

175
    def download_file(self, url, target, cwd=None):
4✔
176
        """
177
        (internal) Download an ``url`` to a ``target``.
178
        """
179
        if not url:
4!
180
            return
×
181

182
        info('Downloading {} from {}'.format(self.name, url))
4✔
183

184
        if cwd:
4!
185
            target = join(cwd, target)
×
186

187
        parsed_url = urlparse(url)
4✔
188
        if parsed_url.scheme in ('http', 'https'):
4!
189
            def report_hook(index, blksize, size):
4✔
190
                if size <= 0:
×
191
                    progression = '{0} bytes'.format(index * blksize)
×
192
                else:
193
                    progression = '{0:.2f}%'.format(
×
194
                        index * blksize * 100. / float(size))
195
                if "CI" not in environ:
×
196
                    stdout.write('- Download {}\r'.format(progression))
×
197
                    stdout.flush()
×
198

199
            if exists(target):
4!
200
                unlink(target)
×
201

202
            # Download item with multiple attempts (for bad connections):
203
            attempts = 0
4✔
204
            seconds = 1
4✔
205
            while True:
2✔
206
                try:
4✔
207
                    # jqueryui.com returns a 403 w/ the default user agent
208
                    # Mozilla/5.0 doesnt handle redirection for liblzma
209
                    url_opener.addheaders = [('User-agent', 'Wget/1.0')]
4✔
210
                    urlretrieve(url, target, report_hook)
4✔
211
                except OSError as e:
4✔
212
                    attempts += 1
4✔
213
                    if attempts >= 5:
4✔
214
                        raise
4✔
215
                    stdout.write('Download failed: {}; retrying in {} second(s)...'.format(e, seconds))
4✔
216
                    time.sleep(seconds)
4✔
217
                    seconds *= 2
4✔
218
                    continue
4✔
219
                finally:
220
                    url_opener.addheaders = url_orig_headers
4✔
221
                break
3✔
222
            return target
4✔
223
        elif parsed_url.scheme in ('git', 'git+file', 'git+ssh', 'git+http', 'git+https'):
×
224
            if not isdir(target):
×
225
                if url.startswith('git+'):
×
226
                    url = url[4:]
×
227
                # if 'version' is specified, do a shallow clone
228
                if self.version:
×
229
                    ensure_dir(target)
×
230
                    with current_directory(target):
×
231
                        shprint(sh.git, 'init')
×
232
                        shprint(sh.git, 'remote', 'add', 'origin', url)
×
233
                else:
234
                    shprint(sh.git, 'clone', '--recursive', url, target)
×
235
            with current_directory(target):
×
236
                if self.version:
×
237
                    shprint(sh.git, 'fetch', '--tags', '--depth', '1')
×
238
                    shprint(sh.git, 'checkout', self.version)
×
239
                branch = sh.git('branch', '--show-current')
×
240
                if branch:
×
241
                    shprint(sh.git, 'pull')
×
242
                    shprint(sh.git, 'pull', '--recurse-submodules')
×
243
                shprint(sh.git, 'submodule', 'update', '--recursive', '--init', '--depth', '1')
×
244
            return target
×
245

246
    def apply_patch(self, filename, arch, build_dir=None):
4✔
247
        """
248
        Apply a patch from the current recipe directory into the current
249
        build directory.
250

251
        .. versionchanged:: 0.6.0
252
            Add ability to apply patch from any dir via kwarg `build_dir`'''
253
        """
254
        info("Applying patch {}".format(filename))
4✔
255
        build_dir = build_dir if build_dir else self.get_build_dir(arch)
4✔
256
        filename = join(self.get_recipe_dir(), filename)
4✔
257
        shprint(sh.patch, "-t", "-d", build_dir, "-p1",
4✔
258
                "-i", filename, _tail=10)
259

260
    def copy_file(self, filename, dest):
4✔
261
        info("Copy {} to {}".format(filename, dest))
×
262
        filename = join(self.get_recipe_dir(), filename)
×
263
        dest = join(self.build_dir, dest)
×
264
        shutil.copy(filename, dest)
×
265

266
    def append_file(self, filename, dest):
4✔
267
        info("Append {} to {}".format(filename, dest))
×
268
        filename = join(self.get_recipe_dir(), filename)
×
269
        dest = join(self.build_dir, dest)
×
270
        with open(filename, "rb") as fd:
×
271
            data = fd.read()
×
272
        with open(dest, "ab") as fd:
×
273
            fd.write(data)
×
274

275
    @property
4✔
276
    def name(self):
4✔
277
        '''The name of the recipe, the same as the folder containing it.'''
278
        modname = self.__class__.__module__
4✔
279
        return modname.split(".", 2)[-1]
4✔
280

281
    @property
4✔
282
    def filtered_archs(self):
4✔
283
        '''Return archs of self.ctx that are valid build archs
284
        for the Recipe.'''
285
        result = []
×
286
        for arch in self.ctx.archs:
×
287
            if not self.archs or (arch.arch in self.archs):
×
288
                result.append(arch)
×
289
        return result
×
290

291
    def check_recipe_choices(self):
4✔
292
        '''Checks what recipes are being built to see which of the alternative
293
        and optional dependencies are being used,
294
        and returns a list of these.'''
295
        recipes = []
4✔
296
        built_recipes = self.ctx.recipe_build_order
4✔
297
        for recipe in self.depends:
4✔
298
            if isinstance(recipe, (tuple, list)):
4!
299
                for alternative in recipe:
×
300
                    if alternative in built_recipes:
×
301
                        recipes.append(alternative)
×
302
                        break
×
303
        for recipe in self.opt_depends:
4✔
304
            if recipe in built_recipes:
4!
305
                recipes.append(recipe)
×
306
        return sorted(recipes)
4✔
307

308
    def get_opt_depends_in_list(self, recipes):
4✔
309
        '''Given a list of recipe names, returns those that are also in
310
        self.opt_depends.
311
        '''
312
        return [recipe for recipe in recipes if recipe in self.opt_depends]
4✔
313

314
    def get_build_container_dir(self, arch):
4✔
315
        '''Given the arch name, returns the directory where it will be
316
        built.
317

318
        This returns a different directory depending on what
319
        alternative or optional dependencies are being built.
320
        '''
321
        dir_name = self.get_dir_name()
4✔
322
        return join(self.ctx.build_dir, 'other_builds',
4✔
323
                    dir_name, '{}__ndk_target_{}'.format(arch, self.ctx.ndk_api))
324

325
    def get_dir_name(self):
4✔
326
        choices = self.check_recipe_choices()
4✔
327
        dir_name = '-'.join([self.name] + choices)
4✔
328
        return dir_name
4✔
329

330
    def get_build_dir(self, arch):
4✔
331
        '''Given the arch name, returns the directory where the
332
        downloaded/copied package will be built.'''
333

334
        return join(self.get_build_container_dir(arch), self.name)
4✔
335

336
    def get_recipe_dir(self):
4✔
337
        """
338
        Returns the local recipe directory or defaults to the core recipe
339
        directory.
340
        """
341
        if self.ctx.local_recipes is not None:
4!
342
            local_recipe_dir = join(self.ctx.local_recipes, self.name)
×
343
            if exists(local_recipe_dir):
×
344
                return local_recipe_dir
×
345
        return join(self.ctx.root_dir, 'recipes', self.name)
4✔
346

347
    # Public Recipe API to be subclassed if needed
348

349
    def download_if_necessary(self):
4✔
350
        info_main('Downloading {}'.format(self.name))
4✔
351
        user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
4✔
352
        if user_dir is not None:
4✔
353
            info('P4A_{}_DIR is set, skipping download for {}'.format(
4✔
354
                self.name, self.name))
355
            return
4✔
356
        self.download()
4✔
357

358
    def download(self):
4✔
359
        if self.url is None:
4✔
360
            info('Skipping {} download as no URL is set'.format(self.name))
4✔
361
            return
4✔
362

363
        url = self.versioned_url
4✔
364
        expected_digests = {}
4✔
365
        for alg in set(hashlib.algorithms_guaranteed) | set(('md5', 'sha512', 'blake2b')):
4✔
366
            expected_digest = getattr(self, alg + 'sum') if hasattr(self, alg + 'sum') else None
4✔
367
            ma = match(u'^(.+)#' + alg + u'=([0-9a-f]{32,})$', url)
4✔
368
            if ma:                # fragmented URL?
4!
369
                if expected_digest:
×
370
                    raise ValueError(
×
371
                        ('Received {}sum from both the {} recipe '
372
                         'and its url').format(alg, self.name))
373
                url = ma.group(1)
×
374
                expected_digest = ma.group(2)
×
375
            if expected_digest:
4!
376
                expected_digests[alg] = expected_digest
×
377

378
        ensure_dir(join(self.ctx.packages_path, self.name))
4✔
379

380
        with current_directory(join(self.ctx.packages_path, self.name)):
4✔
381
            filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8')
4✔
382

383
            do_download = True
4✔
384
            marker_filename = '.mark-{}'.format(filename)
4✔
385
            if exists(filename) and isfile(filename):
4!
386
                if not exists(marker_filename):
×
387
                    shprint(sh.rm, filename)
×
388
                else:
389
                    for alg, expected_digest in expected_digests.items():
×
390
                        current_digest = algsum(alg, filename)
×
391
                        if current_digest != expected_digest:
×
392
                            debug('* Generated {}sum: {}'.format(alg,
×
393
                                                                 current_digest))
394
                            debug('* Expected {}sum: {}'.format(alg,
×
395
                                                                expected_digest))
396
                            raise ValueError(
×
397
                                ('Generated {0}sum does not match expected {0}sum '
398
                                 'for {1} recipe').format(alg, self.name))
399
                    do_download = False
×
400

401
            # If we got this far, we will download
402
            if do_download:
4!
403
                debug('Downloading {} from {}'.format(self.name, url))
4✔
404

405
                shprint(sh.rm, '-f', marker_filename)
4✔
406
                self.download_file(self.versioned_url, filename)
4✔
407
                touch(marker_filename)
4✔
408

409
                if exists(filename) and isfile(filename):
4!
410
                    for alg, expected_digest in expected_digests.items():
×
411
                        current_digest = algsum(alg, filename)
×
412
                        if current_digest != expected_digest:
×
413
                            debug('* Generated {}sum: {}'.format(alg,
×
414
                                                                 current_digest))
415
                            debug('* Expected {}sum: {}'.format(alg,
×
416
                                                                expected_digest))
417
                            raise ValueError(
×
418
                                ('Generated {0}sum does not match expected {0}sum '
419
                                 'for {1} recipe').format(alg, self.name))
420
            else:
421
                info('{} download already cached, skipping'.format(self.name))
×
422

423
    def unpack(self, arch):
4✔
424
        info_main('Unpacking {} for {}'.format(self.name, arch))
×
425

426
        build_dir = self.get_build_container_dir(arch)
×
427

428
        user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
×
429
        if user_dir is not None:
×
430
            info('P4A_{}_DIR exists, symlinking instead'.format(
×
431
                self.name.lower()))
432
            if exists(self.get_build_dir(arch)):
×
433
                return
×
434
            rmdir(build_dir)
×
435
            ensure_dir(build_dir)
×
436
            shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
×
437
            return
×
438

439
        if self.url is None:
×
440
            info('Skipping {} unpack as no URL is set'.format(self.name))
×
441
            return
×
442

443
        # TODO: make sure we're passing here, maybe an assert False to be sure
UNCOV
444
        filename = shprint(
×
445
            sh.basename, self.versioned_url)[:-1]
446
        ma = match(u'^(.+)#[a-z0-9_]{3,}=([0-9a-f]{32,})$', filename)
×
447
        if ma:                  # fragmented URL?
×
448
            filename = ma.group(1)
×
449

450
        with current_directory(build_dir):
×
451
            directory_name = self.get_build_dir(arch)
×
452

453
            if not exists(directory_name) or not isdir(directory_name):
×
454
                extraction_filename = join(
×
455
                    self.ctx.packages_path, self.name, filename)
456
                if isfile(extraction_filename):
×
457
                    if extraction_filename.endswith(('.zip', '.whl')):
×
458
                        try:
×
459
                            sh.unzip(extraction_filename)
×
460
                        except (sh.ErrorReturnCode_1, sh.ErrorReturnCode_2):
×
461
                            # return code 1 means unzipping had
462
                            # warnings but did complete,
463
                            # apparently happens sometimes with
464
                            # github zips
465
                            pass
×
466
                        fileh = zipfile.ZipFile(extraction_filename, 'r')
×
467
                        root_directory = fileh.filelist[0].filename.split('/')[0]
×
468
                        if root_directory != basename(directory_name):
×
469
                            move(root_directory, directory_name)
×
470
                    elif extraction_filename.endswith(
×
471
                            ('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz', '.txz')):
472
                        sh.tar('xf', extraction_filename)
×
NEW
473
                        root_directory = sh.tar('tf', extraction_filename).split('\n')[0].split('/')[0]
×
UNCOV
474
                        if root_directory != basename(directory_name):
×
475
                            move(root_directory, directory_name)
×
476
                    else:
477
                        raise Exception(
×
478
                            'Could not extract {} download, it must be .zip, '
479
                            '.tar.gz or .tar.bz2 or .tar.xz'.format(extraction_filename))
480
                elif isdir(extraction_filename):
×
481
                    ensure_dir(directory_name)
×
482
                    for entry in listdir(extraction_filename):
×
483
                        # Previously we filtered out the .git folder, but during the build process for some recipes
484
                        # (e.g. when version is parsed by `setuptools_scm`) that may be needed.
485
                        shprint(sh.cp, '-Rv',
×
486
                                join(extraction_filename, entry),
487
                                directory_name)
488
                else:
489
                    raise Exception(
×
490
                        'Given path is neither a file nor a directory: {}'
491
                        .format(extraction_filename))
492

493
            else:
494
                info('{} is already unpacked, skipping'.format(self.name))
×
495

496
    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
4✔
497
        """Return the env specialized for the recipe
498
        """
499
        if arch is None:
4!
500
            arch = self.filtered_archs[0]
×
501
        env = arch.get_env(with_flags_in_cc=with_flags_in_cc)
4✔
502
        return env
4✔
503

504
    def prebuild_arch(self, arch):
4✔
505
        '''Run any pre-build tasks for the Recipe. By default, this checks if
506
        any prebuild_archname methods exist for the archname of the current
507
        architecture, and runs them if so.'''
508
        prebuild = "prebuild_{}".format(arch.arch.replace('-', '_'))
4✔
509
        if hasattr(self, prebuild):
4!
510
            getattr(self, prebuild)()
×
511
        else:
512
            info('{} has no {}, skipping'.format(self.name, prebuild))
4✔
513

514
    def is_patched(self, arch):
4✔
515
        build_dir = self.get_build_dir(arch.arch)
4✔
516
        return exists(join(build_dir, '.patched'))
4✔
517

518
    def apply_patches(self, arch, build_dir=None):
4✔
519
        '''Apply any patches for the Recipe.
520

521
        .. versionchanged:: 0.6.0
522
            Add ability to apply patches from any dir via kwarg `build_dir`'''
523
        if self.patches:
×
524
            info_main('Applying patches for {}[{}]'
×
525
                      .format(self.name, arch.arch))
526

527
            if self.is_patched(arch):
×
528
                info_main('{} already patched, skipping'.format(self.name))
×
529
                return
×
530

531
            build_dir = build_dir if build_dir else self.get_build_dir(arch.arch)
×
532
            for patch in self.patches:
×
533
                if isinstance(patch, (tuple, list)):
×
534
                    patch, patch_check = patch
×
535
                    if not patch_check(arch=arch, recipe=self):
×
536
                        continue
×
537

538
                self.apply_patch(
×
539
                        patch.format(version=self.version, arch=arch.arch),
540
                        arch.arch, build_dir=build_dir)
541

542
            touch(join(build_dir, '.patched'))
×
543

544
    def should_build(self, arch):
4✔
545
        '''Should perform any necessary test and return True only if it needs
546
        building again. Per default we implement a library test, in case that
547
        we detect so.
548

549
        '''
550
        if self.built_libraries:
4!
551
            return not all(
4✔
552
                exists(lib) for lib in self.get_libraries(arch.arch)
553
            )
554
        return True
×
555

556
    def build_arch(self, arch):
4✔
557
        '''Run any build tasks for the Recipe. By default, this checks if
558
        any build_archname methods exist for the archname of the current
559
        architecture, and runs them if so.'''
560
        build = "build_{}".format(arch.arch)
×
561
        if hasattr(self, build):
×
562
            getattr(self, build)()
×
563

564
    def install_libraries(self, arch):
4✔
565
        '''This method is always called after `build_arch`. In case that we
566
        detect a library recipe, defined by the class attribute
567
        `built_libraries`, we will copy all defined libraries into the
568
         right location.
569
        '''
570
        if not self.built_libraries:
4!
571
            return
×
572
        shared_libs = [
4✔
573
            lib for lib in self.get_libraries(arch) if lib.endswith(".so")
574
        ]
575
        self.install_libs(arch, *shared_libs)
4✔
576

577
    def postbuild_arch(self, arch):
4✔
578
        '''Run any post-build tasks for the Recipe. By default, this checks if
579
        any postbuild_archname methods exist for the archname of the
580
        current architecture, and runs them if so.
581
        '''
582
        postbuild = "postbuild_{}".format(arch.arch)
4✔
583
        if hasattr(self, postbuild):
4!
584
            getattr(self, postbuild)()
×
585

586
        if self.need_stl_shared:
4!
587
            self.install_stl_lib(arch)
4✔
588

589
    def prepare_build_dir(self, arch):
4✔
590
        '''Copies the recipe data into a build dir for the given arch. By
591
        default, this unpacks a downloaded recipe. You should override
592
        it (or use a Recipe subclass with different behaviour) if you
593
        want to do something else.
594
        '''
595
        self.unpack(arch)
×
596

597
    def clean_build(self, arch=None):
4✔
598
        '''Deletes all the build information of the recipe.
599

600
        If arch is not None, only this arch dir is deleted. Otherwise
601
        (the default) all builds for all archs are deleted.
602

603
        By default, this just deletes the main build dir. If the
604
        recipe has e.g. object files biglinked, or .so files stored
605
        elsewhere, you should override this method.
606

607
        This method is intended for testing purposes, it may have
608
        strange results. Rebuild everything if this seems to happen.
609

610
        '''
611
        if arch is None:
×
612
            base_dir = join(self.ctx.build_dir, 'other_builds', self.name)
×
613
        else:
614
            base_dir = self.get_build_container_dir(arch)
×
615
        dirs = glob.glob(base_dir + '-*')
×
616
        if exists(base_dir):
×
617
            dirs.append(base_dir)
×
618
        if not dirs:
×
619
            warning('Attempted to clean build for {} but found no existing '
×
620
                    'build dirs'.format(self.name))
621

622
        for directory in dirs:
×
623
            rmdir(directory)
×
624

625
        # Delete any Python distributions to ensure the recipe build
626
        # doesn't persist in site-packages
627
        rmdir(self.ctx.python_installs_dir)
×
628

629
    def install_libs(self, arch, *libs):
4✔
630
        libs_dir = self.ctx.get_libs_dir(arch.arch)
4✔
631
        if not libs:
4!
632
            warning('install_libs called with no libraries to install!')
×
633
            return
×
634
        args = libs + (libs_dir,)
4✔
635
        shprint(sh.cp, *args)
4✔
636

637
    def has_libs(self, arch, *libs):
4✔
638
        return all(map(lambda lib: self.ctx.has_lib(arch.arch, lib), libs))
×
639

640
    def get_libraries(self, arch_name, in_context=False):
4✔
641
        """Return the full path of the library depending on the architecture.
642
        Per default, the build library path it will be returned, unless
643
        `get_libraries` has been called with kwarg `in_context` set to
644
        True.
645

646
        .. note:: this method should be used for library recipes only
647
        """
648
        recipe_libs = set()
4✔
649
        if not self.built_libraries:
4!
650
            return recipe_libs
×
651
        for lib, rel_path in self.built_libraries.items():
4✔
652
            if not in_context:
4!
653
                abs_path = join(self.get_build_dir(arch_name), rel_path, lib)
4✔
654
                if rel_path in {".", "", None}:
4✔
655
                    abs_path = join(self.get_build_dir(arch_name), lib)
4✔
656
            else:
657
                abs_path = join(self.ctx.get_libs_dir(arch_name), lib)
×
658
            recipe_libs.add(abs_path)
4✔
659
        return recipe_libs
4✔
660

661
    @classmethod
4✔
662
    def recipe_dirs(cls, ctx):
4✔
663
        recipe_dirs = []
4✔
664
        if ctx.local_recipes is not None:
4✔
665
            recipe_dirs.append(realpath(ctx.local_recipes))
4✔
666
        if ctx.storage_dir:
4✔
667
            recipe_dirs.append(join(ctx.storage_dir, 'recipes'))
4✔
668
        recipe_dirs.append(join(ctx.root_dir, "recipes"))
4✔
669
        return recipe_dirs
4✔
670

671
    @classmethod
4✔
672
    def list_recipes(cls, ctx):
4✔
673
        forbidden_dirs = ('__pycache__', )
4✔
674
        for recipes_dir in cls.recipe_dirs(ctx):
4✔
675
            if recipes_dir and exists(recipes_dir):
4✔
676
                for name in listdir(recipes_dir):
4✔
677
                    if name in forbidden_dirs:
4✔
678
                        continue
4✔
679
                    fn = join(recipes_dir, name)
4✔
680
                    if isdir(fn):
4✔
681
                        yield name
4✔
682

683
    @classmethod
4✔
684
    def get_recipe(cls, name, ctx):
4✔
685
        '''Returns the Recipe with the given name, if it exists.'''
686
        name = name.lower()
4✔
687
        if not hasattr(cls, "recipes"):
4✔
688
            cls.recipes = {}
4✔
689
        if name in cls.recipes:
4✔
690
            return cls.recipes[name]
4✔
691

692
        recipe_file = None
4✔
693
        for recipes_dir in cls.recipe_dirs(ctx):
4✔
694
            if not exists(recipes_dir):
4✔
695
                continue
4✔
696
            # Find matching folder (may differ in case):
697
            for subfolder in listdir(recipes_dir):
4✔
698
                if subfolder.lower() == name:
4✔
699
                    recipe_file = join(recipes_dir, subfolder, '__init__.py')
4✔
700
                    if exists(recipe_file):
4!
701
                        name = subfolder  # adapt to actual spelling
4✔
702
                        break
4✔
703
                    recipe_file = None
×
704
            if recipe_file is not None:
4✔
705
                break
4✔
706

707
        else:
708
            raise ValueError('Recipe does not exist: {}'.format(name))
4✔
709

710
        mod = import_recipe('pythonforandroid.recipes.{}'.format(name), recipe_file)
4✔
711
        if len(logger.handlers) > 1:
4!
712
            logger.removeHandler(logger.handlers[1])
×
713
        recipe = mod.recipe
4✔
714
        recipe.ctx = ctx
4✔
715
        cls.recipes[name.lower()] = recipe
4✔
716
        return recipe
4✔
717

718

719
class IncludedFilesBehaviour(object):
4✔
720
    '''Recipe mixin class that will automatically unpack files included in
721
    the recipe directory.'''
722
    src_filename = None
4✔
723

724
    def prepare_build_dir(self, arch):
4✔
725
        if self.src_filename is None:
×
726
            raise BuildInterruptingException(
×
727
                'IncludedFilesBehaviour failed: no src_filename specified')
728
        rmdir(self.get_build_dir(arch))
×
729
        shprint(sh.cp, '-a', join(self.get_recipe_dir(), self.src_filename),
×
730
                self.get_build_dir(arch))
731

732

733
class BootstrapNDKRecipe(Recipe):
4✔
734
    '''A recipe class for recipes built in an Android project jni dir with
735
    an Android.mk. These are not cached separatly, but built in the
736
    bootstrap's own building directory.
737

738
    To build an NDK project which is not part of the bootstrap, see
739
    :class:`~pythonforandroid.recipe.NDKRecipe`.
740

741
    To link with python, call the method :meth:`get_recipe_env`
742
    with the kwarg *with_python=True*.
743
    '''
744

745
    dir_name = None  # The name of the recipe build folder in the jni dir
4✔
746

747
    def get_build_container_dir(self, arch):
4✔
748
        return self.get_jni_dir()
×
749

750
    def get_build_dir(self, arch):
4✔
751
        if self.dir_name is None:
×
752
            raise ValueError('{} recipe doesn\'t define a dir_name, but '
×
753
                             'this is necessary'.format(self.name))
754
        return join(self.get_build_container_dir(arch), self.dir_name)
×
755

756
    def get_jni_dir(self):
4✔
757
        return join(self.ctx.bootstrap.build_dir, 'jni')
4✔
758

759
    def get_recipe_env(self, arch=None, with_flags_in_cc=True, with_python=False):
4✔
760
        env = super().get_recipe_env(arch, with_flags_in_cc)
×
761
        if not with_python:
×
762
            return env
×
763

764
        env['PYTHON_INCLUDE_ROOT'] = self.ctx.python_recipe.include_root(arch.arch)
×
765
        env['PYTHON_LINK_ROOT'] = self.ctx.python_recipe.link_root(arch.arch)
×
766
        env['EXTRA_LDLIBS'] = ' -lpython{}'.format(
×
767
            self.ctx.python_recipe.link_version)
768
        return env
×
769

770

771
class NDKRecipe(Recipe):
4✔
772
    '''A recipe class for any NDK project not included in the bootstrap.'''
773

774
    generated_libraries = []
4✔
775

776
    def should_build(self, arch):
4✔
777
        lib_dir = self.get_lib_dir(arch)
×
778

779
        for lib in self.generated_libraries:
×
780
            if not exists(join(lib_dir, lib)):
×
781
                return True
×
782

783
        return False
×
784

785
    def get_lib_dir(self, arch):
4✔
786
        return join(self.get_build_dir(arch.arch), 'obj', 'local', arch.arch)
×
787

788
    def get_jni_dir(self, arch):
4✔
789
        return join(self.get_build_dir(arch.arch), 'jni')
×
790

791
    def build_arch(self, arch, *extra_args):
4✔
792
        super().build_arch(arch)
×
793

794
        env = self.get_recipe_env(arch)
×
795
        with current_directory(self.get_build_dir(arch.arch)):
×
796
            shprint(
×
797
                sh.Command(join(self.ctx.ndk_dir, "ndk-build")),
798
                'V=1',
799
                'NDK_DEBUG=' + ("1" if self.ctx.build_as_debuggable else "0"),
800
                'APP_PLATFORM=android-' + str(self.ctx.ndk_api),
801
                'APP_ABI=' + arch.arch,
802
                *extra_args, _env=env
803
            )
804

805

806
class PythonRecipe(Recipe):
4✔
807
    site_packages_name = None
4✔
808
    '''The name of the module's folder when installed in the Python
2✔
809
    site-packages (e.g. for pyjnius it is 'jnius')'''
810

811
    call_hostpython_via_targetpython = True
4✔
812
    '''If True, tries to install the module using the hostpython binary
2✔
813
    copied to the target (normally arm) python build dir. However, this
814
    will fail if the module tries to import e.g. _io.so. Set this to False
815
    to call hostpython from its own build dir, installing the module in
816
    the right place via arguments to setup.py. However, this may not set
817
    the environment correctly and so False is not the default.'''
818

819
    install_in_hostpython = False
4✔
820
    '''If True, additionally installs the module in the hostpython build
2✔
821
    dir. This will make it available to other recipes if
822
    call_hostpython_via_targetpython is False.
823
    '''
824

825
    install_in_targetpython = True
4✔
826
    '''If True, installs the module in the targetpython installation dir.
2✔
827
    This is almost always what you want to do.'''
828

829
    setup_extra_args = []
4✔
830
    '''List of extra arguments to pass to setup.py'''
2✔
831

832
    depends = ['python3']
4✔
833
    '''
2✔
834
    .. note:: it's important to keep this depends as a class attribute outside
835
              `__init__` because sometimes we only initialize the class, so the
836
              `__init__` call won't be called and the deps would be missing
837
              (which breaks the dependency graph computation)
838

839
    .. warning:: don't forget to call `super().__init__()` in any recipe's
840
                 `__init__`, or otherwise it may not be ensured that it depends
841
                 on python2 or python3 which can break the dependency graph
842
    '''
843

844
    hostpython_prerequisites = []
4✔
845
    '''List of hostpython packages required to build a recipe'''
2✔
846

847
    def __init__(self, *args, **kwargs):
4✔
848
        super().__init__(*args, **kwargs)
4✔
849
        if 'python3' not in self.depends:
4✔
850
            # We ensure here that the recipe depends on python even it overrode
851
            # `depends`. We only do this if it doesn't already depend on any
852
            # python, since some recipes intentionally don't depend on/work
853
            # with all python variants
854
            depends = self.depends
4✔
855
            depends.append('python3')
4✔
856
            depends = list(set(depends))
4✔
857
            self.depends = depends
4✔
858

859
    def clean_build(self, arch=None):
4✔
860
        super().clean_build(arch=arch)
×
861
        name = self.folder_name
×
862
        python_install_dirs = glob.glob(join(self.ctx.python_installs_dir, '*'))
×
863
        for python_install in python_install_dirs:
×
864
            site_packages_dir = glob.glob(join(python_install, 'lib', 'python*',
×
865
                                               'site-packages'))
866
            if site_packages_dir:
×
867
                build_dir = join(site_packages_dir[0], name)
×
868
                if exists(build_dir):
×
869
                    info('Deleted {}'.format(build_dir))
×
870
                    rmdir(build_dir)
×
871

872
    @property
4✔
873
    def real_hostpython_location(self):
4✔
874
        host_name = 'host{}'.format(self.ctx.python_recipe.name)
4✔
875
        if host_name == 'hostpython3':
4!
876
            python_recipe = Recipe.get_recipe(host_name, self.ctx)
4✔
877
            return python_recipe.python_exe
4✔
878
        else:
879
            python_recipe = self.ctx.python_recipe
×
880
            return 'python{}'.format(python_recipe.version)
×
881

882
    @property
4✔
883
    def hostpython_location(self):
4✔
884
        if not self.call_hostpython_via_targetpython:
4!
885
            return self.real_hostpython_location
4✔
886
        return self.ctx.hostpython
×
887

888
    @property
4✔
889
    def folder_name(self):
4✔
890
        '''The name of the build folders containing this recipe.'''
891
        name = self.site_packages_name
×
892
        if name is None:
×
893
            name = self.name
×
894
        return name
×
895

896
    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
4✔
897
        env = super().get_recipe_env(arch, with_flags_in_cc)
4✔
898
        env['PYTHONNOUSERSITE'] = '1'
4✔
899
        # Set the LANG, this isn't usually important but is a better default
900
        # as it occasionally matters how Python e.g. reads files
901
        env['LANG'] = "en_GB.UTF-8"
4✔
902
        # Binaries made by packages installed by pip
903
        env["PATH"] = join(self.hostpython_site_dir, "bin") + ":" + env["PATH"]
4✔
904

905
        if not self.call_hostpython_via_targetpython:
4!
906
            env['CFLAGS'] += ' -I{}'.format(
4✔
907
                self.ctx.python_recipe.include_root(arch.arch)
908
            )
909
            env['LDFLAGS'] += ' -L{} -lpython{}'.format(
4✔
910
                self.ctx.python_recipe.link_root(arch.arch),
911
                self.ctx.python_recipe.link_version,
912
            )
913

914
            hppath = []
4✔
915
            hppath.append(join(dirname(self.hostpython_location), 'Lib'))
4✔
916
            hppath.append(join(hppath[0], 'site-packages'))
4✔
917
            builddir = join(dirname(self.hostpython_location), 'build')
4✔
918
            if exists(builddir):
4!
919
                hppath += [join(builddir, d) for d in listdir(builddir)
×
920
                           if isdir(join(builddir, d))]
921
            if len(hppath) > 0:
4!
922
                if 'PYTHONPATH' in env:
4!
923
                    env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']])
×
924
                else:
925
                    env['PYTHONPATH'] = ':'.join(hppath)
4✔
926
        return env
4✔
927

928
    def should_build(self, arch):
4✔
929
        name = self.folder_name
×
930
        if self.ctx.has_package(name, arch):
×
931
            info('Python package already exists in site-packages')
×
932
            return False
×
933
        info('{} apparently isn\'t already in site-packages'.format(name))
×
934
        return True
×
935

936
    def build_arch(self, arch):
4✔
937
        '''Install the Python module by calling setup.py install with
938
        the target Python dir.'''
939
        self.install_hostpython_prerequisites()
×
940
        super().build_arch(arch)
×
941
        self.install_python_package(arch)
×
942

943
    def install_python_package(self, arch, name=None, env=None, is_dir=True):
4✔
944
        '''Automate the installation of a Python package (or a cython
945
        package where the cython components are pre-built).'''
946
        # arch = self.filtered_archs[0]  # old kivy-ios way
947
        if name is None:
×
948
            name = self.name
×
949
        if env is None:
×
950
            env = self.get_recipe_env(arch)
×
951

952
        info('Installing {} into site-packages'.format(self.name))
×
953

954
        hostpython = sh.Command(self.hostpython_location)
×
955
        hpenv = env.copy()
×
956
        with current_directory(self.get_build_dir(arch.arch)):
×
957
            shprint(hostpython, 'setup.py', 'install', '-O2',
×
958
                    '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)),
959
                    '--install-lib=.',
960
                    _env=hpenv, *self.setup_extra_args)
961

962
            # If asked, also install in the hostpython build dir
963
            if self.install_in_hostpython:
×
964
                self.install_hostpython_package(arch)
×
965

966
    def get_hostrecipe_env(self, arch):
4✔
967
        env = environ.copy()
×
968
        env['PYTHONPATH'] = self.hostpython_site_dir
×
969
        return env
×
970

971
    @property
4✔
972
    def hostpython_site_dir(self):
4✔
973
        return join(dirname(self.real_hostpython_location), 'Lib', 'site-packages')
4✔
974

975
    def install_hostpython_package(self, arch):
4✔
976
        env = self.get_hostrecipe_env(arch)
×
977
        real_hostpython = sh.Command(self.real_hostpython_location)
×
978
        shprint(real_hostpython, 'setup.py', 'install', '-O2',
×
979
                '--root={}'.format(dirname(self.real_hostpython_location)),
980
                '--install-lib=Lib/site-packages',
981
                _env=env, *self.setup_extra_args)
982

983
    @property
4✔
984
    def python_major_minor_version(self):
4✔
985
        parsed_version = packaging.version.parse(self.ctx.python_recipe.version)
×
986
        return f"{parsed_version.major}.{parsed_version.minor}"
×
987

988
    def install_hostpython_prerequisites(self, packages=None, force_upgrade=True):
4✔
989
        if not packages:
×
990
            packages = self.hostpython_prerequisites
×
991

992
        if len(packages) == 0:
×
993
            return
×
994

995
        pip_options = [
×
996
            "install",
997
            *packages,
998
            "--target", self.hostpython_site_dir, "--python-version",
999
            self.ctx.python_recipe.version,
1000
            # Don't use sources, instead wheels
1001
            "--only-binary=:all:",
1002
        ]
1003
        if force_upgrade:
×
1004
            pip_options.append("--upgrade")
×
1005
        # Use system's pip
1006
        shprint(sh.pip, *pip_options)
×
1007

1008
    def restore_hostpython_prerequisites(self, packages):
4✔
1009
        _packages = []
×
1010
        for package in packages:
×
1011
            original_version = Recipe.get_recipe(package, self.ctx).version
×
1012
            _packages.append(package + "==" + original_version)
×
1013
        self.install_hostpython_prerequisites(packages=_packages)
×
1014

1015

1016
class CompiledComponentsPythonRecipe(PythonRecipe):
4✔
1017
    pre_build_ext = False
4✔
1018

1019
    build_cmd = 'build_ext'
4✔
1020

1021
    def build_arch(self, arch):
4✔
1022
        '''Build any cython components, then install the Python module by
1023
        calling setup.py install with the target Python dir.
1024
        '''
1025
        Recipe.build_arch(self, arch)
×
1026
        self.install_hostpython_prerequisites()
×
1027
        self.build_compiled_components(arch)
×
1028
        self.install_python_package(arch)
×
1029

1030
    def build_compiled_components(self, arch):
4✔
1031
        info('Building compiled components in {}'.format(self.name))
×
1032

1033
        env = self.get_recipe_env(arch)
×
1034
        hostpython = sh.Command(self.hostpython_location)
×
1035
        with current_directory(self.get_build_dir(arch.arch)):
×
1036
            if self.install_in_hostpython:
×
1037
                shprint(hostpython, 'setup.py', 'clean', '--all', _env=env)
×
1038
            shprint(hostpython, 'setup.py', self.build_cmd, '-v',
×
1039
                    _env=env, *self.setup_extra_args)
1040
            build_dir = glob.glob('build/lib.*')[0]
×
1041
            shprint(sh.find, build_dir, '-name', '"*.o"', '-exec',
×
1042
                    env['STRIP'], '{}', ';', _env=env)
1043

1044
    def install_hostpython_package(self, arch):
4✔
1045
        env = self.get_hostrecipe_env(arch)
×
1046
        self.rebuild_compiled_components(arch, env)
×
1047
        super().install_hostpython_package(arch)
×
1048

1049
    def rebuild_compiled_components(self, arch, env):
4✔
1050
        info('Rebuilding compiled components in {}'.format(self.name))
×
1051

1052
        hostpython = sh.Command(self.real_hostpython_location)
×
1053
        shprint(hostpython, 'setup.py', 'clean', '--all', _env=env)
×
1054
        shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env,
×
1055
                *self.setup_extra_args)
1056

1057

1058
class CppCompiledComponentsPythonRecipe(CompiledComponentsPythonRecipe):
4✔
1059
    """ Extensions that require the cxx-stl """
1060
    call_hostpython_via_targetpython = False
4✔
1061
    need_stl_shared = True
4✔
1062

1063

1064
class CythonRecipe(PythonRecipe):
4✔
1065
    pre_build_ext = False
4✔
1066
    cythonize = True
4✔
1067
    cython_args = []
4✔
1068
    call_hostpython_via_targetpython = False
4✔
1069

1070
    def build_arch(self, arch):
4✔
1071
        '''Build any cython components, then install the Python module by
1072
        calling setup.py install with the target Python dir.
1073
        '''
1074
        Recipe.build_arch(self, arch)
×
1075
        self.build_cython_components(arch)
×
1076
        self.install_python_package(arch)
×
1077

1078
    def build_cython_components(self, arch):
4✔
1079
        info('Cythonizing anything necessary in {}'.format(self.name))
×
1080

1081
        env = self.get_recipe_env(arch)
×
1082

1083
        with current_directory(self.get_build_dir(arch.arch)):
×
1084
            hostpython = sh.Command(self.ctx.hostpython)
×
1085
            shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
×
1086
            debug('cwd is {}'.format(realpath(curdir)))
×
1087
            info('Trying first build of {} to get cython files: this is '
×
1088
                 'expected to fail'.format(self.name))
1089

1090
            manually_cythonise = False
×
1091
            try:
×
1092
                shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
×
1093
                        *self.setup_extra_args)
1094
            except sh.ErrorReturnCode_1:
×
1095
                print()
×
1096
                info('{} first build failed (as expected)'.format(self.name))
×
1097
                manually_cythonise = True
×
1098

1099
            if manually_cythonise:
×
1100
                self.cythonize_build(env=env)
×
1101
                shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
×
1102
                        _tail=20, _critical=True, *self.setup_extra_args)
1103
            else:
1104
                info('First build appeared to complete correctly, skipping manual'
×
1105
                     'cythonising.')
1106

1107
            if not self.ctx.with_debug_symbols:
×
1108
                self.strip_object_files(arch, env)
×
1109

1110
    def strip_object_files(self, arch, env, build_dir=None):
4✔
1111
        if build_dir is None:
×
1112
            build_dir = self.get_build_dir(arch.arch)
×
1113
        with current_directory(build_dir):
×
1114
            info('Stripping object files')
×
1115
            shprint(sh.find, '.', '-iname', '*.so', '-exec',
×
1116
                    '/usr/bin/echo', '{}', ';', _env=env)
1117
            shprint(sh.find, '.', '-iname', '*.so', '-exec',
×
1118
                    env['STRIP'].split(' ')[0], '--strip-unneeded',
1119
                    # '/usr/bin/strip', '--strip-unneeded',
1120
                    '{}', ';', _env=env)
1121

1122
    def cythonize_file(self, env, build_dir, filename):
4✔
1123
        short_filename = filename
×
1124
        if filename.startswith(build_dir):
×
1125
            short_filename = filename[len(build_dir) + 1:]
×
1126
        info(u"Cythonize {}".format(short_filename))
×
1127
        cyenv = env.copy()
×
1128
        if 'CYTHONPATH' in cyenv:
×
1129
            cyenv['PYTHONPATH'] = cyenv['CYTHONPATH']
×
1130
        elif 'PYTHONPATH' in cyenv:
×
1131
            del cyenv['PYTHONPATH']
×
1132
        if 'PYTHONNOUSERSITE' in cyenv:
×
1133
            cyenv.pop('PYTHONNOUSERSITE')
×
1134
        python_command = sh.Command("python{}".format(
×
1135
            self.ctx.python_recipe.major_minor_version_string.split(".")[0]
1136
        ))
1137
        shprint(python_command, "-c"
×
1138
                "import sys; from Cython.Compiler.Main import setuptools_main; sys.exit(setuptools_main());",
1139
                filename, *self.cython_args, _env=cyenv)
1140

1141
    def cythonize_build(self, env, build_dir="."):
4✔
1142
        if not self.cythonize:
×
1143
            info('Running cython cancelled per recipe setting')
×
1144
            return
×
1145
        info('Running cython where appropriate')
×
1146
        for root, dirnames, filenames in walk("."):
×
1147
            for filename in fnmatch.filter(filenames, "*.pyx"):
×
1148
                self.cythonize_file(env, build_dir, join(root, filename))
×
1149

1150
    def get_recipe_env(self, arch, with_flags_in_cc=True):
4✔
1151
        env = super().get_recipe_env(arch, with_flags_in_cc)
×
1152
        env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format(
×
1153
            self.ctx.get_libs_dir(arch.arch) +
1154
            ' -L{} '.format(self.ctx.libs_dir) +
1155
            ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local',
1156
                                arch.arch)))
1157

1158
        env['LDSHARED'] = env['CC'] + ' -shared'
×
1159
        # shprint(sh.whereis, env['LDSHARED'], _env=env)
1160
        env['LIBLINK'] = 'NOTNONE'
×
1161
        if self.ctx.copy_libs:
×
1162
            env['COPYLIBS'] = '1'
×
1163

1164
        # Every recipe uses its own liblink path, object files are
1165
        # collected and biglinked later
1166
        liblink_path = join(self.get_build_container_dir(arch.arch),
×
1167
                            'objects_{}'.format(self.name))
1168
        env['LIBLINK_PATH'] = liblink_path
×
1169
        ensure_dir(liblink_path)
×
1170

1171
        return env
×
1172

1173

1174
class PyProjectRecipe(PythonRecipe):
4✔
1175
    '''Recipe for projects which containes `pyproject.toml`'''
1176

1177
    # Extra args to pass to `python -m build ...`
1178
    extra_build_args = []
4✔
1179
    call_hostpython_via_targetpython = False
4✔
1180

1181
    def get_recipe_env(self, arch, **kwargs):
4✔
1182
        # Custom hostpython
1183
        self.ctx.python_recipe.python_exe = join(
4✔
1184
            self.ctx.python_recipe.get_build_dir(arch), "android-build", "python3")
1185
        env = super().get_recipe_env(arch, **kwargs)
4✔
1186
        build_dir = self.get_build_dir(arch)
4✔
1187
        ensure_dir(build_dir)
4✔
1188
        build_opts = join(build_dir, "build-opts.cfg")
4✔
1189

1190
        with open(build_opts, "w") as file:
4✔
1191
            file.write("[bdist_wheel]\nplat-name={}".format(
4✔
1192
                self.get_wheel_platform_tag(arch)
1193
            ))
1194
            file.close()
4✔
1195

1196
        env["DIST_EXTRA_CONFIG"] = build_opts
4✔
1197
        return env
4✔
1198

1199
    def get_wheel_platform_tag(self, arch):
4✔
1200
        return "android_" + {
4✔
1201
            "armeabi-v7a": "arm",
1202
            "arm64-v8a": "aarch64",
1203
            "x86_64": "x86_64",
1204
            "x86": "i686",
1205
        }[arch.arch]
1206

1207
    def install_wheel(self, arch, built_wheels):
4✔
1208
        _wheel = built_wheels[0]
×
1209
        built_wheel_dir = dirname(_wheel)
×
1210
        # Fix wheel platform tag
1211
        wheel_tag = wheel_tags(
×
1212
            _wheel,
1213
            platform_tags=self.get_wheel_platform_tag(arch),
1214
            remove=True,
1215
        )
1216
        selected_wheel = join(built_wheel_dir, wheel_tag)
×
1217

1218
        _dev_wheel_dir = environ.get("P4A_WHEEL_DIR", False)
×
1219
        if _dev_wheel_dir:
×
1220
            ensure_dir(_dev_wheel_dir)
×
1221
            shprint(sh.cp, selected_wheel, _dev_wheel_dir)
×
1222

1223
        info(f"Installing built wheel: {wheel_tag}")
×
1224
        destination = self.ctx.get_python_install_dir(arch.arch)
×
1225
        with WheelFile(selected_wheel) as wf:
×
1226
            for zinfo in wf.filelist:
×
1227
                wf.extract(zinfo, destination)
×
1228
            wf.close()
×
1229

1230
    def build_arch(self, arch):
4✔
1231
        self.install_hostpython_prerequisites(
×
1232
            packages=["build[virtualenv]", "pip"] + self.hostpython_prerequisites
1233
        )
1234
        build_dir = self.get_build_dir(arch.arch)
×
1235
        env = self.get_recipe_env(arch, with_flags_in_cc=True)
×
1236
        # make build dir separatly
1237
        sub_build_dir = join(build_dir, "p4a_android_build")
×
1238
        ensure_dir(sub_build_dir)
×
1239
        # copy hostpython to built python to ensure correct selection of libs and includes
1240
        shprint(sh.cp, self.real_hostpython_location, self.ctx.python_recipe.python_exe)
×
1241

1242
        build_args = [
×
1243
            "-m",
1244
            "build",
1245
            "--wheel",
1246
            "--config-setting",
1247
            "builddir={}".format(sub_build_dir),
1248
        ] + self.extra_build_args
1249

1250
        built_wheels = []
×
1251
        with current_directory(build_dir):
×
1252
            shprint(
×
1253
                sh.Command(self.ctx.python_recipe.python_exe), *build_args, _env=env
1254
            )
1255
            built_wheels = [realpath(whl) for whl in glob.glob("dist/*.whl")]
×
1256
        self.install_wheel(arch, built_wheels)
×
1257

1258

1259
class MesonRecipe(PyProjectRecipe):
4✔
1260
    '''Recipe for projects which uses meson as build system'''
1261

1262
    meson_version = "1.4.0"
4✔
1263
    ninja_version = "1.11.1.1"
4✔
1264

1265
    def sanitize_flags(self, *flag_strings):
4✔
1266
        return " ".join(flag_strings).strip().split(" ")
×
1267

1268
    def get_recipe_meson_options(self, arch):
4✔
1269
        env = self.get_recipe_env(arch, with_flags_in_cc=True)
×
1270
        return {
×
1271
            "binaries": {
1272
                "c": arch.get_clang_exe(with_target=True),
1273
                "cpp": arch.get_clang_exe(with_target=True, plus_plus=True),
1274
                "ar": self.ctx.ndk.llvm_ar,
1275
                "strip": self.ctx.ndk.llvm_strip,
1276
            },
1277
            "built-in options": {
1278
                "c_args": self.sanitize_flags(env["CFLAGS"], env["CPPFLAGS"]),
1279
                "cpp_args": self.sanitize_flags(env["CXXFLAGS"], env["CPPFLAGS"]),
1280
                "c_link_args": self.sanitize_flags(env["LDFLAGS"]),
1281
                "cpp_link_args": self.sanitize_flags(env["LDFLAGS"]),
1282
            },
1283
            "properties": {
1284
                "needs_exe_wrapper": True,
1285
                "sys_root": self.ctx.ndk.sysroot
1286
            },
1287
            "host_machine": {
1288
                "cpu_family": {
1289
                    "arm64-v8a": "aarch64",
1290
                    "armeabi-v7a": "arm",
1291
                    "x86_64": "x86_64",
1292
                    "x86": "x86"
1293
                }[arch.arch],
1294
                "cpu": {
1295
                    "arm64-v8a": "aarch64",
1296
                    "armeabi-v7a": "armv7",
1297
                    "x86_64": "x86_64",
1298
                    "x86": "i686"
1299
                }[arch.arch],
1300
                "endian": "little",
1301
                "system": "android",
1302
            }
1303
        }
1304

1305
    def write_build_options(self, arch):
4✔
1306
        """Writes python dict to meson config file"""
1307
        option_data = ""
×
1308
        build_options = self.get_recipe_meson_options(arch)
×
1309
        for key in build_options.keys():
×
1310
            data_chunk = "[{}]".format(key)
×
1311
            for subkey in build_options[key].keys():
×
1312
                value = build_options[key][subkey]
×
1313
                if isinstance(value, int):
×
1314
                    value = str(value)
×
1315
                elif isinstance(value, str):
×
1316
                    value = "'{}'".format(value)
×
1317
                elif isinstance(value, bool):
×
1318
                    value = "true" if value else "false"
×
1319
                elif isinstance(value, list):
×
1320
                    value = "['" + "', '".join(value) + "']"
×
1321
                data_chunk += "\n" + subkey + " = " + value
×
1322
            option_data += data_chunk + "\n\n"
×
1323
        return option_data
×
1324

1325
    def ensure_args(self, *args):
4✔
1326
        for arg in args:
×
1327
            if arg not in self.extra_build_args:
×
1328
                self.extra_build_args.append(arg)
×
1329

1330
    def build_arch(self, arch):
4✔
1331
        cross_file = join("/tmp", "android.meson.cross")
×
1332
        info("Writing cross file at: {}".format(cross_file))
×
1333
        # write cross config file
1334
        with open(cross_file, "w") as file:
×
1335
            file.write(self.write_build_options(arch))
×
1336
            file.close()
×
1337
        # set cross file
1338
        self.ensure_args('-Csetup-args=--cross-file', '-Csetup-args={}'.format(cross_file))
×
1339
        # ensure ninja and meson
1340
        for dep in [
×
1341
            "ninja=={}".format(self.ninja_version),
1342
            "meson=={}".format(self.meson_version),
1343
        ]:
1344
            if dep not in self.hostpython_prerequisites:
×
1345
                self.hostpython_prerequisites.append(dep)
×
1346
        super().build_arch(arch)
×
1347

1348

1349
class RustCompiledComponentsRecipe(PyProjectRecipe):
4✔
1350
    # Rust toolchain codes
1351
    # https://doc.rust-lang.org/nightly/rustc/platform-support.html
1352
    RUST_ARCH_CODES = {
4✔
1353
        "arm64-v8a": "aarch64-linux-android",
1354
        "armeabi-v7a": "armv7-linux-androideabi",
1355
        "x86_64": "x86_64-linux-android",
1356
        "x86": "i686-linux-android",
1357
    }
1358

1359
    call_hostpython_via_targetpython = False
4✔
1360

1361
    def get_recipe_env(self, arch, **kwargs):
4✔
1362
        env = super().get_recipe_env(arch, **kwargs)
×
1363

1364
        # Set rust build target
1365
        build_target = self.RUST_ARCH_CODES[arch.arch]
×
1366
        cargo_linker_name = "CARGO_TARGET_{}_LINKER".format(
×
1367
            build_target.upper().replace("-", "_")
1368
        )
1369
        env["CARGO_BUILD_TARGET"] = build_target
×
1370
        env[cargo_linker_name] = join(
×
1371
            self.ctx.ndk.llvm_prebuilt_dir,
1372
            "bin",
1373
            "{}{}-clang".format(
1374
                # NDK's Clang format
1375
                build_target.replace("7", "7a")
1376
                if build_target.startswith("armv7")
1377
                else build_target,
1378
                self.ctx.ndk_api,
1379
            ),
1380
        )
1381
        realpython_dir = self.ctx.python_recipe.get_build_dir(arch.arch)
×
1382

1383
        env["RUSTFLAGS"] = "-Clink-args=-L{} -L{}".format(
×
1384
            self.ctx.get_libs_dir(arch.arch), join(realpython_dir, "android-build")
1385
        )
1386

1387
        env["PYO3_CROSS_LIB_DIR"] = realpath(glob.glob(join(
×
1388
            realpython_dir, "android-build", "build",
1389
            "lib.linux-*-{}/".format(self.python_major_minor_version),
1390
        ))[0])
1391

1392
        info_main("Ensuring rust build toolchain")
×
1393
        shprint(sh.rustup, "target", "add", build_target)
×
1394

1395
        # Add host python to PATH
1396
        env["PATH"] = ("{hostpython_dir}:{old_path}").format(
×
1397
            hostpython_dir=Recipe.get_recipe(
1398
                "hostpython3", self.ctx
1399
            ).get_path_to_python(),
1400
            old_path=env["PATH"],
1401
        )
1402
        return env
×
1403

1404
    def check_host_deps(self):
4✔
1405
        if not hasattr(sh, "rustup"):
×
1406
            error(
×
1407
                "`rustup` was not found on host system."
1408
                "Please install it using :"
1409
                "\n`curl https://sh.rustup.rs -sSf | sh`\n"
1410
            )
1411
            exit(1)
×
1412

1413
    def build_arch(self, arch):
4✔
1414
        self.check_host_deps()
×
1415
        super().build_arch(arch)
×
1416

1417

1418
class TargetPythonRecipe(Recipe):
4✔
1419
    '''Class for target python recipes. Sets ctx.python_recipe to point to
1420
    itself, so as to know later what kind of Python was built or used.'''
1421

1422
    def __init__(self, *args, **kwargs):
4✔
1423
        self._ctx = None
4✔
1424
        super().__init__(*args, **kwargs)
4✔
1425

1426
    def prebuild_arch(self, arch):
4✔
1427
        super().prebuild_arch(arch)
×
1428
        self.ctx.python_recipe = self
×
1429

1430
    def include_root(self, arch):
4✔
1431
        '''The root directory from which to include headers.'''
1432
        raise NotImplementedError('Not implemented in TargetPythonRecipe')
1433

1434
    def link_root(self):
4✔
1435
        raise NotImplementedError('Not implemented in TargetPythonRecipe')
1436

1437
    @property
4✔
1438
    def major_minor_version_string(self):
4✔
1439
        parsed_version = packaging.version.parse(self.version)
4✔
1440
        return f"{parsed_version.major}.{parsed_version.minor}"
4✔
1441

1442
    def create_python_bundle(self, dirn, arch):
4✔
1443
        """
1444
        Create a packaged python bundle in the target directory, by
1445
        copying all the modules and standard library to the right
1446
        place.
1447
        """
1448
        raise NotImplementedError('{} does not implement create_python_bundle'.format(self))
1449

1450
    def reduce_object_file_names(self, dirn):
4✔
1451
        """Recursively renames all files named XXX.cpython-...-linux-gnu.so"
1452
        to "XXX.so", i.e. removing the erroneous architecture name
1453
        coming from the local system.
1454
        """
1455
        py_so_files = shprint(sh.find, dirn, '-iname', '*.so')
4✔
1456
        filens = py_so_files.split('\n')[:-1]
4✔
1457
        for filen in filens:
4!
1458
            file_dirname, file_basename = split(filen)
×
1459
            parts = file_basename.split('.')
×
1460
            if len(parts) <= 2:
×
1461
                continue
×
1462
            # PySide6 libraries end with .abi3.so
1463
            if parts[1] == "abi3":
×
1464
                continue
×
1465
            move(filen, join(file_dirname, parts[0] + '.so'))
×
1466

1467

1468
def algsum(alg, filen):
4✔
1469
    '''Calculate the digest of a file.
1470
    '''
1471
    with open(filen, 'rb') as fileh:
×
1472
        digest = getattr(hashlib, alg)(fileh.read())
×
1473

1474
    return digest.hexdigest()
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc