• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

kivy / python-for-android / 6774900670

06 Nov 2023 06:20PM UTC coverage: 59.187% (+0.08%) from 59.106%
6774900670

push

github

web-flow
Remove `distutils` usage, as is not available anymore on Python `3.12` (#2912)

* Remove distutils usage, as is not available anymore on Python 3.12

* Updated testapps to use setuptools instead of distutils

943 of 2239 branches covered (0.0%)

Branch coverage included in aggregate %.

24 of 26 new or added lines in 6 files covered. (92.31%)

1 existing line in 1 file now uncovered.

4733 of 7351 relevant lines covered (64.39%)

2.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.95
/pythonforandroid/recipes/python3/__init__.py
1
import glob
4✔
2
import sh
4✔
3
import subprocess
4✔
4

5
from multiprocessing import cpu_count
4✔
6
from os import environ, utime
4✔
7
from os.path import dirname, exists, join
4✔
8
from pathlib import Path
4✔
9
import shutil
4✔
10

11
from pythonforandroid.logger import info, warning, shprint
4✔
12
from pythonforandroid.patching import version_starts_with
4✔
13
from pythonforandroid.recipe import Recipe, TargetPythonRecipe
4✔
14
from pythonforandroid.util import (
4✔
15
    current_directory,
16
    ensure_dir,
17
    walk_valid_filens,
18
    BuildInterruptingException,
19
)
20

21
NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (
4✔
22
    'Target ndk-api is {ndk_api}, '
23
    'but the python3 recipe supports only {min_ndk_api}+'
24
)
25

26

27
class Python3Recipe(TargetPythonRecipe):
4✔
28
    '''
29
    The python3's recipe
30
    ^^^^^^^^^^^^^^^^^^^^
31

32
    The python 3 recipe can be built with some extra python modules, but to do
33
    so, we need some libraries. By default, we ship the python3 recipe with
34
    some common libraries, defined in ``depends``. We also support some optional
35
    libraries, which are less common that the ones defined in ``depends``, so
36
    we added them as optional dependencies (``opt_depends``).
37

38
    Below you have a relationship between the python modules and the recipe
39
    libraries::
40

41
        - _ctypes: you must add the recipe for ``libffi``.
42
        - _sqlite3: you must add the recipe for ``sqlite3``.
43
        - _ssl: you must add the recipe for ``openssl``.
44
        - _bz2: you must add the recipe for ``libbz2`` (optional).
45
        - _lzma: you must add the recipe for ``liblzma`` (optional).
46

47
    .. note:: This recipe can be built only against API 21+.
48

49
    .. versionchanged:: 2019.10.06.post0
50
        - Refactored from deleted class ``python.GuestPythonRecipe`` into here
51
        - Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
52
          and :mod:`~pythonforandroid.recipes.liblzma`
53

54
    .. versionchanged:: 0.6.0
55
        Refactored into class
56
        :class:`~pythonforandroid.python.GuestPythonRecipe`
57
    '''
58

59
    version = '3.10.10'
4✔
60
    url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz'
4✔
61
    name = 'python3'
4✔
62

63
    patches = [
4✔
64
        'patches/pyconfig_detection.patch',
65
        'patches/reproducible-buildinfo.diff',
66

67
        # Python 3.7.1
68
        ('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")),
69
        ('patches/py3.7.1_fix-zlib-version.patch', version_starts_with("3.7")),
70

71
        # Python 3.8.1 & 3.9.X
72
        ('patches/py3.8.1.patch', version_starts_with("3.8")),
73
        ('patches/py3.8.1.patch', version_starts_with("3.9")),
74
        ('patches/py3.8.1.patch', version_starts_with("3.10"))
75
    ]
76

77
    if shutil.which('lld') is not None:
4✔
78
        patches = patches + [
4✔
79
            ("patches/py3.7.1_fix_cortex_a8.patch", version_starts_with("3.7")),
80
            ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.8")),
81
            ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.9")),
82
            ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10"))
83
        ]
84

85
    depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
4✔
86
    # those optional depends allow us to build python compression modules:
87
    #   - _bz2.so
88
    #   - _lzma.so
89
    opt_depends = ['libbz2', 'liblzma']
4✔
90
    '''The optional libraries which we would like to get our python linked'''
2✔
91

92
    configure_args = (
4✔
93
        '--host={android_host}',
94
        '--build={android_build}',
95
        '--enable-shared',
96
        '--enable-ipv6',
97
        'ac_cv_file__dev_ptmx=yes',
98
        'ac_cv_file__dev_ptc=no',
99
        '--without-ensurepip',
100
        'ac_cv_little_endian_double=yes',
101
        'ac_cv_header_sys_eventfd_h=no',
102
        '--prefix={prefix}',
103
        '--exec-prefix={exec_prefix}',
104
        '--enable-loadable-sqlite-extensions')
105
    '''The configure arguments needed to build the python recipe. Those are
2✔
106
    used in method :meth:`build_arch` (if not overwritten like python3's
107
    recipe does).
108
    '''
109

110
    MIN_NDK_API = 21
4✔
111
    '''Sets the minimal ndk api number needed to use the recipe.
2✔
112

113
    .. warning:: This recipe can be built only against API 21+, so it means
114
        that any class which inherits from class:`GuestPythonRecipe` will have
115
        this limitation.
116
    '''
117

118
    stdlib_dir_blacklist = {
4✔
119
        '__pycache__',
120
        'test',
121
        'tests',
122
        'lib2to3',
123
        'ensurepip',
124
        'idlelib',
125
        'tkinter',
126
    }
127
    '''The directories that we want to omit for our python bundle'''
2✔
128

129
    stdlib_filen_blacklist = [
4✔
130
        '*.py',
131
        '*.exe',
132
        '*.whl',
133
    ]
134
    '''The file extensions that we want to blacklist for our python bundle'''
2✔
135

136
    site_packages_dir_blacklist = {
4✔
137
        '__pycache__',
138
        'tests'
139
    }
140
    '''The directories from site packages dir that we don't want to be included
2✔
141
    in our python bundle.'''
142

143
    site_packages_filen_blacklist = [
4✔
144
        '*.py'
145
    ]
146
    '''The file extensions from site packages dir that we don't want to be
2✔
147
    included in our python bundle.'''
148

149
    compiled_extension = '.pyc'
4✔
150
    '''the default extension for compiled python files.
2✔
151

152
    .. note:: the default extension for compiled python files has been .pyo for
153
        python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
154
        longer used and has been removed in favour of extension .pyc
155
    '''
156

157
    def __init__(self, *args, **kwargs):
4✔
158
        self._ctx = None
4✔
159
        super().__init__(*args, **kwargs)
4✔
160

161
    @property
4✔
162
    def _libpython(self):
4✔
163
        '''return the python's library name (with extension)'''
164
        return 'libpython{link_version}.so'.format(
4✔
165
            link_version=self.link_version
166
        )
167

168
    @property
4✔
169
    def link_version(self):
4✔
170
        '''return the python's library link version e.g. 3.7m, 3.8'''
171
        major, minor = self.major_minor_version_string.split('.')
4✔
172
        flags = ''
4✔
173
        if major == '3' and int(minor) < 8:
4!
174
            flags += 'm'
×
175
        return '{major}.{minor}{flags}'.format(
4✔
176
            major=major,
177
            minor=minor,
178
            flags=flags
179
        )
180

181
    def include_root(self, arch_name):
4✔
182
        return join(self.get_build_dir(arch_name), 'Include')
4✔
183

184
    def link_root(self, arch_name):
4✔
185
        return join(self.get_build_dir(arch_name), 'android-build')
4✔
186

187
    def should_build(self, arch):
4✔
188
        return not Path(self.link_root(arch.arch), self._libpython).is_file()
4✔
189

190
    def prebuild_arch(self, arch):
4✔
191
        super().prebuild_arch(arch)
×
192
        self.ctx.python_recipe = self
×
193

194
    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
4✔
195
        env = super().get_recipe_env(arch)
4✔
196
        env['HOSTARCH'] = arch.command_prefix
4✔
197

198
        env['CC'] = arch.get_clang_exe(with_target=True)
4✔
199

200
        env['PATH'] = (
4✔
201
            '{hostpython_dir}:{old_path}').format(
202
                hostpython_dir=self.get_recipe(
203
                    'host' + self.name, self.ctx).get_path_to_python(),
204
                old_path=env['PATH'])
205

206
        env['CFLAGS'] = ' '.join(
4✔
207
            [
208
                '-fPIC',
209
                '-DANDROID'
210
            ]
211
        )
212

213
        env['LDFLAGS'] = env.get('LDFLAGS', '')
4✔
214
        if shutil.which('lld') is not None:
4!
215
            # Note: The -L. is to fix a bug in python 3.7.
216
            # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
217
            env['LDFLAGS'] += ' -L. -fuse-ld=lld'
4✔
218
        else:
UNCOV
219
            warning('lld not found, linking without it. '
×
220
                    'Consider installing lld if linker errors occur.')
221

222
        return env
4✔
223

224
    def set_libs_flags(self, env, arch):
4✔
225
        '''Takes care to properly link libraries with python depending on our
226
        requirements and the attribute :attr:`opt_depends`.
227
        '''
228
        def add_flags(include_flags, link_dirs, link_libs):
4✔
229
            env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
4✔
230
            env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
4✔
231
            env['LIBS'] = env.get('LIBS', '') + link_libs
4✔
232

233
        if 'sqlite3' in self.ctx.recipe_build_order:
4!
234
            info('Activating flags for sqlite3')
×
235
            recipe = Recipe.get_recipe('sqlite3', self.ctx)
×
236
            add_flags(' -I' + recipe.get_build_dir(arch.arch),
×
237
                      ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')
238

239
        if 'libffi' in self.ctx.recipe_build_order:
4!
240
            info('Activating flags for libffi')
×
241
            recipe = Recipe.get_recipe('libffi', self.ctx)
×
242
            # In order to force the correct linkage for our libffi library, we
243
            # set the following variable to point where is our libffi.pc file,
244
            # because the python build system uses pkg-config to configure it.
245
            env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch)
×
246
            add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
×
247
                      ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
248
                      ' -lffi')
249

250
        if 'openssl' in self.ctx.recipe_build_order:
4!
251
            info('Activating flags for openssl')
×
252
            recipe = Recipe.get_recipe('openssl', self.ctx)
×
253
            self.configure_args += \
×
254
                ('--with-openssl=' + recipe.get_build_dir(arch.arch),)
255
            add_flags(recipe.include_flags(arch),
×
256
                      recipe.link_dirs_flags(arch), recipe.link_libs_flags())
257

258
        for library_name in {'libbz2', 'liblzma'}:
4✔
259
            if library_name in self.ctx.recipe_build_order:
4!
260
                info(f'Activating flags for {library_name}')
×
261
                recipe = Recipe.get_recipe(library_name, self.ctx)
×
262
                add_flags(recipe.get_library_includes(arch),
×
263
                          recipe.get_library_ldflags(arch),
264
                          recipe.get_library_libs_flag())
265

266
        # python build system contains hardcoded zlib version which prevents
267
        # the build of zlib module, here we search for android's zlib version
268
        # and sets the right flags, so python can be build with android's zlib
269
        info("Activating flags for android's zlib")
4✔
270
        zlib_lib_path = arch.ndk_lib_dir_versioned
4✔
271
        zlib_includes = self.ctx.ndk.sysroot_include_dir
4✔
272
        zlib_h = join(zlib_includes, 'zlib.h')
4✔
273
        try:
4✔
274
            with open(zlib_h) as fileh:
4✔
275
                zlib_data = fileh.read()
4✔
276
        except IOError:
×
277
            raise BuildInterruptingException(
×
278
                "Could not determine android's zlib version, no zlib.h ({}) in"
279
                " the NDK dir includes".format(zlib_h)
280
            )
281
        for line in zlib_data.split('\n'):
4!
282
            if line.startswith('#define ZLIB_VERSION '):
4!
283
                break
4✔
284
        else:
285
            raise BuildInterruptingException(
×
286
                'Could not parse zlib.h...so we cannot find zlib version,'
287
                'required by python build,'
288
            )
289
        env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
4✔
290
        add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')
4✔
291

292
        return env
4✔
293

294
    def build_arch(self, arch):
4✔
295
        if self.ctx.ndk_api < self.MIN_NDK_API:
4✔
296
            raise BuildInterruptingException(
4✔
297
                NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
298
                    ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
299
                ),
300
            )
301

302
        recipe_build_dir = self.get_build_dir(arch.arch)
4✔
303

304
        # Create a subdirectory to actually perform the build
305
        build_dir = join(recipe_build_dir, 'android-build')
4✔
306
        ensure_dir(build_dir)
4✔
307

308
        # TODO: Get these dynamically, like bpo-30386 does
309
        sys_prefix = '/usr/local'
4✔
310
        sys_exec_prefix = '/usr/local'
4✔
311

312
        env = self.get_recipe_env(arch)
4✔
313
        env = self.set_libs_flags(env, arch)
4✔
314

315
        android_build = sh.Command(
4✔
316
            join(recipe_build_dir,
317
                 'config.guess'))().stdout.strip().decode('utf-8')
318

319
        with current_directory(build_dir):
4✔
320
            if not exists('config.status'):
4!
321
                shprint(
4✔
322
                    sh.Command(join(recipe_build_dir, 'configure')),
323
                    *(' '.join(self.configure_args).format(
324
                                    android_host=env['HOSTARCH'],
325
                                    android_build=android_build,
326
                                    prefix=sys_prefix,
327
                                    exec_prefix=sys_exec_prefix)).split(' '),
328
                    _env=env)
329

330
            shprint(
4✔
331
                sh.make, 'all', '-j', str(cpu_count()),
332
                'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
333
                _env=env
334
            )
335

336
            # TODO: Look into passing the path to pyconfig.h in a
337
            # better way, although this is probably acceptable
338
            sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))
4✔
339

340
    def compile_python_files(self, dir):
4✔
341
        '''
342
        Compile the python files (recursively) for the python files inside
343
        a given folder.
344

345
        .. note:: python2 compiles the files into extension .pyo, but in
346
            python3, and as of Python 3.5, the .pyo filename extension is no
347
            longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
348
        '''
349
        args = [self.ctx.hostpython]
4✔
350
        args += ['-OO', '-m', 'compileall', '-b', '-f', dir]
4✔
351
        subprocess.call(args)
4✔
352

353
    def create_python_bundle(self, dirn, arch):
4✔
354
        """
355
        Create a packaged python bundle in the target directory, by
356
        copying all the modules and standard library to the right
357
        place.
358
        """
359
        # Todo: find a better way to find the build libs folder
360
        modules_build_dir = join(
4✔
361
            self.get_build_dir(arch.arch),
362
            'android-build',
363
            'build',
364
            'lib.linux{}-{}-{}'.format(
365
                '2' if self.version[0] == '2' else '',
366
                arch.command_prefix.split('-')[0],
367
                self.major_minor_version_string
368
            ))
369

370
        # Compile to *.pyc the python modules
371
        self.compile_python_files(modules_build_dir)
4✔
372
        # Compile to *.pyc the standard python library
373
        self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
4✔
374
        # Compile to *.pyc the other python packages (site-packages)
375
        self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))
4✔
376

377
        # Bundle compiled python modules to a folder
378
        modules_dir = join(dirn, 'modules')
4✔
379
        c_ext = self.compiled_extension
4✔
380
        ensure_dir(modules_dir)
4✔
381
        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
4✔
382
                         glob.glob(join(modules_build_dir, '*' + c_ext)))
383
        info("Copy {} files into the bundle".format(len(module_filens)))
4✔
384
        for filen in module_filens:
4!
385
            info(" - copy {}".format(filen))
×
386
            shutil.copy2(filen, modules_dir)
×
387

388
        # zip up the standard library
389
        stdlib_zip = join(dirn, 'stdlib.zip')
4✔
390
        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
4✔
391
            stdlib_filens = list(walk_valid_filens(
4✔
392
                '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
393
            if 'SOURCE_DATE_EPOCH' in environ:
4!
394
                # for reproducible builds
395
                stdlib_filens.sort()
×
396
                timestamp = int(environ['SOURCE_DATE_EPOCH'])
×
397
                for filen in stdlib_filens:
×
398
                    utime(filen, (timestamp, timestamp))
×
399
            info("Zip {} files into the bundle".format(len(stdlib_filens)))
4✔
400
            shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)
4✔
401

402
        # copy the site-packages into place
403
        ensure_dir(join(dirn, 'site-packages'))
4✔
404
        ensure_dir(self.ctx.get_python_install_dir(arch.arch))
4✔
405
        # TODO: Improve the API around walking and copying the files
406
        with current_directory(self.ctx.get_python_install_dir(arch.arch)):
4✔
407
            filens = list(walk_valid_filens(
4✔
408
                '.', self.site_packages_dir_blacklist,
409
                self.site_packages_filen_blacklist))
410
            info("Copy {} files into the site-packages".format(len(filens)))
4✔
411
            for filen in filens:
4✔
412
                info(" - copy {}".format(filen))
4✔
413
                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
4✔
414
                shutil.copy2(filen, join(dirn, 'site-packages', filen))
4✔
415

416
        # copy the python .so files into place
417
        python_build_dir = join(self.get_build_dir(arch.arch),
4✔
418
                                'android-build')
419
        python_lib_name = 'libpython' + self.link_version
4✔
420
        shprint(
4✔
421
            sh.cp,
422
            join(python_build_dir, python_lib_name + '.so'),
423
            join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
424
        )
425

426
        info('Renaming .so files to reflect cross-compile')
4✔
427
        self.reduce_object_file_names(join(dirn, 'site-packages'))
4✔
428

429
        return join(dirn, 'site-packages')
4✔
430

431

432
recipe = Python3Recipe()
4✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc