17090290875

Committed 20 Aug 2025 06:20AM UTC coverage: 59.287% (+0.07%) from 59.214%

Build # 17090290875

Build Type

Pull #3180

github

Committed by

web-flow

Commit Message

Merge 7cb4f3f5e into 5330267b5

Pull Request Pull Request #3180: `python`: add `3.13` support

Run Details

1068 of 2403 branches covered (44.44%)

Branch coverage included in aggregate %.

121 of 163 new or added lines in 17 files covered. (74.23%)

12 existing lines in 5 files now uncovered.

5003 of 7837 relevant lines covered (63.84%)

2.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.97

/pythonforandroid/recipes/python3/__init__.py

import glob
import sh
import subprocess

from os import environ, utime
from os.path import dirname, exists, join, isfile
from multiprocessing import cpu_count
import shutil

from packaging.version import Version
from pythonforandroid.logger import info, shprint, warning
from pythonforandroid.recipe import Recipe, TargetPythonRecipe
from pythonforandroid.util import (
    current_directory,
    ensure_dir,
    walk_valid_filens,
    BuildInterruptingException,
)

NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (
    'Target ndk-api is {ndk_api}, '
    'but the python3 recipe supports only {min_ndk_api}+'
)


class Python3Recipe(TargetPythonRecipe):
    '''
    The python3's recipe
    ^^^^^^^^^^^^^^^^^^^^

    The python 3 recipe can be built with some extra python modules, but to do
    so, we need some libraries. By default, we ship the python3 recipe with
    some common libraries, defined in ``depends``. We also support some optional
    libraries, which are less common that the ones defined in ``depends``, so
    we added them as optional dependencies (``opt_depends``).

    Below you have a relationship between the python modules and the recipe
    libraries::

        - _ctypes: you must add the recipe for ``libffi``.
        - _sqlite3: you must add the recipe for ``sqlite3``.
        - _ssl: you must add the recipe for ``openssl``.
        - _bz2: you must add the recipe for ``libbz2`` (optional).
        - _lzma: you must add the recipe for ``liblzma`` (optional).

    .. note:: This recipe can be built only against API 21+.

    .. versionchanged:: 2019.10.06.post0
        - Refactored from deleted class ``python.GuestPythonRecipe`` into here
        - Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
          and :mod:`~pythonforandroid.recipes.liblzma`

    .. versionchanged:: 0.6.0
        Refactored into class
        :class:`~pythonforandroid.python.GuestPythonRecipe`
    '''

    version = '3.11.13'
    _p_version = Version(version)
    url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
    name = 'python3'

    patches = [
        'patches/pyconfig_detection.patch',
        'patches/reproducible-buildinfo.diff',
    ]

    if _p_version.major == 3 and _p_version.minor == 7:
        patches += [
            'patches/py3.7.1_fix-ctypes-util-find-library.patch',
            'patches/py3.7.1_fix-zlib-version.patch',
        ]

    if 8 <= _p_version.minor <= 10:
        patches.append('patches/py3.8.1.patch')

    if _p_version.minor >= 11:
        patches.append('patches/cpython-311-ctypes-find-library.patch')

    if shutil.which('lld') is not None:
        if _p_version.minor == 7:
            patches.append("patches/py3.7.1_fix_cortex_a8.patch")
        elif _p_version.minor >= 8:
            patches.append("patches/py3.8.1_fix_cortex_a8.patch")

    depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
    # those optional depends allow us to build python compression modules:
    #   - _bz2.so
    #   - _lzma.so
    opt_depends = ['libbz2', 'liblzma']
    '''The optional libraries which we would like to get our python linked'''

    configure_args = [
        '--host={android_host}',
        '--build={android_build}',
        '--enable-shared',
        '--enable-ipv6',
        '--enable-loadable-sqlite-extensions',
        '--without-static-libpython',
        '--without-readline',
        '--without-ensurepip',

        # Android prefix
        '--prefix={prefix}',
        '--exec-prefix={exec_prefix}',
        '--enable-loadable-sqlite-extensions',

        # Special cross compile args
        'ac_cv_file__dev_ptmx=yes',
        'ac_cv_file__dev_ptc=no',
        'ac_cv_header_sys_eventfd_h=no',
        'ac_cv_little_endian_double=yes',
        'ac_cv_header_bzlib_h=no',
    ]

    if _p_version.minor >= 11:
        configure_args.extend([
            '--with-build-python={python_host_bin}',
        ])

    '''The configure arguments needed to build the python recipe. Those are
    used in method :meth:`build_arch` (if not overwritten like python3's
    recipe does).
    '''

    MIN_NDK_API = 21
    '''Sets the minimal ndk api number needed to use the recipe.

    .. warning:: This recipe can be built only against API 21+, so it means
        that any class which inherits from class:`GuestPythonRecipe` will have
        this limitation.
    '''

    stdlib_dir_blacklist = {
        '__pycache__',
        'test',
        'tests',
        'lib2to3',
        'ensurepip',
        'idlelib',
        'tkinter',
    }
    '''The directories that we want to omit for our python bundle'''

    stdlib_filen_blacklist = [
        '*.py',
        '*.exe',
        '*.whl',
    ]
    '''The file extensions that we want to blacklist for our python bundle'''

    site_packages_dir_blacklist = {
        '__pycache__',
        'tests'
    }
    '''The directories from site packages dir that we don't want to be included
    in our python bundle.'''

    site_packages_excluded_dir_exceptions = [
        # 'numpy' is excluded here because importing with `import numpy as np`
        # can fail if the `tests` directory inside the numpy package is excluded.
        'numpy',
    ]
    '''Directories from `site_packages_dir_blacklist` will not be excluded
    if the full path contains any of these exceptions.'''

    site_packages_filen_blacklist = [
        '*.py'
    ]
    '''The file extensions from site packages dir that we don't want to be
    included in our python bundle.'''

    compiled_extension = '.pyc'
    '''the default extension for compiled python files.

    .. note:: the default extension for compiled python files has been .pyo for
        python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
        longer used and has been removed in favour of extension .pyc
    '''

    disable_gil = False
    '''python3.13 experimental free-threading build'''

    def __init__(self, *args, **kwargs):
        self._ctx = None
        super().__init__(*args, **kwargs)

    @property
    def _libpython(self):
        '''return the python's library name (with extension)'''
        return 'libpython{link_version}.so'.format(
            link_version=self.link_version
        )

    @property
    def link_version(self):
        '''return the python's library link version e.g. 3.7m, 3.8'''
        major, minor = self.major_minor_version_string.split('.')
        flags = ''
        if major == '3' and int(minor) < 8:
            flags += 'm'
        return '{major}.{minor}{flags}'.format(
            major=major,
            minor=minor,
            flags=flags
        )

    def include_root(self, arch_name):
        return join(self.get_build_dir(arch_name), 'Include')

    def link_root(self, arch_name):
        return join(self.get_build_dir(arch_name), 'android-build')

    def should_build(self, arch):
        return not isfile(join(self.link_root(arch.arch), self._libpython))

    def prebuild_arch(self, arch):
        super().prebuild_arch(arch)
        self.ctx.python_recipe = self

    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
        env = super().get_recipe_env(arch)
        env['HOSTARCH'] = arch.command_prefix

        env['CC'] = arch.get_clang_exe(with_target=True)

        env['PATH'] = (
            '{hostpython_dir}:{old_path}').format(
                hostpython_dir=self.get_recipe(
                    'host' + self.name, self.ctx).get_path_to_python(),
                old_path=env['PATH'])

        env['CFLAGS'] = ' '.join(
            [
                '-fPIC',
                '-DANDROID'
            ]
        )

        env['LDFLAGS'] = env.get('LDFLAGS', '')
        if shutil.which('lld') is not None:
            # Note: The -L. is to fix a bug in python 3.7.
            # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
            env['LDFLAGS'] += ' -L. -fuse-ld=lld'
        else:
            warning('lld not found, linking without it. '
                    'Consider installing lld if linker errors occur.')

        return env

    def set_libs_flags(self, env, arch):
        '''Takes care to properly link libraries with python depending on our
        requirements and the attribute :attr:`opt_depends`.
        '''
        def add_flags(include_flags, link_dirs, link_libs):
            env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
            env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
            env['LIBS'] = env.get('LIBS', '') + link_libs

        info('Activating flags for sqlite3')
        recipe = Recipe.get_recipe('sqlite3', self.ctx)
        add_flags(' -I' + recipe.get_build_dir(arch.arch),
                  ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')

        info('Activating flags for libffi')
        recipe = Recipe.get_recipe('libffi', self.ctx)
        # In order to force the correct linkage for our libffi library, we
        # set the following variable to point where is our libffi.pc file,
        # because the python build system uses pkg-config to configure it.
        env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)
        add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
                  ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
                  ' -lffi')

        info('Activating flags for openssl')
        recipe = Recipe.get_recipe('openssl', self.ctx)
        self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))
        add_flags(recipe.include_flags(arch),
                  recipe.link_dirs_flags(arch), recipe.link_libs_flags())

        for library_name in {'libbz2', 'liblzma'}:
            if library_name in self.ctx.recipe_build_order:
                info(f'Activating flags for {library_name}')
                recipe = Recipe.get_recipe(library_name, self.ctx)
                add_flags(recipe.get_library_includes(arch),
                          recipe.get_library_ldflags(arch),
                          recipe.get_library_libs_flag())

        # python build system contains hardcoded zlib version which prevents
        # the build of zlib module, here we search for android's zlib version
        # and sets the right flags, so python can be build with android's zlib
        info("Activating flags for android's zlib")
        zlib_lib_path = arch.ndk_lib_dir_versioned
        zlib_includes = self.ctx.ndk.sysroot_include_dir
        zlib_h = join(zlib_includes, 'zlib.h')
        try:
            with open(zlib_h) as fileh:
                zlib_data = fileh.read()
        except IOError:
            raise BuildInterruptingException(
                "Could not determine android's zlib version, no zlib.h ({}) in"
                " the NDK dir includes".format(zlib_h)
            )
        for line in zlib_data.split('\n'):
            if line.startswith('#define ZLIB_VERSION '):
                break
        else:
            raise BuildInterruptingException(
                'Could not parse zlib.h...so we cannot find zlib version,'
                'required by python build,'
            )
        env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
        add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')

        if self._p_version.minor >= 13 and self.disable_gil:
            self.configure_args.append("--disable-gil")

        return env

    def build_arch(self, arch):
        if self.ctx.ndk_api < self.MIN_NDK_API:
            raise BuildInterruptingException(
                NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
                    ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
                ),
            )

        recipe_build_dir = self.get_build_dir(arch.arch)

        # Create a subdirectory to actually perform the build
        build_dir = join(recipe_build_dir, 'android-build')
        ensure_dir(build_dir)

        # TODO: Get these dynamically, like bpo-30386 does
        sys_prefix = '/usr/local'
        sys_exec_prefix = '/usr/local'

        env = self.get_recipe_env(arch)
        env = self.set_libs_flags(env, arch)

        android_build = sh.Command(
            join(recipe_build_dir,
                 'config.guess'))().strip()

        # disable blake2
        self.configure_args.append("--with-builtin-hashlib-hashes=md5,sha1,sha2,sha3")

        with current_directory(build_dir):
            if not exists('config.status'):
                shprint(
                    sh.Command(join(recipe_build_dir, 'configure')),
                    *(' '.join(self.configure_args).format(
                                    android_host=env['HOSTARCH'],
                                    android_build=android_build,
                                    python_host_bin=join(self.get_recipe(
                                        'host' + self.name, self.ctx
                                    ).get_path_to_python(), "python3"),
                                    prefix=sys_prefix,
                                    exec_prefix=sys_exec_prefix)).split(' '),
                    _env=env)

            shprint(
                sh.make,
                '-j',
                str(cpu_count()),
                'all',
                'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
                _env=env
            )

            # TODO: Look into passing the path to pyconfig.h in a
            # better way, although this is probably acceptable
            sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))

    def compile_python_files(self, dir):
        '''
        Compile the python files (recursively) for the python files inside
        a given folder.

        .. note:: python2 compiles the files into extension .pyo, but in
            python3, and as of Python 3.5, the .pyo filename extension is no
            longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
        '''
        args = [self.ctx.hostpython]
        args += ['-OO', '-m', 'compileall', '-b', '-f', dir]
        subprocess.call(args)

    def create_python_bundle(self, dirn, arch):
        """
        Create a packaged python bundle in the target directory, by
        copying all the modules and standard library to the right
        place.
        """
        # Todo: find a better way to find the build libs folder
        modules_build_dir = join(
            self.get_build_dir(arch.arch),
            'android-build',
            'build',
            'lib.{}{}-{}-{}'.format(
                # android is now supported platform
                "android" if self._p_version.minor >= 13 else "linux",
                '2' if self.version[0] == '2' else '',
                arch.command_prefix.split('-')[0],
                self.major_minor_version_string
                ))

        # Compile to *.pyc the python modules
        self.compile_python_files(modules_build_dir)
        # Compile to *.pyc the standard python library
        self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
        # Compile to *.pyc the other python packages (site-packages)
        self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))

        # Bundle compiled python modules to a folder
        modules_dir = join(dirn, 'modules')
        c_ext = self.compiled_extension
        ensure_dir(modules_dir)
        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
                         glob.glob(join(modules_build_dir, '*' + c_ext)))
        info("Copy {} files into the bundle".format(len(module_filens)))
        for filen in module_filens:
            info(" - copy {}".format(filen))
            shutil.copy2(filen, modules_dir)

        # zip up the standard library
        stdlib_zip = join(dirn, 'stdlib.zip')
        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
            stdlib_filens = list(walk_valid_filens(
                '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
            if 'SOURCE_DATE_EPOCH' in environ:
                # for reproducible builds
                stdlib_filens.sort()
                timestamp = int(environ['SOURCE_DATE_EPOCH'])
                for filen in stdlib_filens:
                    utime(filen, (timestamp, timestamp))
            info("Zip {} files into the bundle".format(len(stdlib_filens)))
            shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)

        # copy the site-packages into place
        ensure_dir(join(dirn, 'site-packages'))
        ensure_dir(self.ctx.get_python_install_dir(arch.arch))
        # TODO: Improve the API around walking and copying the files
        with current_directory(self.ctx.get_python_install_dir(arch.arch)):
            filens = list(walk_valid_filens(
                '.', self.site_packages_dir_blacklist,
                self.site_packages_filen_blacklist,
                excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
            info("Copy {} files into the site-packages".format(len(filens)))
            for filen in filens:
                info(" - copy {}".format(filen))
                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
                shutil.copy2(filen, join(dirn, 'site-packages', filen))

        # copy the python .so files into place
        python_build_dir = join(self.get_build_dir(arch.arch),
                                'android-build')
        python_lib_name = 'libpython' + self.link_version
        shprint(
            sh.cp,
            join(python_build_dir, python_lib_name + '.so'),
            join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
        )

        info('Renaming .so files to reflect cross-compile')
        self.reduce_object_file_names(join(dirn, 'site-packages'))

        return join(dirn, 'site-packages')


recipe = Python3Recipe()

1	import glob	4✔
2	import sh	4✔
3	import subprocess	4✔
4
5	from os import environ, utime	4✔
6	from os.path import dirname, exists, join, isfile	4✔
7	from multiprocessing import cpu_count	4✔
8	import shutil	4✔
9
10	from packaging.version import Version	4✔
11	from pythonforandroid.logger import info, shprint, warning	4✔
12	from pythonforandroid.recipe import Recipe, TargetPythonRecipe	4✔
13	from pythonforandroid.util import (	4✔
14	current_directory,
15	ensure_dir,
16	walk_valid_filens,
17	BuildInterruptingException,
18	)
19
20	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (	4✔
21	'Target ndk-api is {ndk_api}, '
22	'but the python3 recipe supports only {min_ndk_api}+'
23	)
24
25
26	class Python3Recipe(TargetPythonRecipe):	4✔
27	'''
28	The python3's recipe
29	^^^^^^^^^^^^^^^^^^^^
30
31	The python 3 recipe can be built with some extra python modules, but to do
32	so, we need some libraries. By default, we ship the python3 recipe with
33	some common libraries, defined in ``depends``. We also support some optional
34	libraries, which are less common that the ones defined in ``depends``, so
35	we added them as optional dependencies (``opt_depends``).
36
37	Below you have a relationship between the python modules and the recipe
38	libraries::
39
40	- _ctypes: you must add the recipe for ``libffi``.
41	- _sqlite3: you must add the recipe for ``sqlite3``.
42	- _ssl: you must add the recipe for ``openssl``.
43	- _bz2: you must add the recipe for ``libbz2`` (optional).
44	- _lzma: you must add the recipe for ``liblzma`` (optional).
45
46	.. note:: This recipe can be built only against API 21+.
47
48	.. versionchanged:: 2019.10.06.post0
49	- Refactored from deleted class ``python.GuestPythonRecipe`` into here
50	- Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
51	and :mod:`~pythonforandroid.recipes.liblzma`
52
53	.. versionchanged:: 0.6.0
54	Refactored into class
55	:class:`~pythonforandroid.python.GuestPythonRecipe`
56	'''
57
58	version = '3.11.13'	4✔
59	_p_version = Version(version)	4✔
60	url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'	4✔
61	name = 'python3'	4✔
62
63	patches = [	4✔
64	'patches/pyconfig_detection.patch',
65	'patches/reproducible-buildinfo.diff',
66	]
67
68	if _p_version.major == 3 and _p_version.minor == 7:	4!
UNCOV 69	patches += [	×
70	'patches/py3.7.1_fix-ctypes-util-find-library.patch',
71	'patches/py3.7.1_fix-zlib-version.patch',
72	]
73
74	if 8 <= _p_version.minor <= 10:	4!
NEW 75	patches.append('patches/py3.8.1.patch')	×
76
77	if _p_version.minor >= 11:	4!
78	patches.append('patches/cpython-311-ctypes-find-library.patch')	4✔
79
80	if shutil.which('lld') is not None:	4✔
81	if _p_version.minor == 7:	4!
NEW 82	patches.append("patches/py3.7.1_fix_cortex_a8.patch")	×
83	elif _p_version.minor >= 8:	4!
84	patches.append("patches/py3.8.1_fix_cortex_a8.patch")	4✔
85
86	depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']	4✔
87	# those optional depends allow us to build python compression modules:
88	# - _bz2.so
89	# - _lzma.so
90	opt_depends = ['libbz2', 'liblzma']	4✔
91	'''The optional libraries which we would like to get our python linked'''	2✔
92
93	configure_args = [	4✔
94	'--host={android_host}',
95	'--build={android_build}',
96	'--enable-shared',
97	'--enable-ipv6',
98	'--enable-loadable-sqlite-extensions',
99	'--without-static-libpython',
100	'--without-readline',
101	'--without-ensurepip',
102
103	# Android prefix
104	'--prefix={prefix}',
105	'--exec-prefix={exec_prefix}',
106	'--enable-loadable-sqlite-extensions',
107
108	# Special cross compile args
109	'ac_cv_file__dev_ptmx=yes',
110	'ac_cv_file__dev_ptc=no',
111	'ac_cv_header_sys_eventfd_h=no',
112	'ac_cv_little_endian_double=yes',
113	'ac_cv_header_bzlib_h=no',
114	]
115
116	if _p_version.minor >= 11:	4!
117	configure_args.extend([	4✔
118	'--with-build-python={python_host_bin}',
119	])
120
121	'''The configure arguments needed to build the python recipe. Those are	2✔
122	used in method :meth:`build_arch` (if not overwritten like python3's
123	recipe does).
124	'''
125
126	MIN_NDK_API = 21	4✔
127	'''Sets the minimal ndk api number needed to use the recipe.	2✔
128
129	.. warning:: This recipe can be built only against API 21+, so it means
130	that any class which inherits from class:`GuestPythonRecipe` will have
131	this limitation.
132	'''
133
134	stdlib_dir_blacklist = {	4✔
135	'__pycache__',
136	'test',
137	'tests',
138	'lib2to3',
139	'ensurepip',
140	'idlelib',
141	'tkinter',
142	}
143	'''The directories that we want to omit for our python bundle'''	2✔
144
145	stdlib_filen_blacklist = [	4✔
146	'*.py',
147	'*.exe',
148	'*.whl',
149	]
150	'''The file extensions that we want to blacklist for our python bundle'''	2✔
151
152	site_packages_dir_blacklist = {	4✔
153	'__pycache__',
154	'tests'
155	}
156	'''The directories from site packages dir that we don't want to be included	2✔
157	in our python bundle.'''
158
159	site_packages_excluded_dir_exceptions = [	4✔
160	# 'numpy' is excluded here because importing with `import numpy as np`
161	# can fail if the `tests` directory inside the numpy package is excluded.
162	'numpy',
163	]
164	'''Directories from `site_packages_dir_blacklist` will not be excluded	2✔
165	if the full path contains any of these exceptions.'''
166
167	site_packages_filen_blacklist = [	4✔
168	'*.py'
169	]
170	'''The file extensions from site packages dir that we don't want to be	2✔
171	included in our python bundle.'''
172
173	compiled_extension = '.pyc'	4✔
174	'''the default extension for compiled python files.	2✔
175
176	.. note:: the default extension for compiled python files has been .pyo for
177	python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
178	longer used and has been removed in favour of extension .pyc
179	'''
180
181	disable_gil = False	4✔
182	'''python3.13 experimental free-threading build'''	2✔
183
184	def __init__(self, args, *kwargs):	4✔
185	self._ctx = None	4✔
186	super().__init__(args, *kwargs)	4✔
187
188	@property	4✔
189	def _libpython(self):	4✔
190	'''return the python's library name (with extension)'''
191	return 'libpython{link_version}.so'.format(	4✔
192	link_version=self.link_version
193	)
194
195	@property	4✔
196	def link_version(self):	4✔
197	'''return the python's library link version e.g. 3.7m, 3.8'''
198	major, minor = self.major_minor_version_string.split('.')	4✔
199	flags = ''	4✔
200	if major == '3' and int(minor) < 8:	4!
201	flags += 'm'	×
202	return '{major}.{minor}{flags}'.format(	4✔
203	major=major,
204	minor=minor,
205	flags=flags
206	)
207
208	def include_root(self, arch_name):	4✔
209	return join(self.get_build_dir(arch_name), 'Include')	4✔
210
211	def link_root(self, arch_name):	4✔
212	return join(self.get_build_dir(arch_name), 'android-build')	4✔
213
214	def should_build(self, arch):	4✔
NEW 215	return not isfile(join(self.link_root(arch.arch), self._libpython))	×
216
217	def prebuild_arch(self, arch):	4✔
218	super().prebuild_arch(arch)	×
219	self.ctx.python_recipe = self	×
220
221	def get_recipe_env(self, arch=None, with_flags_in_cc=True):	4✔
222	env = super().get_recipe_env(arch)	4✔
223	env['HOSTARCH'] = arch.command_prefix	4✔
224
225	env['CC'] = arch.get_clang_exe(with_target=True)	4✔
226
227	env['PATH'] = (	4✔
228	'{hostpython_dir}:{old_path}').format(
229	hostpython_dir=self.get_recipe(
230	'host' + self.name, self.ctx).get_path_to_python(),
231	old_path=env['PATH'])
232
233	env['CFLAGS'] = ' '.join(	4✔
234	[
235	'-fPIC',
236	'-DANDROID'
237	]
238	)
239
240	env['LDFLAGS'] = env.get('LDFLAGS', '')	4✔
241	if shutil.which('lld') is not None:	4!
242	# Note: The -L. is to fix a bug in python 3.7.
243	# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
244	env['LDFLAGS'] += ' -L. -fuse-ld=lld'	4✔
245	else:
246	warning('lld not found, linking without it. '	×
247	'Consider installing lld if linker errors occur.')
248
249	return env	4✔
250
251	def set_libs_flags(self, env, arch):	4✔
252	'''Takes care to properly link libraries with python depending on our
253	requirements and the attribute :attr:`opt_depends`.
254	'''
255	def add_flags(include_flags, link_dirs, link_libs):	4✔
256	env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags	4✔
257	env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs	4✔
258	env['LIBS'] = env.get('LIBS', '') + link_libs	4✔
259
260	info('Activating flags for sqlite3')	4✔
261	recipe = Recipe.get_recipe('sqlite3', self.ctx)	4✔
262	add_flags(' -I' + recipe.get_build_dir(arch.arch),	4✔
263	' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')
264
265	info('Activating flags for libffi')	4✔
266	recipe = Recipe.get_recipe('libffi', self.ctx)	4✔
267	# In order to force the correct linkage for our libffi library, we
268	# set the following variable to point where is our libffi.pc file,
269	# because the python build system uses pkg-config to configure it.
270	env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)	4✔
271	add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),	4✔
272	' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
273	' -lffi')
274
275	info('Activating flags for openssl')	4✔
276	recipe = Recipe.get_recipe('openssl', self.ctx)	4✔
277	self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))	4✔
278	add_flags(recipe.include_flags(arch),	4✔
279	recipe.link_dirs_flags(arch), recipe.link_libs_flags())
280
281	for library_name in {'libbz2', 'liblzma'}:	4✔
282	if library_name in self.ctx.recipe_build_order:	4!
283	info(f'Activating flags for {library_name}')	×
284	recipe = Recipe.get_recipe(library_name, self.ctx)	×
285	add_flags(recipe.get_library_includes(arch),	×
286	recipe.get_library_ldflags(arch),
287	recipe.get_library_libs_flag())
288
289	# python build system contains hardcoded zlib version which prevents
290	# the build of zlib module, here we search for android's zlib version
291	# and sets the right flags, so python can be build with android's zlib
292	info("Activating flags for android's zlib")	4✔
293	zlib_lib_path = arch.ndk_lib_dir_versioned	4✔
294	zlib_includes = self.ctx.ndk.sysroot_include_dir	4✔
295	zlib_h = join(zlib_includes, 'zlib.h')	4✔
296	try:	4✔
297	with open(zlib_h) as fileh:	4✔
298	zlib_data = fileh.read()	4✔
299	except IOError:	×
300	raise BuildInterruptingException(	×
301	"Could not determine android's zlib version, no zlib.h ({}) in"
302	" the NDK dir includes".format(zlib_h)
303	)
304	for line in zlib_data.split('\n'):	4!
305	if line.startswith('#define ZLIB_VERSION '):	4!
306	break	4✔
307	else:
308	raise BuildInterruptingException(	×
309	'Could not parse zlib.h...so we cannot find zlib version,'
310	'required by python build,'
311	)
312	env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')	4✔
313	add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')	4✔
314
315	if self._p_version.minor >= 13 and self.disable_gil:	4!
NEW 316	self.configure_args.append("--disable-gil")	×
317
318	return env	4✔
319
320	def build_arch(self, arch):	4✔
321	if self.ctx.ndk_api < self.MIN_NDK_API:	4✔
322	raise BuildInterruptingException(	4✔
323	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
324	ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
325	),
326	)
327
328	recipe_build_dir = self.get_build_dir(arch.arch)	4✔
329
330	# Create a subdirectory to actually perform the build
331	build_dir = join(recipe_build_dir, 'android-build')	4✔
332	ensure_dir(build_dir)	4✔
333
334	# TODO: Get these dynamically, like bpo-30386 does
335	sys_prefix = '/usr/local'	4✔
336	sys_exec_prefix = '/usr/local'	4✔
337
338	env = self.get_recipe_env(arch)	4✔
339	env = self.set_libs_flags(env, arch)	4✔
340
341	android_build = sh.Command(	4✔
342	join(recipe_build_dir,
343	'config.guess'))().strip()
344
345	# disable blake2
346	self.configure_args.append("--with-builtin-hashlib-hashes=md5,sha1,sha2,sha3")	4✔
347
348	with current_directory(build_dir):	4✔
349	if not exists('config.status'):	4!
350	shprint(	4✔
351	sh.Command(join(recipe_build_dir, 'configure')),
352	*(' '.join(self.configure_args).format(
353	android_host=env['HOSTARCH'],
354	android_build=android_build,
355	python_host_bin=join(self.get_recipe(
356	'host' + self.name, self.ctx
357	).get_path_to_python(), "python3"),
358	prefix=sys_prefix,
359	exec_prefix=sys_exec_prefix)).split(' '),
360	_env=env)
361
362	shprint(	4✔
363	sh.make,
364	'-j',
365	str(cpu_count()),
366	'all',
367	'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
368	_env=env
369	)
370
371	# TODO: Look into passing the path to pyconfig.h in a
372	# better way, although this is probably acceptable
373	sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))	4✔
374
375	def compile_python_files(self, dir):	4✔
376	'''
377	Compile the python files (recursively) for the python files inside
378	a given folder.
379
380	.. note:: python2 compiles the files into extension .pyo, but in
381	python3, and as of Python 3.5, the .pyo filename extension is no
382	longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
383	'''
384	args = [self.ctx.hostpython]	4✔
385	args += ['-OO', '-m', 'compileall', '-b', '-f', dir]	4✔
386	subprocess.call(args)	4✔
387
388	def create_python_bundle(self, dirn, arch):	4✔
389	"""
390	Create a packaged python bundle in the target directory, by
391	copying all the modules and standard library to the right
392	place.
393	"""
394	# Todo: find a better way to find the build libs folder
395	modules_build_dir = join(	4✔
396	self.get_build_dir(arch.arch),
397	'android-build',
398	'build',
399	'lib.{}{}-{}-{}'.format(
400	# android is now supported platform
401	"android" if self._p_version.minor >= 13 else "linux",
402	'2' if self.version[0] == '2' else '',
403	arch.command_prefix.split('-')[0],
404	self.major_minor_version_string
405	))
406
407	# Compile to *.pyc the python modules
408	self.compile_python_files(modules_build_dir)	4✔
409	# Compile to *.pyc the standard python library
410	self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))	4✔
411	# Compile to *.pyc the other python packages (site-packages)
412	self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))	4✔
413
414	# Bundle compiled python modules to a folder
415	modules_dir = join(dirn, 'modules')	4✔
416	c_ext = self.compiled_extension	4✔
417	ensure_dir(modules_dir)	4✔
418	module_filens = (glob.glob(join(modules_build_dir, '*.so')) +	4✔
419	glob.glob(join(modules_build_dir, '*' + c_ext)))
420	info("Copy {} files into the bundle".format(len(module_filens)))	4✔
421	for filen in module_filens:	4!
422	info(" - copy {}".format(filen))	×
423	shutil.copy2(filen, modules_dir)	×
424
425	# zip up the standard library
426	stdlib_zip = join(dirn, 'stdlib.zip')	4✔
427	with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):	4✔
428	stdlib_filens = list(walk_valid_filens(	4✔
429	'.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
430	if 'SOURCE_DATE_EPOCH' in environ:	4!
431	# for reproducible builds
432	stdlib_filens.sort()	×
433	timestamp = int(environ['SOURCE_DATE_EPOCH'])	×
434	for filen in stdlib_filens:	×
435	utime(filen, (timestamp, timestamp))	×
436	info("Zip {} files into the bundle".format(len(stdlib_filens)))	4✔
437	shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)	4✔
438
439	# copy the site-packages into place
440	ensure_dir(join(dirn, 'site-packages'))	4✔
441	ensure_dir(self.ctx.get_python_install_dir(arch.arch))	4✔
442	# TODO: Improve the API around walking and copying the files
443	with current_directory(self.ctx.get_python_install_dir(arch.arch)):	4✔
444	filens = list(walk_valid_filens(	4✔
445	'.', self.site_packages_dir_blacklist,
446	self.site_packages_filen_blacklist,
447	excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
448	info("Copy {} files into the site-packages".format(len(filens)))	4✔
449	for filen in filens:	4✔
450	info(" - copy {}".format(filen))	4✔
451	ensure_dir(join(dirn, 'site-packages', dirname(filen)))	4✔
452	shutil.copy2(filen, join(dirn, 'site-packages', filen))	4✔
453
454	# copy the python .so files into place
455	python_build_dir = join(self.get_build_dir(arch.arch),	4✔
456	'android-build')
457	python_lib_name = 'libpython' + self.link_version	4✔
458	shprint(	4✔
459	sh.cp,
460	join(python_build_dir, python_lib_name + '.so'),
461	join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
462	)
463
464	info('Renaming .so files to reflect cross-compile')	4✔
465	self.reduce_object_file_names(join(dirn, 'site-packages'))	4✔
466
467	return join(dirn, 'site-packages')	4✔
468
469
470	recipe = Python3Recipe()	4✔

kivy / python-for-android / 17090290875

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous