36313254676

Committed 27 Sep 2026 10:39AM UTC coverage: 63.064% (+0.02%) from 63.041%

Build # 36313254676

Build Type

Pull #3383

github

Committed by

web-flow

Commit Message

Merge 5857040ae into e772ad93f

Pull Request Pull Request #3383: Attempt on optimizing the builds

Coverage Stats

1829 of 3170 branches covered (57.7%)

Branch coverage included in aggregate %.

8 of 8 new or added lines in 2 files covered. (100.0%)

34 existing lines in 1 file now uncovered.

5424 of 8331 relevant lines covered (65.11%)

3.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.45

/pythonforandroid/recipes/python3/__init__.py

import glob
import sh
import subprocess

from os import environ, utime
from os.path import dirname, exists, join, isfile
import shutil

from packaging.version import Version
from pythonforandroid.logger import info, shprint, warning
from pythonforandroid.recipe import Recipe, TargetPythonRecipe
from pythonforandroid.util import (
    current_directory,
    ensure_dir,
    walk_valid_filens,
    BuildInterruptingException,
)

NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (
    'Target ndk-api is {ndk_api}, '
    'but the python3 recipe supports only {min_ndk_api}+'
)


class Python3Recipe(TargetPythonRecipe):
    '''
    The python3's recipe
    ^^^^^^^^^^^^^^^^^^^^

    The python 3 recipe can be built with some extra python modules, but to do
    so, we need some libraries. By default, we ship the python3 recipe with
    some common libraries, defined in ``depends``. We also support some optional
    libraries, which are less common that the ones defined in ``depends``, so
    we added them as optional dependencies (``opt_depends``).

    Below you have a relationship between the python modules and the recipe
    libraries::

        - _ctypes: you must add the recipe for ``libffi``.
        - _sqlite3: you must add the recipe for ``sqlite3``.
        - _ssl: you must add the recipe for ``openssl``.
        - _bz2: you must add the recipe for ``libbz2`` (optional).
        - _lzma: you must add the recipe for ``liblzma`` (optional).

    .. note:: This recipe can be built only against API 21+.

    .. versionchanged:: 2019.10.06.post0
        - Refactored from deleted class ``python.GuestPythonRecipe`` into here
        - Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
          and :mod:`~pythonforandroid.recipes.liblzma`

    .. versionchanged:: 0.6.0
        Refactored into class
        :class:`~pythonforandroid.python.GuestPythonRecipe`
    '''

    version = '3.14.2'
    url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
    name = 'python3'

    patches = [
        'patches/pyconfig_detection.patch',
        'patches/reproducible-buildinfo.diff',
    ]

    depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
    # those optional depends allow us to build python compression modules:
    #   - _bz2.so
    #   - _lzma.so
    opt_depends = ['libbz2', 'liblzma']
    '''The optional libraries which we would like to get our python linked'''

    configure_args = [
        '--host={android_host}',
        '--build={android_build}',
        '--enable-ipv6',
        '--enable-loadable-sqlite-extensions',
        '--enable-shared',

        # Attempt on making the builds lighter
        '--disable-test-modules',
        '--without-c-locale-coercion',
        '--without-doc-strings',
        '--without-ensurepip',
        '--without-readline',
        '--without-static-libpython',

        # Android prefix
        '--prefix={prefix}',

        # Special cross compile args
        'ac_cv_file__dev_ptc=no',
        'ac_cv_file__dev_ptmx=yes',
        'ac_cv_header_bzlib_h=no',
        'ac_cv_header_sys_eventfd_h=no',
        'py_cv_module__curses=n/a',
        'py_cv_module__curses_panel=n/a',
        'py_cv_module__tkinter=n/a'
    ]

    '''The configure arguments needed to build the python recipe. Those are
    used in method :meth:`build_arch` (if not overwritten like python3's
    recipe does).
    '''

    MIN_NDK_API = 21
    '''Sets the minimal ndk api number needed to use the recipe.

    .. warning:: Starting from Python 3.14 this recipe can only be built
       against API 21+, so it means that any class which inherits from
       class:`GuestPythonRecipe` will have this limitation.
    '''

    stdlib_dir_blacklist = {
        '__pycache__',
        'curses',
        'ensurepip',
        'idlelib',
        'lib2to3',
        'pydoc_data',
        'test',
        'tests',
        'tkinter',
        'turtledemo',
        'venv'
    }
    '''The directories that we want to omit for our python bundle'''

    stdlib_filen_blacklist = [
        '*.exe',
        '*.py',
        '*.whl',
        'turtle.pyc'
    ]
    '''The file extensions that we want to blacklist for our python bundle'''

    site_packages_dir_blacklist = {
        '*.dist-info',
        '__pycache__',
        '_distutils_hack',
        'bin',
        'setuptools',
        'tests'
    }
    '''The directories from site packages dir that we don't want to be included
    in our python bundle.'''

    site_packages_excluded_dir_exceptions = [
        # 'numpy' is excluded here because importing with `import numpy as np`
        # can fail if the `tests` directory inside the numpy package is excluded.
        'numpy',
    ]
    '''Directories from `site_packages_dir_blacklist` will not be excluded
    if the full path contains any of these exceptions.'''

    site_packages_filen_blacklist = [
        '*.py',
        '*.pyx'
    ]
    '''The file extensions from site packages dir that we don't want to be
    included in our python bundle.'''

    compiled_extension = '.pyc'
    '''the default extension for compiled python files.

    .. note:: the default extension for compiled python files has been .pyo for
        python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
        longer used and has been removed in favour of extension .pyc
    '''

    disable_gil = False
    '''python3.13 experimental free-threading build'''

    built_libraries = {"libpythonbin.so": "./android-build/"}

    def __init__(self, *args, **kwargs):
        self._ctx = None
        super().__init__(*args, **kwargs)

    @property
    def _libpython(self):
        '''return the python's library name (with extension)'''
        return 'libpython{link_version}.so'.format(
            link_version=self.link_version
        )

    @property
    def link_version(self):
        '''return the python's library link version e.g. 3.7m, 3.8'''
        major, minor = self.major_minor_version_string.split('.')
        flags = ''
        if major == '3' and int(minor) < 8:
            flags += 'm'
        return '{major}.{minor}{flags}'.format(
            major=major,
            minor=minor,
            flags=flags
        )

    def apply_patches(self, arch, build_dir=None):

        _p_version = Version(self.version)
        if _p_version.major == 3 and _p_version.minor == 7:
            self.patches += [
                'patches/py3.7.1_fix-ctypes-util-find-library.patch',
                'patches/py3.7.1_fix-zlib-version.patch',
            ]

        if 8 <= _p_version.minor <= 10:
            self.patches.append('patches/py3.8.1.patch')

        if _p_version.minor >= 11:
            self.patches.append('patches/cpython-311-ctypes-find-library.patch')

        if _p_version.minor >= 14:
            self.patches.append('patches/3.14_armv7l_fix.patch')
            self.patches.append('patches/3.14_fix_remote_debug.patch')

        if shutil.which('lld') is not None:
            if _p_version.minor == 7:
                self.patches.append("patches/py3.7.1_fix_cortex_a8.patch")
            elif _p_version.minor >= 8:
                self.patches.append("patches/py3.8.1_fix_cortex_a8.patch")

        self.patches = list(dict.fromkeys(self.patches))  # preserve order for reproducibility
        super().apply_patches(arch, build_dir)

    def include_root(self, arch_name):
        _p_version = Version(self.version)
        return join(
            self.get_build_dir(arch_name), 'android-build', 'android-root',
            'include', f'python{_p_version.major}.{_p_version.minor}'
        )

    def link_root(self, arch_name):
        return join(self.get_build_dir(arch_name), 'android-build')

    def get_python_root(self, arch):
        return join(self.get_build_dir(arch.arch), 'android-build', 'android-root')

    def get_android_python_exe(self, arch):
        return join(self.get_python_root(arch), 'bin', self.name)

    def should_build(self, arch):
        return not isfile(join(self.link_root(arch.arch), self._libpython))

    def prebuild_arch(self, arch):
        super().prebuild_arch(arch)
        self.ctx.python_recipe = self

    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
        env = super().get_recipe_env(arch)
        env['HOSTARCH'] = arch.command_prefix
        env['CC'] = arch.get_clang_exe(with_target=True)
        env['PATH'] = '{hostpython_dir}:{old_path}'.format(
            hostpython_dir=self.get_recipe(
                'host' + self.name, self.ctx
            ).get_path_to_python(),
            old_path=env['PATH']
        )
        env['CFLAGS'] = ' '.join(
            [
                '-ffunction-sections',
                '-fdata-sections',
                '-fPIC'
            ]
        )

        env['LDFLAGS'] = env.get('LDFLAGS', '')
        if shutil.which('lld') is not None:
            # Note: The -L. is to fix a bug in python 3.7.
            # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
            env['LDFLAGS'] += ' -L. -fuse-ld=lld'
        else:
            warning('lld not found, linking without it. '
                    'Consider installing lld if linker errors occur.')

        return env

    def set_libs_flags(self, env, arch):
        '''Takes care to properly link libraries with python depending on our
        requirements and the attribute :attr:`opt_depends`.
        '''
        def add_flags(include_flags, link_dirs, link_libs):
            env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
            env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
            env['LIBS'] = env.get('LIBS', '') + link_libs

        info('Activating flags for sqlite3')
        recipe = Recipe.get_recipe('sqlite3', self.ctx)
        add_flags(' -I' + recipe.get_build_dir(arch.arch),
                  ' -L' + recipe.get_build_dir(arch.arch), ' -lsqlite3')

        info('Activating flags for libffi')
        recipe = Recipe.get_recipe('libffi', self.ctx)
        # In order to force the correct linkage for our libffi library, we
        # set the following variable to point where is our libffi.pc file,
        # because the python build system uses pkg-config to configure it.
        env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)
        add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
                  ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
                  ' -lffi')

        info('Activating flags for openssl')
        recipe = Recipe.get_recipe('openssl', self.ctx)
        self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))
        add_flags(recipe.include_flags(arch),
                  recipe.link_dirs_flags(arch), recipe.link_libs_flags())

        for library_name in {'libbz2', 'liblzma'}:
            if library_name in self.ctx.recipe_build_order:
                info(f'Activating flags for {library_name}')
                recipe = Recipe.get_recipe(library_name, self.ctx)
                add_flags(recipe.get_library_includes(arch),
                          recipe.get_library_ldflags(arch),
                          recipe.get_library_libs_flag())

        # python build system contains hardcoded zlib version which prevents
        # the build of zlib module, here we search for android's zlib version
        # and sets the right flags, so python can be build with android's zlib
        info("Activating flags for android's zlib")
        zlib_lib_path = arch.ndk_lib_dir_versioned
        zlib_includes = self.ctx.ndk.sysroot_include_dir
        zlib_h = join(zlib_includes, 'zlib.h')
        try:
            with open(zlib_h) as fileh:
                zlib_data = fileh.read()
        except IOError:
            raise BuildInterruptingException(
                "Could not determine android's zlib version, no zlib.h ({}) in"
                " the NDK dir includes".format(zlib_h)
            )
        for line in zlib_data.split('\n'):
            if line.startswith('#define ZLIB_VERSION '):
                break
        else:
            raise BuildInterruptingException(
                'Could not parse zlib.h...so we cannot find zlib version,'
                'required by python build,'
            )
        env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
        add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')

        _p_version = Version(self.version)
        if _p_version.minor >= 11:
            self.configure_args.append('--with-build-python={python_host_bin}')

        if _p_version.minor >= 13 and self.disable_gil:
            self.configure_args.append("--disable-gil")

        self.configure_args = list(dict.fromkeys(self.configure_args))  # preserve order for reproducibility

        return env

    def build_arch(self, arch):
        if self.ctx.ndk_api < self.MIN_NDK_API:
            raise BuildInterruptingException(
                NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
                    ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
                ),
            )

        recipe_build_dir = self.get_build_dir(arch.arch)

        # Create a subdirectory to actually perform the build
        build_dir = join(recipe_build_dir, 'android-build')
        ensure_dir(build_dir)

        sys_prefix = join(build_dir, "android-root")
        ensure_dir(sys_prefix)

        env = self.get_recipe_env(arch)
        env = self.set_libs_flags(env, arch)

        android_build = sh.Command(
            join(recipe_build_dir,
                 'config.guess'))().strip()

        with current_directory(build_dir):
            if not exists('config.status'):
                shprint(
                    sh.Command(join(recipe_build_dir, 'configure')),
                    *(' '.join(self.configure_args).format(
                                    android_host=env['HOSTARCH'],
                                    android_build=android_build,
                                    python_host_bin=self.get_recipe(
                                        'host' + self.name, self.ctx
                                    ).python_exe,
                                    prefix=sys_prefix).split(' ')),
                    _env=env)

            shprint(
                sh.make,
                'all',
                'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
                _env=env
            )
            shprint(sh.make, 'install', _env=env)

            # rename executable
            if isfile("python"):
                sh.cp('python', 'libpythonbin.so')
            elif isfile("python.exe"):  # for macos
                sh.cp('python.exe', 'libpythonbin.so')

            # TODO: Look into passing the path to pyconfig.h in a
            # better way, although this is probably acceptable
            sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))

    def compile_python_files(self, dir):
        '''
        Compile the python files (recursively) for the python files inside
        a given folder.

        .. note:: python2 compiles the files into extension .pyo, but in
            python3, and as of Python 3.5, the .pyo filename extension is no
            longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
        '''
        args = [self.ctx.hostpython]
        args += ['-OO', '-m', 'compileall', '-b', '-f', '-q', dir]
        subprocess.call(args)

    def create_python_bundle(self, dirn, arch):
        """
        Create a packaged python bundle in the target directory, by
        copying all the modules and standard library to the right
        place.
        """
        modules_build_dir = glob.glob(join(
            self.get_build_dir(arch.arch),
            'android-build',
            'build',
            'lib.*'
        ))[0]
        # Compile to *.pyc the python modules
        self.compile_python_files(modules_build_dir)
        # Compile to *.pyc the standard python library
        self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
        # Compile to *.pyc the other python packages (site-packages)
        self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))

        # Bundle compiled python modules to a folder
        modules_dir = join(dirn, 'modules')
        c_ext = self.compiled_extension
        ensure_dir(modules_dir)
        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
                         glob.glob(join(modules_build_dir, '*' + c_ext)))
        info("Copy {} files into the bundle".format(len(module_filens)))
        for filen in module_filens:
            info(" - copy {}".format(filen))
            shutil.copy2(filen, modules_dir)

        # zip up the standard library
        stdlib_zip = join(dirn, 'stdlib.zip')
        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
            stdlib_filens = list(walk_valid_filens(
                '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
            if 'SOURCE_DATE_EPOCH' in environ:
                # for reproducible builds
                stdlib_filens.sort()
                timestamp = int(environ['SOURCE_DATE_EPOCH'])
                for filen in stdlib_filens:
                    utime(filen, (timestamp, timestamp))
            info("Zip {} files into the bundle".format(len(stdlib_filens)))
            shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)

        # copy the site-packages into place
        ensure_dir(join(dirn, 'site-packages'))
        ensure_dir(self.ctx.get_python_install_dir(arch.arch))
        # TODO: Improve the API around walking and copying the files
        with current_directory(self.ctx.get_python_install_dir(arch.arch)):
            filens = list(walk_valid_filens(
                '.', self.site_packages_dir_blacklist,
                self.site_packages_filen_blacklist,
                excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
            info("Copy {} files into the site-packages".format(len(filens)))
            for filen in filens:
                info(" - copy {}".format(filen))
                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
                shutil.copy2(filen, join(dirn, 'site-packages', filen))

        # copy the python .so files into place
        python_build_dir = join(self.get_build_dir(arch.arch),
                                'android-build')
        python_lib_name = 'libpython' + self.link_version
        shprint(
            sh.cp,
            join(python_build_dir, python_lib_name + '.so'),
            join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
        )

        info('Renaming .so files to reflect cross-compile')
        self.reduce_object_file_names(join(dirn, 'site-packages'))

        return join(dirn, 'site-packages')


recipe = Python3Recipe()

1	import glob	6 ✔
2	import sh	6 ✔
3	import subprocess	6 ✔
4
5	from os import environ, utime	6 ✔
6	from os.path import dirname, exists, join, isfile	6 ✔
7	import shutil	6 ✔
8
9	from packaging.version import Version	6 ✔
10	from pythonforandroid.logger import info, shprint, warning	6 ✔
11	from pythonforandroid.recipe import Recipe, TargetPythonRecipe	6 ✔
12	from pythonforandroid.util import (	6 ✔
13	current_directory,
14	ensure_dir,
15	walk_valid_filens,
16	BuildInterruptingException,
17	)
18
19	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (	6 ✔
20	'Target ndk-api is {ndk_api}, '
21	'but the python3 recipe supports only {min_ndk_api}+'
22	)
23
24
25	class Python3Recipe(TargetPythonRecipe):	6 ✔
26	'''
27	The python3's recipe
28	^^^^^^^^^^^^^^^^^^^^
29
30	The python 3 recipe can be built with some extra python modules, but to do
31	so, we need some libraries. By default, we ship the python3 recipe with
32	some common libraries, defined in ``depends``. We also support some optional
33	libraries, which are less common that the ones defined in ``depends``, so
34	we added them as optional dependencies (``opt_depends``).
35
36	Below you have a relationship between the python modules and the recipe
37	libraries::
38
39	- _ctypes: you must add the recipe for ``libffi``.
40	- _sqlite3: you must add the recipe for ``sqlite3``.
41	- _ssl: you must add the recipe for ``openssl``.
42	- _bz2: you must add the recipe for ``libbz2`` (optional).
43	- _lzma: you must add the recipe for ``liblzma`` (optional).
44
45	.. note:: This recipe can be built only against API 21+.
46
47	.. versionchanged:: 2019.10.06.post0
48	- Refactored from deleted class ``python.GuestPythonRecipe`` into here
49	- Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
50	and :mod:`~pythonforandroid.recipes.liblzma`
51
52	.. versionchanged:: 0.6.0
53	Refactored into class
54	:class:`~pythonforandroid.python.GuestPythonRecipe`
55	'''
56
57	version = '3.14.2'	6 ✔
58	url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'	6 ✔
59	name = 'python3'	6 ✔
60
61	patches = [	6 ✔
62	'patches/pyconfig_detection.patch',
63	'patches/reproducible-buildinfo.diff',
64	]
65
66	depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']	6 ✔
67	# those optional depends allow us to build python compression modules:
68	# - _bz2.so
69	# - _lzma.so
70	opt_depends = ['libbz2', 'liblzma']	6 ✔
71	'''The optional libraries which we would like to get our python linked'''	6 ✔
72
73	configure_args = [	6 ✔
74	'--host={android_host}',
75	'--build={android_build}',
76	'--enable-ipv6',
77	'--enable-loadable-sqlite-extensions',
78	'--enable-shared',
79
80	# Attempt on making the builds lighter
81	'--disable-test-modules',
82	'--without-c-locale-coercion',
83	'--without-doc-strings',
84	'--without-ensurepip',
85	'--without-readline',
86	'--without-static-libpython',
87
88	# Android prefix
89	'--prefix={prefix}',
90
91	# Special cross compile args
92	'ac_cv_file__dev_ptc=no',
93	'ac_cv_file__dev_ptmx=yes',
94	'ac_cv_header_bzlib_h=no',
95	'ac_cv_header_sys_eventfd_h=no',
96	'py_cv_module__curses=n/a',
97	'py_cv_module__curses_panel=n/a',
98	'py_cv_module__tkinter=n/a'
99	]
100
101	'''The configure arguments needed to build the python recipe. Those are	6 ✔
102	used in method :meth:`build_arch` (if not overwritten like python3's
103	recipe does).
104	'''
105
106	MIN_NDK_API = 21	6 ✔
107	'''Sets the minimal ndk api number needed to use the recipe.	6 ✔
108
109	.. warning:: Starting from Python 3.14 this recipe can only be built
110	against API 21+, so it means that any class which inherits from
111	class:`GuestPythonRecipe` will have this limitation.
112	'''
113
114	stdlib_dir_blacklist = {	6 ✔
115	'__pycache__',
116	'curses',
117	'ensurepip',
118	'idlelib',
119	'lib2to3',
120	'pydoc_data',
121	'test',
122	'tests',
123	'tkinter',
124	'turtledemo',
125	'venv'
126	}
127	'''The directories that we want to omit for our python bundle'''	6 ✔
128
129	stdlib_filen_blacklist = [	6 ✔
130	'*.exe',
131	'*.py',
132	'*.whl',
133	'turtle.pyc'
134	]
135	'''The file extensions that we want to blacklist for our python bundle'''	6 ✔
136
137	site_packages_dir_blacklist = {	6 ✔
138	'*.dist-info',
139	'__pycache__',
140	'_distutils_hack',
141	'bin',
142	'setuptools',
143	'tests'
144	}
145	'''The directories from site packages dir that we don't want to be included	6 ✔
146	in our python bundle.'''
147
148	site_packages_excluded_dir_exceptions = [	6 ✔
149	# 'numpy' is excluded here because importing with `import numpy as np`
150	# can fail if the `tests` directory inside the numpy package is excluded.
151	'numpy',
152	]
153	'''Directories from `site_packages_dir_blacklist` will not be excluded	6 ✔
154	if the full path contains any of these exceptions.'''
155
156	site_packages_filen_blacklist = [	6 ✔
157	'*.py',
158	'*.pyx'
159	]
160	'''The file extensions from site packages dir that we don't want to be	6 ✔
161	included in our python bundle.'''
162
163	compiled_extension = '.pyc'	6 ✔
164	'''the default extension for compiled python files.	6 ✔
165
166	.. note:: the default extension for compiled python files has been .pyo for
167	python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
168	longer used and has been removed in favour of extension .pyc
169	'''
170
171	disable_gil = False	6 ✔
172	'''python3.13 experimental free-threading build'''	6 ✔
173
174	built_libraries = {"libpythonbin.so": "./android-build/"}	6 ✔
175
176	def __init__(self, args, *kwargs):	6 ✔
177	self._ctx = None	6 ✔
178	super().__init__(args, *kwargs)	6 ✔
179
180	@property	6 ✔
181	def _libpython(self):	6 ✔
182	'''return the python's library name (with extension)'''
183	return 'libpython{link_version}.so'.format(	6 ✔
184	link_version=self.link_version
185	)
186
187	@property	6 ✔
188	def link_version(self):	6 ✔
189	'''return the python's library link version e.g. 3.7m, 3.8'''
190	major, minor = self.major_minor_version_string.split('.')	6 ✔
191	flags = ''	6 ✔
192	if major == '3' and int(minor) < 8:	6 !
UNCOV 193	flags += 'm'	×
194	return '{major}.{minor}{flags}'.format(	6 ✔
195	major=major,
196	minor=minor,
197	flags=flags
198	)
199
200	def apply_patches(self, arch, build_dir=None):	6 ✔
201
202	_p_version = Version(self.version)	×
203	if _p_version.major == 3 and _p_version.minor == 7:	×
UNCOV 204	self.patches += [	×
205	'patches/py3.7.1_fix-ctypes-util-find-library.patch',
206	'patches/py3.7.1_fix-zlib-version.patch',
207	]
208
209	if 8 <= _p_version.minor <= 10:	×
UNCOV 210	self.patches.append('patches/py3.8.1.patch')	×
211
212	if _p_version.minor >= 11:	×
UNCOV 213	self.patches.append('patches/cpython-311-ctypes-find-library.patch')	×
214
215	if _p_version.minor >= 14:	×
216	self.patches.append('patches/3.14_armv7l_fix.patch')	×
UNCOV 217	self.patches.append('patches/3.14_fix_remote_debug.patch')	×
218
219	if shutil.which('lld') is not None:	×
220	if _p_version.minor == 7:	×
221	self.patches.append("patches/py3.7.1_fix_cortex_a8.patch")	×
222	elif _p_version.minor >= 8:	×
UNCOV 223	self.patches.append("patches/py3.8.1_fix_cortex_a8.patch")	×
224
225	self.patches = list(dict.fromkeys(self.patches)) # preserve order for reproducibility	×
UNCOV 226	super().apply_patches(arch, build_dir)	×
227
228	def include_root(self, arch_name):	6 ✔
229	_p_version = Version(self.version)	6 ✔
230	return join(	6 ✔
231	self.get_build_dir(arch_name), 'android-build', 'android-root',
232	'include', f'python{_p_version.major}.{_p_version.minor}'
233	)
234
235	def link_root(self, arch_name):	6 ✔
236	return join(self.get_build_dir(arch_name), 'android-build')	6 ✔
237
238	def get_python_root(self, arch):	6 ✔
UNCOV 239	return join(self.get_build_dir(arch.arch), 'android-build', 'android-root')	×
240
241	def get_android_python_exe(self, arch):	6 ✔
UNCOV 242	return join(self.get_python_root(arch), 'bin', self.name)	×
243
244	def should_build(self, arch):	6 ✔
UNCOV 245	return not isfile(join(self.link_root(arch.arch), self._libpython))	×
246
247	def prebuild_arch(self, arch):	6 ✔
248	super().prebuild_arch(arch)	×
UNCOV 249	self.ctx.python_recipe = self	×
250
251	def get_recipe_env(self, arch=None, with_flags_in_cc=True):	6 ✔
252	env = super().get_recipe_env(arch)	6 ✔
253	env['HOSTARCH'] = arch.command_prefix	6 ✔
254	env['CC'] = arch.get_clang_exe(with_target=True)	6 ✔
255	env['PATH'] = '{hostpython_dir}:{old_path}'.format(	6 ✔
256	hostpython_dir=self.get_recipe(
257	'host' + self.name, self.ctx
258	).get_path_to_python(),
259	old_path=env['PATH']
260	)
261	env['CFLAGS'] = ' '.join(	6 ✔
262	[
263	'-ffunction-sections',
264	'-fdata-sections',
265	'-fPIC'
266	]
267	)
268
269	env['LDFLAGS'] = env.get('LDFLAGS', '')	6 ✔
270	if shutil.which('lld') is not None:	6 !
271	# Note: The -L. is to fix a bug in python 3.7.
272	# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
273	env['LDFLAGS'] += ' -L. -fuse-ld=lld'	6 ✔
274	else:
UNCOV 275	warning('lld not found, linking without it. '	×
276	'Consider installing lld if linker errors occur.')
277
278	return env	6 ✔
279
280	def set_libs_flags(self, env, arch):	6 ✔
281	'''Takes care to properly link libraries with python depending on our
282	requirements and the attribute :attr:`opt_depends`.
283	'''
284	def add_flags(include_flags, link_dirs, link_libs):	6 ✔
285	env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags	6 ✔
286	env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs	6 ✔
287	env['LIBS'] = env.get('LIBS', '') + link_libs	6 ✔
288
289	info('Activating flags for sqlite3')	6 ✔
290	recipe = Recipe.get_recipe('sqlite3', self.ctx)	6 ✔
291	add_flags(' -I' + recipe.get_build_dir(arch.arch),	6 ✔
292	' -L' + recipe.get_build_dir(arch.arch), ' -lsqlite3')
293
294	info('Activating flags for libffi')	6 ✔
295	recipe = Recipe.get_recipe('libffi', self.ctx)	6 ✔
296	# In order to force the correct linkage for our libffi library, we
297	# set the following variable to point where is our libffi.pc file,
298	# because the python build system uses pkg-config to configure it.
299	env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)	6 ✔
300	add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),	6 ✔
301	' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
302	' -lffi')
303
304	info('Activating flags for openssl')	6 ✔
305	recipe = Recipe.get_recipe('openssl', self.ctx)	6 ✔
306	self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))	6 ✔
307	add_flags(recipe.include_flags(arch),	6 ✔
308	recipe.link_dirs_flags(arch), recipe.link_libs_flags())
309
310	for library_name in {'libbz2', 'liblzma'}:	6 ✔
311	if library_name in self.ctx.recipe_build_order:	6 !
312	info(f'Activating flags for {library_name}')	×
313	recipe = Recipe.get_recipe(library_name, self.ctx)	×
UNCOV 314	add_flags(recipe.get_library_includes(arch),	×
315	recipe.get_library_ldflags(arch),
316	recipe.get_library_libs_flag())
317
318	# python build system contains hardcoded zlib version which prevents
319	# the build of zlib module, here we search for android's zlib version
320	# and sets the right flags, so python can be build with android's zlib
321	info("Activating flags for android's zlib")	6 ✔
322	zlib_lib_path = arch.ndk_lib_dir_versioned	6 ✔
323	zlib_includes = self.ctx.ndk.sysroot_include_dir	6 ✔
324	zlib_h = join(zlib_includes, 'zlib.h')	6 ✔
325	try:	6 ✔
326	with open(zlib_h) as fileh:	6 ✔
327	zlib_data = fileh.read()	6 ✔
328	except IOError:	×
UNCOV 329	raise BuildInterruptingException(	×
330	"Could not determine android's zlib version, no zlib.h ({}) in"
331	" the NDK dir includes".format(zlib_h)
332	)
333	for line in zlib_data.split('\n'):	6 !
334	if line.startswith('#define ZLIB_VERSION '):	6 !
335	break	6 ✔
336	else:
UNCOV 337	raise BuildInterruptingException(	×
338	'Could not parse zlib.h...so we cannot find zlib version,'
339	'required by python build,'
340	)
341	env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')	6 ✔
342	add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')	6 ✔
343
344	_p_version = Version(self.version)	6 ✔
345	if _p_version.minor >= 11:	6 !
346	self.configure_args.append('--with-build-python={python_host_bin}')	6 ✔
347
348	if _p_version.minor >= 13 and self.disable_gil:	6 !
UNCOV 349	self.configure_args.append("--disable-gil")	×
350
351	self.configure_args = list(dict.fromkeys(self.configure_args)) # preserve order for reproducibility	6 ✔
352
353	return env	6 ✔
354
355	def build_arch(self, arch):	6 ✔
356	if self.ctx.ndk_api < self.MIN_NDK_API:	6 ✔
357	raise BuildInterruptingException(	6 ✔
358	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
359	ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
360	),
361	)
362
363	recipe_build_dir = self.get_build_dir(arch.arch)	6 ✔
364
365	# Create a subdirectory to actually perform the build
366	build_dir = join(recipe_build_dir, 'android-build')	6 ✔
367	ensure_dir(build_dir)	6 ✔
368
369	sys_prefix = join(build_dir, "android-root")	6 ✔
370	ensure_dir(sys_prefix)	6 ✔
371
372	env = self.get_recipe_env(arch)	6 ✔
373	env = self.set_libs_flags(env, arch)	6 ✔
374
375	android_build = sh.Command(	6 ✔
376	join(recipe_build_dir,
377	'config.guess'))().strip()
378
379	with current_directory(build_dir):	6 ✔
380	if not exists('config.status'):	6 !
381	shprint(	6 ✔
382	sh.Command(join(recipe_build_dir, 'configure')),
383	*(' '.join(self.configure_args).format(
384	android_host=env['HOSTARCH'],
385	android_build=android_build,
386	python_host_bin=self.get_recipe(
387	'host' + self.name, self.ctx
388	).python_exe,
389	prefix=sys_prefix).split(' ')),
390	_env=env)
391
392	shprint(	6 ✔
393	sh.make,
394	'all',
395	'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
396	_env=env
397	)
398	shprint(sh.make, 'install', _env=env)	6 ✔
399
400	# rename executable
401	if isfile("python"):	6 !
UNCOV 402	sh.cp('python', 'libpythonbin.so')	×
403	elif isfile("python.exe"): # for macos	6 !
UNCOV 404	sh.cp('python.exe', 'libpythonbin.so')	×
405
406	# TODO: Look into passing the path to pyconfig.h in a
407	# better way, although this is probably acceptable
408	sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))	6 ✔
409
410	def compile_python_files(self, dir):	6 ✔
411	'''
412	Compile the python files (recursively) for the python files inside
413	a given folder.
414
415	.. note:: python2 compiles the files into extension .pyo, but in
416	python3, and as of Python 3.5, the .pyo filename extension is no
417	longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
418	'''
419	args = [self.ctx.hostpython]	6 ✔
420	args += ['-OO', '-m', 'compileall', '-b', '-f', '-q', dir]	6 ✔
421	subprocess.call(args)	6 ✔
422
423	def create_python_bundle(self, dirn, arch):	6 ✔
424	"""
425	Create a packaged python bundle in the target directory, by
426	copying all the modules and standard library to the right
427	place.
428	"""
UNCOV 429	modules_build_dir = glob.glob(join(	×
430	self.get_build_dir(arch.arch),
431	'android-build',
432	'build',
433	'lib.*'
434	))[0]
435	# Compile to *.pyc the python modules
UNCOV 436	self.compile_python_files(modules_build_dir)	×
437	# Compile to *.pyc the standard python library
UNCOV 438	self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))	×
439	# Compile to *.pyc the other python packages (site-packages)
UNCOV 440	self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))	×
441
442	# Bundle compiled python modules to a folder
443	modules_dir = join(dirn, 'modules')	×
444	c_ext = self.compiled_extension	×
445	ensure_dir(modules_dir)	×
UNCOV 446	module_filens = (glob.glob(join(modules_build_dir, '*.so')) +	×
447	glob.glob(join(modules_build_dir, '*' + c_ext)))
448	info("Copy {} files into the bundle".format(len(module_filens)))	×
449	for filen in module_filens:	×
450	info(" - copy {}".format(filen))	×
UNCOV 451	shutil.copy2(filen, modules_dir)	×
452
453	# zip up the standard library
454	stdlib_zip = join(dirn, 'stdlib.zip')	×
455	with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):	×
UNCOV 456	stdlib_filens = list(walk_valid_filens(	×
457	'.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
UNCOV 458	if 'SOURCE_DATE_EPOCH' in environ:	×
459	# for reproducible builds
460	stdlib_filens.sort()	×
461	timestamp = int(environ['SOURCE_DATE_EPOCH'])	×
462	for filen in stdlib_filens:	×
463	utime(filen, (timestamp, timestamp))	×
464	info("Zip {} files into the bundle".format(len(stdlib_filens)))	×
UNCOV 465	shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)	×
466
467	# copy the site-packages into place
468	ensure_dir(join(dirn, 'site-packages'))	×
UNCOV 469	ensure_dir(self.ctx.get_python_install_dir(arch.arch))	×
470	# TODO: Improve the API around walking and copying the files
471	with current_directory(self.ctx.get_python_install_dir(arch.arch)):	×
UNCOV 472	filens = list(walk_valid_filens(	×
473	'.', self.site_packages_dir_blacklist,
474	self.site_packages_filen_blacklist,
475	excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
476	info("Copy {} files into the site-packages".format(len(filens)))	×
477	for filen in filens:	×
478	info(" - copy {}".format(filen))	×
479	ensure_dir(join(dirn, 'site-packages', dirname(filen)))	×
UNCOV 480	shutil.copy2(filen, join(dirn, 'site-packages', filen))	×
481
482	# copy the python .so files into place
UNCOV 483	python_build_dir = join(self.get_build_dir(arch.arch),	×
484	'android-build')
485	python_lib_name = 'libpython' + self.link_version	×
UNCOV 486	shprint(	×
487	sh.cp,
488	join(python_build_dir, python_lib_name + '.so'),
489	join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
490	)
491
492	info('Renaming .so files to reflect cross-compile')	×
UNCOV 493	self.reduce_object_file_names(join(dirn, 'site-packages'))	×
494
UNCOV 495	return join(dirn, 'site-packages')	×
496
497
498	recipe = Python3Recipe()	6 ✔

kivy / python-for-android / 36313254676

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous