36241023637

Committed 26 Sep 2026 12:09PM UTC coverage: 63.064% (+0.02%) from 63.041%

Build # 36241023637

Build Type

Pull #3383

github

Committed by

web-flow

Commit Message

Merge 4b6628dc4 into e772ad93f

Pull Request Pull Request #3383: Attempt on optimizing the builds

Coverage Stats

1829 of 3170 branches covered (57.7%)

Branch coverage included in aggregate %.

10 of 10 new or added lines in 2 files covered. (100.0%)

39 existing lines in 1 file now uncovered.

5424 of 8331 relevant lines covered (65.11%)

3.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.45

/pythonforandroid/recipes/python3/__init__.py

import glob
import sh
import subprocess

from os import environ, utime
from os.path import dirname, exists, join, isfile
import shutil

from packaging.version import Version
from pythonforandroid.logger import info, shprint, warning
from pythonforandroid.recipe import Recipe, TargetPythonRecipe
from pythonforandroid.util import (
    current_directory,
    ensure_dir,
    walk_valid_filens,
    BuildInterruptingException,
)

NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (
    'Target ndk-api is {ndk_api}, '
    'but the python3 recipe supports only {min_ndk_api}+'
)


class Python3Recipe(TargetPythonRecipe):
    '''
    The python3's recipe
    ^^^^^^^^^^^^^^^^^^^^

    The python 3 recipe can be built with some extra python modules, but to do
    so, we need some libraries. By default, we ship the python3 recipe with
    some common libraries, defined in ``depends``. We also support some optional
    libraries, which are less common that the ones defined in ``depends``, so
    we added them as optional dependencies (``opt_depends``).

    Below you have a relationship between the python modules and the recipe
    libraries::

        - _ctypes: you must add the recipe for ``libffi``.
        - _sqlite3: you must add the recipe for ``sqlite3``.
        - _ssl: you must add the recipe for ``openssl``.
        - _bz2: you must add the recipe for ``libbz2`` (optional).
        - _lzma: you must add the recipe for ``liblzma`` (optional).

    .. note:: This recipe can be built only against API 21+.

    .. versionchanged:: 2019.10.06.post0
        - Refactored from deleted class ``python.GuestPythonRecipe`` into here
        - Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
          and :mod:`~pythonforandroid.recipes.liblzma`

    .. versionchanged:: 0.6.0
        Refactored into class
        :class:`~pythonforandroid.python.GuestPythonRecipe`
    '''

    version = '3.14.2'
    url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
    name = 'python3'

    patches = [
        'patches/pyconfig_detection.patch',
        'patches/reproducible-buildinfo.diff',
    ]

    depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
    # those optional depends allow us to build python compression modules:
    #   - _bz2.so
    #   - _lzma.so
    opt_depends = ['libbz2', 'liblzma']
    '''The optional libraries which we would like to get our python linked'''

    configure_args = [
        '--host={android_host}',
        '--build={android_build}',
        '--enable-ipv6',
        '--enable-loadable-sqlite-extensions',
        '--enable-shared',

        # Attempt on making the builds lighter
        '--disable-test-modules',
        '--without-c-locale-coercion',
        '--without-decimal-contextvar',
        '--without-doc-strings',
        '--without-ensurepip',
        '--without-readline',
        '--without-static-libpython',

        # Android prefix
        '--prefix={prefix}',

        # Special cross compile args
        'ac_cv_header_bzlib_h=no',
        'ac_cv_header_sys_eventfd_h=no',
        'py_cv_module__curses=n/a',
        'py_cv_module__curses_panel=n/a',
        'py_cv_module__tkinter=n/a'
    ]

    '''The configure arguments needed to build the python recipe. Those are
    used in method :meth:`build_arch` (if not overwritten like python3's
    recipe does).
    '''

    MIN_NDK_API = 21
    '''Sets the minimal ndk api number needed to use the recipe.

    .. warning:: Starting from Python 3.14 this recipe can only be built
       against API 21+, so it means that any class which inherits from
       class:`GuestPythonRecipe` will have this limitation.
    '''

    stdlib_dir_blacklist = {
        '__pycache__',
        'curses',
        'ensurepip',
        'idlelib',
        'lib2to3',
        'msilib',
        'multiprocessing',
        'pydoc_data',
        'test',
        'tests',
        'tkinter',
        'turtledemo',
        'venv'
    }
    '''The directories that we want to omit for our python bundle'''

    stdlib_filen_blacklist = [
        '*.exe',
        '*.py',
        '*.whl',
        'turtle.pyc'
    ]
    '''The file extensions that we want to blacklist for our python bundle'''

    site_packages_dir_blacklist = {
        '__pycache__',
        '*.dist-info',
        'bin',
        'tests'
    }
    '''The directories from site packages dir that we don't want to be included
    in our python bundle.'''

    site_packages_excluded_dir_exceptions = [
        # 'numpy' is excluded here because importing with `import numpy as np`
        # can fail if the `tests` directory inside the numpy package is excluded.
        'numpy',
    ]
    '''Directories from `site_packages_dir_blacklist` will not be excluded
    if the full path contains any of these exceptions.'''

    site_packages_filen_blacklist = [
        '*.py',
        '*.pyx'
    ]
    '''The file extensions from site packages dir that we don't want to be
    included in our python bundle.'''

    compiled_extension = '.pyc'
    '''the default extension for compiled python files.

    .. note:: the default extension for compiled python files has been .pyo for
        python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
        longer used and has been removed in favour of extension .pyc
    '''

    disable_gil = False
    '''python3.13 experimental free-threading build'''

    built_libraries = {"libpythonbin.so": "./android-build/"}

    def __init__(self, *args, **kwargs):
        self._ctx = None
        super().__init__(*args, **kwargs)

    @property
    def _libpython(self):
        '''return the python's library name (with extension)'''
        return 'libpython{link_version}.so'.format(
            link_version=self.link_version
        )

    @property
    def link_version(self):
        '''return the python's library link version e.g. 3.7m, 3.8'''
        major, minor = self.major_minor_version_string.split('.')
        flags = ''
        if major == '3' and int(minor) < 8:
            flags += 'm'
        return '{major}.{minor}{flags}'.format(
            major=major,
            minor=minor,
            flags=flags
        )

    def apply_patches(self, arch, build_dir=None):

        _p_version = Version(self.version)
        if _p_version.major == 3 and _p_version.minor == 7:
            self.patches += [
                'patches/py3.7.1_fix-ctypes-util-find-library.patch',
                'patches/py3.7.1_fix-zlib-version.patch',
            ]

        if 8 <= _p_version.minor <= 10:
            self.patches.append('patches/py3.8.1.patch')

        if _p_version.minor >= 11:
            self.patches.append('patches/cpython-311-ctypes-find-library.patch')

        if _p_version.minor >= 14:
            self.patches.append('patches/3.14_armv7l_fix.patch')
            self.patches.append('patches/3.14_fix_remote_debug.patch')

        if shutil.which('lld') is not None:
            if _p_version.minor == 7:
                self.patches.append("patches/py3.7.1_fix_cortex_a8.patch")
            elif _p_version.minor >= 8:
                self.patches.append("patches/py3.8.1_fix_cortex_a8.patch")

        self.patches = list(dict.fromkeys(self.patches))  # preserve order for reproducibility
        super().apply_patches(arch, build_dir)

    def include_root(self, arch_name):
        _p_version = Version(self.version)
        return join(
            self.get_build_dir(arch_name), 'android-build', 'android-root',
            'include', f'python{_p_version.major}.{_p_version.minor}'
        )

    def link_root(self, arch_name):
        return join(self.get_build_dir(arch_name), 'android-build')

    def get_python_root(self, arch):
        return join(self.get_build_dir(arch.arch), 'android-build', 'android-root')

    def get_android_python_exe(self, arch):
        return join(self.get_python_root(arch), 'bin', self.name)

    def should_build(self, arch):
        return not isfile(join(self.link_root(arch.arch), self._libpython))

    def prebuild_arch(self, arch):
        super().prebuild_arch(arch)
        self.ctx.python_recipe = self

    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
        env = super().get_recipe_env(arch)
        env['HOSTARCH'] = arch.command_prefix
        env['CC'] = arch.get_clang_exe(with_target=True)
        env['PATH'] = '{hostpython_dir}:{old_path}'.format(
            hostpython_dir=self.get_recipe(
                'host' + self.name, self.ctx
            ).get_path_to_python(),
            old_path=env['PATH']
        )
        env['CFLAGS'] = ' '.join(
            [
                '-ffunction-sections',
                '-fdata-sections',
                '-fPIC'
            ]
        )

        env['LDFLAGS'] = env.get('LDFLAGS', '')
        if shutil.which('lld') is not None:
            # Note: The -L. is to fix a bug in python 3.7.
            # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
            env['LDFLAGS'] += ' -L. -fuse-ld=lld'
        else:
            warning('lld not found, linking without it. '
                    'Consider installing lld if linker errors occur.')

        return env

    def set_libs_flags(self, env, arch):
        '''Takes care to properly link libraries with python depending on our
        requirements and the attribute :attr:`opt_depends`.
        '''
        def add_flags(include_flags, link_dirs, link_libs):
            env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
            env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
            env['LIBS'] = env.get('LIBS', '') + link_libs

        info('Activating flags for sqlite3')
        recipe = Recipe.get_recipe('sqlite3', self.ctx)
        add_flags(' -I' + recipe.get_build_dir(arch.arch),
                  ' -L' + recipe.get_build_dir(arch.arch), ' -lsqlite3')

        info('Activating flags for libffi')
        recipe = Recipe.get_recipe('libffi', self.ctx)
        # In order to force the correct linkage for our libffi library, we
        # set the following variable to point where is our libffi.pc file,
        # because the python build system uses pkg-config to configure it.
        env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)
        add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
                  ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
                  ' -lffi')

        info('Activating flags for openssl')
        recipe = Recipe.get_recipe('openssl', self.ctx)
        self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))
        add_flags(recipe.include_flags(arch),
                  recipe.link_dirs_flags(arch), recipe.link_libs_flags())

        for library_name in {'libbz2', 'liblzma'}:
            if library_name in self.ctx.recipe_build_order:
                info(f'Activating flags for {library_name}')
                recipe = Recipe.get_recipe(library_name, self.ctx)
                add_flags(recipe.get_library_includes(arch),
                          recipe.get_library_ldflags(arch),
                          recipe.get_library_libs_flag())

        # python build system contains hardcoded zlib version which prevents
        # the build of zlib module, here we search for android's zlib version
        # and sets the right flags, so python can be build with android's zlib
        info("Activating flags for android's zlib")
        zlib_lib_path = arch.ndk_lib_dir_versioned
        zlib_includes = self.ctx.ndk.sysroot_include_dir
        zlib_h = join(zlib_includes, 'zlib.h')
        try:
            with open(zlib_h) as fileh:
                zlib_data = fileh.read()
        except IOError:
            raise BuildInterruptingException(
                "Could not determine android's zlib version, no zlib.h ({}) in"
                " the NDK dir includes".format(zlib_h)
            )
        for line in zlib_data.split('\n'):
            if line.startswith('#define ZLIB_VERSION '):
                break
        else:
            raise BuildInterruptingException(
                'Could not parse zlib.h...so we cannot find zlib version,'
                'required by python build,'
            )
        env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
        add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')

        _p_version = Version(self.version)
        if _p_version.minor >= 11:
            self.configure_args.append('--with-build-python={python_host_bin}')

        if _p_version.minor >= 13 and self.disable_gil:
            self.configure_args.append("--disable-gil")

        self.configure_args = list(dict.fromkeys(self.configure_args))  # preserve order for reproducibility

        return env

    def build_arch(self, arch):
        if self.ctx.ndk_api < self.MIN_NDK_API:
            raise BuildInterruptingException(
                NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
                    ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
                ),
            )

        recipe_build_dir = self.get_build_dir(arch.arch)

        # Create a subdirectory to actually perform the build
        build_dir = join(recipe_build_dir, 'android-build')
        ensure_dir(build_dir)

        sys_prefix = join(build_dir, "android-root")
        ensure_dir(sys_prefix)

        env = self.get_recipe_env(arch)
        env = self.set_libs_flags(env, arch)

        android_build = sh.Command(
            join(recipe_build_dir,
                 'config.guess'))().strip()

        with current_directory(build_dir):
            if not exists('config.status'):
                shprint(
                    sh.Command(join(recipe_build_dir, 'configure')),
                    *(' '.join(self.configure_args).format(
                                    android_host=env['HOSTARCH'],
                                    android_build=android_build,
                                    python_host_bin=self.get_recipe(
                                        'host' + self.name, self.ctx
                                    ).python_exe,
                                    prefix=sys_prefix).split(' ')),
                    _env=env)

            shprint(
                sh.make,
                'all',
                'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
                _env=env
            )
            shprint(sh.make, 'install', _env=env)

            # rename executable
            if isfile("python"):
                sh.cp('python', 'libpythonbin.so')
            elif isfile("python.exe"):  # for macos
                sh.cp('python.exe', 'libpythonbin.so')

            # TODO: Look into passing the path to pyconfig.h in a
            # better way, although this is probably acceptable
            sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))

    def compile_python_files(self, dir):
        '''
        Compile the python files (recursively) for the python files inside
        a given folder.

        .. note:: python2 compiles the files into extension .pyo, but in
            python3, and as of Python 3.5, the .pyo filename extension is no
            longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
        '''
        args = [self.ctx.hostpython]
        args += ['-OO', '-m', 'compileall', '-b', '-f', '-q', dir]
        subprocess.call(args)

    def create_python_bundle(self, dirn, arch):
        """
        Create a packaged python bundle in the target directory, by
        copying all the modules and standard library to the right
        place.
        """
        modules_build_dir = glob.glob(join(
            self.get_build_dir(arch.arch),
            'android-build',
            'build',
            'lib.*'
        ))[0]
        # Compile to *.pyc the python modules
        self.compile_python_files(modules_build_dir)
        # Compile to *.pyc the standard python library
        self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
        # Compile to *.pyc the other python packages (site-packages)
        self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))

        # Bundle compiled python modules to a folder
        modules_dir = join(dirn, 'modules')
        c_ext = self.compiled_extension
        ensure_dir(modules_dir)
        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
                         glob.glob(join(modules_build_dir, '*' + c_ext)))
        info("Copy {} files into the bundle".format(len(module_filens)))
        for filen in module_filens:
            info(" - copy {}".format(filen))
            shutil.copy2(filen, modules_dir)

        # zip up the standard library
        stdlib_zip = join(dirn, 'stdlib.zip')
        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
            stdlib_filens = list(walk_valid_filens(
                '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
            if 'SOURCE_DATE_EPOCH' in environ:
                # for reproducible builds
                stdlib_filens.sort()
                timestamp = int(environ['SOURCE_DATE_EPOCH'])
                for filen in stdlib_filens:
                    utime(filen, (timestamp, timestamp))
            info("Zip {} files into the bundle".format(len(stdlib_filens)))
            shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)

        # copy the site-packages into place
        ensure_dir(join(dirn, 'site-packages'))
        ensure_dir(self.ctx.get_python_install_dir(arch.arch))
        # TODO: Improve the API around walking and copying the files
        with current_directory(self.ctx.get_python_install_dir(arch.arch)):
            filens = list(walk_valid_filens(
                '.', self.site_packages_dir_blacklist,
                self.site_packages_filen_blacklist,
                excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
            info("Copy {} files into the site-packages".format(len(filens)))
            for filen in filens:
                info(" - copy {}".format(filen))
                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
                shutil.copy2(filen, join(dirn, 'site-packages', filen))

        # copy the python .so files into place
        python_build_dir = join(self.get_build_dir(arch.arch),
                                'android-build')
        python_lib_name = 'libpython' + self.link_version
        shprint(
            sh.cp,
            join(python_build_dir, python_lib_name + '.so'),
            join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
        )

        info('Renaming .so files to reflect cross-compile')
        self.reduce_object_file_names(join(dirn, 'site-packages'))

        return join(dirn, 'site-packages')


recipe = Python3Recipe()

1	import glob	6✔
2	import sh	6✔
3	import subprocess	6✔
4
5	from os import environ, utime	6✔
6	from os.path import dirname, exists, join, isfile	6✔
7	import shutil	6✔
8
9	from packaging.version import Version	6✔
10	from pythonforandroid.logger import info, shprint, warning	6✔
11	from pythonforandroid.recipe import Recipe, TargetPythonRecipe	6✔
12	from pythonforandroid.util import (	6✔
13	current_directory,
14	ensure_dir,
15	walk_valid_filens,
16	BuildInterruptingException,
17	)
18
19	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (	6✔
20	'Target ndk-api is {ndk_api}, '
21	'but the python3 recipe supports only {min_ndk_api}+'
22	)
23
24
25	class Python3Recipe(TargetPythonRecipe):	6✔
26	'''
27	The python3's recipe
28	^^^^^^^^^^^^^^^^^^^^
29
30	The python 3 recipe can be built with some extra python modules, but to do
31	so, we need some libraries. By default, we ship the python3 recipe with
32	some common libraries, defined in ``depends``. We also support some optional
33	libraries, which are less common that the ones defined in ``depends``, so
34	we added them as optional dependencies (``opt_depends``).
35
36	Below you have a relationship between the python modules and the recipe
37	libraries::
38
39	- _ctypes: you must add the recipe for ``libffi``.
40	- _sqlite3: you must add the recipe for ``sqlite3``.
41	- _ssl: you must add the recipe for ``openssl``.
42	- _bz2: you must add the recipe for ``libbz2`` (optional).
43	- _lzma: you must add the recipe for ``liblzma`` (optional).
44
45	.. note:: This recipe can be built only against API 21+.
46
47	.. versionchanged:: 2019.10.06.post0
48	- Refactored from deleted class ``python.GuestPythonRecipe`` into here
49	- Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
50	and :mod:`~pythonforandroid.recipes.liblzma`
51
52	.. versionchanged:: 0.6.0
53	Refactored into class
54	:class:`~pythonforandroid.python.GuestPythonRecipe`
55	'''
56
57	version = '3.14.2'	6✔
58	url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'	6✔
59	name = 'python3'	6✔
60
61	patches = [	6✔
62	'patches/pyconfig_detection.patch',
63	'patches/reproducible-buildinfo.diff',
64	]
65
66	depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']	6✔
67	# those optional depends allow us to build python compression modules:
68	# - _bz2.so
69	# - _lzma.so
70	opt_depends = ['libbz2', 'liblzma']	6✔
71	'''The optional libraries which we would like to get our python linked'''	6✔
72
73	configure_args = [	6✔
74	'--host={android_host}',
75	'--build={android_build}',
76	'--enable-ipv6',
77	'--enable-loadable-sqlite-extensions',
78	'--enable-shared',
79
80	# Attempt on making the builds lighter
81	'--disable-test-modules',
82	'--without-c-locale-coercion',
83	'--without-decimal-contextvar',
84	'--without-doc-strings',
85	'--without-ensurepip',
86	'--without-readline',
87	'--without-static-libpython',
88
89	# Android prefix
90	'--prefix={prefix}',
91
92	# Special cross compile args
93	'ac_cv_header_bzlib_h=no',
94	'ac_cv_header_sys_eventfd_h=no',
95	'py_cv_module__curses=n/a',
96	'py_cv_module__curses_panel=n/a',
97	'py_cv_module__tkinter=n/a'
98	]
99
100	'''The configure arguments needed to build the python recipe. Those are	6✔
101	used in method :meth:`build_arch` (if not overwritten like python3's
102	recipe does).
103	'''
104
105	MIN_NDK_API = 21	6✔
106	'''Sets the minimal ndk api number needed to use the recipe.	6✔
107
108	.. warning:: Starting from Python 3.14 this recipe can only be built
109	against API 21+, so it means that any class which inherits from
110	class:`GuestPythonRecipe` will have this limitation.
111	'''
112
113	stdlib_dir_blacklist = {	6✔
114	'__pycache__',
115	'curses',
116	'ensurepip',
117	'idlelib',
118	'lib2to3',
119	'msilib',
120	'multiprocessing',
121	'pydoc_data',
122	'test',
123	'tests',
124	'tkinter',
125	'turtledemo',
126	'venv'
127	}
128	'''The directories that we want to omit for our python bundle'''	6✔
129
130	stdlib_filen_blacklist = [	6✔
131	'*.exe',
132	'*.py',
133	'*.whl',
134	'turtle.pyc'
135	]
136	'''The file extensions that we want to blacklist for our python bundle'''	6✔
137
138	site_packages_dir_blacklist = {	6✔
139	'__pycache__',
140	'*.dist-info',
141	'bin',
142	'tests'
143	}
144	'''The directories from site packages dir that we don't want to be included	6✔
145	in our python bundle.'''
146
147	site_packages_excluded_dir_exceptions = [	6✔
148	# 'numpy' is excluded here because importing with `import numpy as np`
149	# can fail if the `tests` directory inside the numpy package is excluded.
150	'numpy',
151	]
152	'''Directories from `site_packages_dir_blacklist` will not be excluded	6✔
153	if the full path contains any of these exceptions.'''
154
155	site_packages_filen_blacklist = [	6✔
156	'*.py',
157	'*.pyx'
158	]
159	'''The file extensions from site packages dir that we don't want to be	6✔
160	included in our python bundle.'''
161
162	compiled_extension = '.pyc'	6✔
163	'''the default extension for compiled python files.	6✔
164
165	.. note:: the default extension for compiled python files has been .pyo for
166	python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
167	longer used and has been removed in favour of extension .pyc
168	'''
169
170	disable_gil = False	6✔
171	'''python3.13 experimental free-threading build'''	6✔
172
173	built_libraries = {"libpythonbin.so": "./android-build/"}	6✔
174
175	def __init__(self, args, *kwargs):	6✔
176	self._ctx = None	6✔
177	super().__init__(args, *kwargs)	6✔
178
179	@property	6✔
180	def _libpython(self):	6✔
181	'''return the python's library name (with extension)'''
182	return 'libpython{link_version}.so'.format(	6✔
183	link_version=self.link_version
184	)
185
186	@property	6✔
187	def link_version(self):	6✔
188	'''return the python's library link version e.g. 3.7m, 3.8'''
189	major, minor = self.major_minor_version_string.split('.')	6✔
190	flags = ''	6✔
191	if major == '3' and int(minor) < 8:	6!
UNCOV 192	flags += 'm'	×
193	return '{major}.{minor}{flags}'.format(	6✔
194	major=major,
195	minor=minor,
196	flags=flags
197	)
198
199	def apply_patches(self, arch, build_dir=None):	6✔
200
UNCOV 201	_p_version = Version(self.version)	×
UNCOV 202	if _p_version.major == 3 and _p_version.minor == 7:	×
203	self.patches += [	×
204	'patches/py3.7.1_fix-ctypes-util-find-library.patch',
205	'patches/py3.7.1_fix-zlib-version.patch',
206	]
207
UNCOV 208	if 8 <= _p_version.minor <= 10:	×
UNCOV 209	self.patches.append('patches/py3.8.1.patch')	×
210
211	if _p_version.minor >= 11:	×
UNCOV 212	self.patches.append('patches/cpython-311-ctypes-find-library.patch')	×
213
214	if _p_version.minor >= 14:	×
UNCOV 215	self.patches.append('patches/3.14_armv7l_fix.patch')	×
216	self.patches.append('patches/3.14_fix_remote_debug.patch')	×
217
218	if shutil.which('lld') is not None:	×
UNCOV 219	if _p_version.minor == 7:	×
220	self.patches.append("patches/py3.7.1_fix_cortex_a8.patch")	×
221	elif _p_version.minor >= 8:	×
222	self.patches.append("patches/py3.8.1_fix_cortex_a8.patch")	×
223
224	self.patches = list(dict.fromkeys(self.patches)) # preserve order for reproducibility	×
UNCOV 225	super().apply_patches(arch, build_dir)	×
226
227	def include_root(self, arch_name):	6✔
228	_p_version = Version(self.version)	6✔
229	return join(	6✔
230	self.get_build_dir(arch_name), 'android-build', 'android-root',
231	'include', f'python{_p_version.major}.{_p_version.minor}'
232	)
233
234	def link_root(self, arch_name):	6✔
235	return join(self.get_build_dir(arch_name), 'android-build')	6✔
236
237	def get_python_root(self, arch):	6✔
UNCOV 238	return join(self.get_build_dir(arch.arch), 'android-build', 'android-root')	×
239
240	def get_android_python_exe(self, arch):	6✔
UNCOV 241	return join(self.get_python_root(arch), 'bin', self.name)	×
242
243	def should_build(self, arch):	6✔
UNCOV 244	return not isfile(join(self.link_root(arch.arch), self._libpython))	×
245
246	def prebuild_arch(self, arch):	6✔
UNCOV 247	super().prebuild_arch(arch)	×
UNCOV 248	self.ctx.python_recipe = self	×
249
250	def get_recipe_env(self, arch=None, with_flags_in_cc=True):	6✔
251	env = super().get_recipe_env(arch)	6✔
252	env['HOSTARCH'] = arch.command_prefix	6✔
253	env['CC'] = arch.get_clang_exe(with_target=True)	6✔
254	env['PATH'] = '{hostpython_dir}:{old_path}'.format(	6✔
255	hostpython_dir=self.get_recipe(
256	'host' + self.name, self.ctx
257	).get_path_to_python(),
258	old_path=env['PATH']
259	)
260	env['CFLAGS'] = ' '.join(	6✔
261	[
262	'-ffunction-sections',
263	'-fdata-sections',
264	'-fPIC'
265	]
266	)
267
268	env['LDFLAGS'] = env.get('LDFLAGS', '')	6✔
269	if shutil.which('lld') is not None:	6!
270	# Note: The -L. is to fix a bug in python 3.7.
271	# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
272	env['LDFLAGS'] += ' -L. -fuse-ld=lld'	6✔
273	else:
UNCOV 274	warning('lld not found, linking without it. '	×
275	'Consider installing lld if linker errors occur.')
276
277	return env	6✔
278
279	def set_libs_flags(self, env, arch):	6✔
280	'''Takes care to properly link libraries with python depending on our
281	requirements and the attribute :attr:`opt_depends`.
282	'''
283	def add_flags(include_flags, link_dirs, link_libs):	6✔
284	env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags	6✔
285	env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs	6✔
286	env['LIBS'] = env.get('LIBS', '') + link_libs	6✔
287
288	info('Activating flags for sqlite3')	6✔
289	recipe = Recipe.get_recipe('sqlite3', self.ctx)	6✔
290	add_flags(' -I' + recipe.get_build_dir(arch.arch),	6✔
291	' -L' + recipe.get_build_dir(arch.arch), ' -lsqlite3')
292
293	info('Activating flags for libffi')	6✔
294	recipe = Recipe.get_recipe('libffi', self.ctx)	6✔
295	# In order to force the correct linkage for our libffi library, we
296	# set the following variable to point where is our libffi.pc file,
297	# because the python build system uses pkg-config to configure it.
298	env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)	6✔
299	add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),	6✔
300	' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
301	' -lffi')
302
303	info('Activating flags for openssl')	6✔
304	recipe = Recipe.get_recipe('openssl', self.ctx)	6✔
305	self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))	6✔
306	add_flags(recipe.include_flags(arch),	6✔
307	recipe.link_dirs_flags(arch), recipe.link_libs_flags())
308
309	for library_name in {'libbz2', 'liblzma'}:	6✔
310	if library_name in self.ctx.recipe_build_order:	6!
UNCOV 311	info(f'Activating flags for {library_name}')	×
UNCOV 312	recipe = Recipe.get_recipe(library_name, self.ctx)	×
313	add_flags(recipe.get_library_includes(arch),	×
314	recipe.get_library_ldflags(arch),
315	recipe.get_library_libs_flag())
316
317	# python build system contains hardcoded zlib version which prevents
318	# the build of zlib module, here we search for android's zlib version
319	# and sets the right flags, so python can be build with android's zlib
320	info("Activating flags for android's zlib")	6✔
321	zlib_lib_path = arch.ndk_lib_dir_versioned	6✔
322	zlib_includes = self.ctx.ndk.sysroot_include_dir	6✔
323	zlib_h = join(zlib_includes, 'zlib.h')	6✔
324	try:	6✔
325	with open(zlib_h) as fileh:	6✔
326	zlib_data = fileh.read()	6✔
UNCOV 327	except IOError:	×
UNCOV 328	raise BuildInterruptingException(	×
329	"Could not determine android's zlib version, no zlib.h ({}) in"
330	" the NDK dir includes".format(zlib_h)
331	)
332	for line in zlib_data.split('\n'):	6!
333	if line.startswith('#define ZLIB_VERSION '):	6!
334	break	6✔
335	else:
UNCOV 336	raise BuildInterruptingException(	×
337	'Could not parse zlib.h...so we cannot find zlib version,'
338	'required by python build,'
339	)
340	env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')	6✔
341	add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')	6✔
342
343	_p_version = Version(self.version)	6✔
344	if _p_version.minor >= 11:	6!
345	self.configure_args.append('--with-build-python={python_host_bin}')	6✔
346
347	if _p_version.minor >= 13 and self.disable_gil:	6!
UNCOV 348	self.configure_args.append("--disable-gil")	×
349
350	self.configure_args = list(dict.fromkeys(self.configure_args)) # preserve order for reproducibility	6✔
351
352	return env	6✔
353
354	def build_arch(self, arch):	6✔
355	if self.ctx.ndk_api < self.MIN_NDK_API:	6✔
356	raise BuildInterruptingException(	6✔
357	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
358	ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
359	),
360	)
361
362	recipe_build_dir = self.get_build_dir(arch.arch)	6✔
363
364	# Create a subdirectory to actually perform the build
365	build_dir = join(recipe_build_dir, 'android-build')	6✔
366	ensure_dir(build_dir)	6✔
367
368	sys_prefix = join(build_dir, "android-root")	6✔
369	ensure_dir(sys_prefix)	6✔
370
371	env = self.get_recipe_env(arch)	6✔
372	env = self.set_libs_flags(env, arch)	6✔
373
374	android_build = sh.Command(	6✔
375	join(recipe_build_dir,
376	'config.guess'))().strip()
377
378	with current_directory(build_dir):	6✔
379	if not exists('config.status'):	6!
380	shprint(	6✔
381	sh.Command(join(recipe_build_dir, 'configure')),
382	*(' '.join(self.configure_args).format(
383	android_host=env['HOSTARCH'],
384	android_build=android_build,
385	python_host_bin=self.get_recipe(
386	'host' + self.name, self.ctx
387	).python_exe,
388	prefix=sys_prefix).split(' ')),
389	_env=env)
390
391	shprint(	6✔
392	sh.make,
393	'all',
394	'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
395	_env=env
396	)
397	shprint(sh.make, 'install', _env=env)	6✔
398
399	# rename executable
400	if isfile("python"):	6!
UNCOV 401	sh.cp('python', 'libpythonbin.so')	×
402	elif isfile("python.exe"): # for macos	6!
403	sh.cp('python.exe', 'libpythonbin.so')	×
404
405	# TODO: Look into passing the path to pyconfig.h in a
406	# better way, although this is probably acceptable
407	sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))	6✔
408
409	def compile_python_files(self, dir):	6✔
410	'''
411	Compile the python files (recursively) for the python files inside
412	a given folder.
413
414	.. note:: python2 compiles the files into extension .pyo, but in
415	python3, and as of Python 3.5, the .pyo filename extension is no
416	longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
417	'''
418	args = [self.ctx.hostpython]	6✔
419	args += ['-OO', '-m', 'compileall', '-b', '-f', '-q', dir]	6✔
420	subprocess.call(args)	6✔
421
422	def create_python_bundle(self, dirn, arch):	6✔
423	"""
424	Create a packaged python bundle in the target directory, by
425	copying all the modules and standard library to the right
426	place.
427	"""
UNCOV 428	modules_build_dir = glob.glob(join(	×
429	self.get_build_dir(arch.arch),
430	'android-build',
431	'build',
432	'lib.*'
433	))[0]
434	# Compile to *.pyc the python modules
UNCOV 435	self.compile_python_files(modules_build_dir)	×
436	# Compile to *.pyc the standard python library
437	self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))	×
438	# Compile to *.pyc the other python packages (site-packages)
439	self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))	×
440
441	# Bundle compiled python modules to a folder
UNCOV 442	modules_dir = join(dirn, 'modules')	×
UNCOV 443	c_ext = self.compiled_extension	×
444	ensure_dir(modules_dir)	×
445	module_filens = (glob.glob(join(modules_build_dir, '*.so')) +	×
446	glob.glob(join(modules_build_dir, '*' + c_ext)))
447	info("Copy {} files into the bundle".format(len(module_filens)))	×
UNCOV 448	for filen in module_filens:	×
449	info(" - copy {}".format(filen))	×
450	shutil.copy2(filen, modules_dir)	×
451
452	# zip up the standard library
UNCOV 453	stdlib_zip = join(dirn, 'stdlib.zip')	×
UNCOV 454	with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):	×
455	stdlib_filens = list(walk_valid_filens(	×
456	'.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
457	if 'SOURCE_DATE_EPOCH' in environ:	×
458	# for reproducible builds
459	stdlib_filens.sort()	×
UNCOV 460	timestamp = int(environ['SOURCE_DATE_EPOCH'])	×
461	for filen in stdlib_filens:	×
462	utime(filen, (timestamp, timestamp))	×
463	info("Zip {} files into the bundle".format(len(stdlib_filens)))	×
464	shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)	×
465
466	# copy the site-packages into place
UNCOV 467	ensure_dir(join(dirn, 'site-packages'))	×
UNCOV 468	ensure_dir(self.ctx.get_python_install_dir(arch.arch))	×
469	# TODO: Improve the API around walking and copying the files
470	with current_directory(self.ctx.get_python_install_dir(arch.arch)):	×
UNCOV 471	filens = list(walk_valid_filens(	×
472	'.', self.site_packages_dir_blacklist,
473	self.site_packages_filen_blacklist,
474	excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
UNCOV 475	info("Copy {} files into the site-packages".format(len(filens)))	×
UNCOV 476	for filen in filens:	×
477	info(" - copy {}".format(filen))	×
478	ensure_dir(join(dirn, 'site-packages', dirname(filen)))	×
479	shutil.copy2(filen, join(dirn, 'site-packages', filen))	×
480
481	# copy the python .so files into place
UNCOV 482	python_build_dir = join(self.get_build_dir(arch.arch),	×
483	'android-build')
484	python_lib_name = 'libpython' + self.link_version	×
UNCOV 485	shprint(	×
486	sh.cp,
487	join(python_build_dir, python_lib_name + '.so'),
488	join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
489	)
490
UNCOV 491	info('Renaming .so files to reflect cross-compile')	×
UNCOV 492	self.reduce_object_file_names(join(dirn, 'site-packages'))	×
493
494	return join(dirn, 'site-packages')	×
495
496
497	recipe = Python3Recipe()	6✔

kivy / python-for-android / 36241023637

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous