36255542538

Committed 26 Sep 2026 04:27PM UTC coverage: 63.064% (+0.02%) from 63.041%

Build # 36255542538

Build Type

Pull #3383

github

Committed by

web-flow

Commit Message

Merge 1cb6620a3 into e772ad93f

Pull Request Pull Request #3383: Attempt on optimizing the builds

Coverage Stats

1829 of 3170 branches covered (57.7%)

Branch coverage included in aggregate %.

7 of 8 new or added lines in 2 files covered. (87.5%)

36 existing lines in 1 file now uncovered.

5424 of 8331 relevant lines covered (65.11%)

3.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.45

/pythonforandroid/recipes/python3/__init__.py

import glob
import sh
import subprocess

from os import environ, utime
from os.path import dirname, exists, join, isfile
import shutil

from packaging.version import Version
from pythonforandroid.logger import info, shprint, warning
from pythonforandroid.recipe import Recipe, TargetPythonRecipe
from pythonforandroid.util import (
    current_directory,
    ensure_dir,
    walk_valid_filens,
    BuildInterruptingException,
)

NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (
    'Target ndk-api is {ndk_api}, '
    'but the python3 recipe supports only {min_ndk_api}+'
)


class Python3Recipe(TargetPythonRecipe):
    '''
    The python3's recipe
    ^^^^^^^^^^^^^^^^^^^^

    The python 3 recipe can be built with some extra python modules, but to do
    so, we need some libraries. By default, we ship the python3 recipe with
    some common libraries, defined in ``depends``. We also support some optional
    libraries, which are less common that the ones defined in ``depends``, so
    we added them as optional dependencies (``opt_depends``).

    Below you have a relationship between the python modules and the recipe
    libraries::

        - _ctypes: you must add the recipe for ``libffi``.
        - _sqlite3: you must add the recipe for ``sqlite3``.
        - _ssl: you must add the recipe for ``openssl``.
        - _bz2: you must add the recipe for ``libbz2`` (optional).
        - _lzma: you must add the recipe for ``liblzma`` (optional).

    .. note:: This recipe can be built only against API 21+.

    .. versionchanged:: 2019.10.06.post0
        - Refactored from deleted class ``python.GuestPythonRecipe`` into here
        - Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
          and :mod:`~pythonforandroid.recipes.liblzma`

    .. versionchanged:: 0.6.0
        Refactored into class
        :class:`~pythonforandroid.python.GuestPythonRecipe`
    '''

    version = '3.14.2'
    url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
    name = 'python3'

    patches = [
        'patches/pyconfig_detection.patch',
        'patches/reproducible-buildinfo.diff',
    ]

    depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
    # those optional depends allow us to build python compression modules:
    #   - _bz2.so
    #   - _lzma.so
    opt_depends = ['libbz2', 'liblzma']
    '''The optional libraries which we would like to get our python linked'''

    configure_args = [
        '--host={android_host}',
        '--build={android_build}',
        '--enable-ipv6',
        '--enable-loadable-sqlite-extensions',
        '--enable-shared',

        # Attempt on making the builds lighter
        '--disable-test-modules',
        '--without-c-locale-coercion',
        '--without-decimal-contextvar',
        '--without-doc-strings',
        '--without-ensurepip',
        '--without-readline',
        '--without-static-libpython',

        # Android prefix
        '--prefix={prefix}',

        # Special cross compile args
        'ac_cv_header_bzlib_h=no',
        'ac_cv_header_sys_eventfd_h=no',
        'py_cv_module__curses=n/a',
        'py_cv_module__curses_panel=n/a',
        'py_cv_module__tkinter=n/a'
    ]

    '''The configure arguments needed to build the python recipe. Those are
    used in method :meth:`build_arch` (if not overwritten like python3's
    recipe does).
    '''

    MIN_NDK_API = 21
    '''Sets the minimal ndk api number needed to use the recipe.

    .. warning:: Starting from Python 3.14 this recipe can only be built
       against API 21+, so it means that any class which inherits from
       class:`GuestPythonRecipe` will have this limitation.
    '''

    stdlib_dir_blacklist = {
        '__pycache__',
        'curses',
        'ensurepip',
        'idlelib',
        'lib2to3',
        'msilib',
        'multiprocessing',
        'pydoc_data',
        'test',
        'tests',
        'tkinter',
        'turtledemo',
        'venv'
    }
    '''The directories that we want to omit for our python bundle'''

    stdlib_filen_blacklist = [
        '*.exe',
        '*.py',
        '*.whl',
        'turtle.pyc'
    ]
    '''The file extensions that we want to blacklist for our python bundle'''

    site_packages_dir_blacklist = {
        '__pycache__',
        '*.dist-info',
        'bin',
        'tests',
        'setuptools',
        '_distutils_hack'
    }
    '''The directories from site packages dir that we don't want to be included
    in our python bundle.'''

    site_packages_excluded_dir_exceptions = [
        # 'numpy' is excluded here because importing with `import numpy as np`
        # can fail if the `tests` directory inside the numpy package is excluded.
        'numpy',
    ]
    '''Directories from `site_packages_dir_blacklist` will not be excluded
    if the full path contains any of these exceptions.'''

    site_packages_filen_blacklist = [
        '*.py',
        '*.pyx'
    ]
    '''The file extensions from site packages dir that we don't want to be
    included in our python bundle.'''

    compiled_extension = '.pyc'
    '''the default extension for compiled python files.

    .. note:: the default extension for compiled python files has been .pyo for
        python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
        longer used and has been removed in favour of extension .pyc
    '''

    disable_gil = False
    '''python3.13 experimental free-threading build'''

    built_libraries = {"libpythonbin.so": "./android-build/"}

    def __init__(self, *args, **kwargs):
        self._ctx = None
        super().__init__(*args, **kwargs)

    @property
    def _libpython(self):
        '''return the python's library name (with extension)'''
        return 'libpython{link_version}.so'.format(
            link_version=self.link_version
        )

    @property
    def link_version(self):
        '''return the python's library link version e.g. 3.7m, 3.8'''
        major, minor = self.major_minor_version_string.split('.')
        flags = ''
        if major == '3' and int(minor) < 8:
            flags += 'm'
        return '{major}.{minor}{flags}'.format(
            major=major,
            minor=minor,
            flags=flags
        )

    def apply_patches(self, arch, build_dir=None):

        _p_version = Version(self.version)
        if _p_version.major == 3 and _p_version.minor == 7:
            self.patches += [
                'patches/py3.7.1_fix-ctypes-util-find-library.patch',
                'patches/py3.7.1_fix-zlib-version.patch',
            ]

        if 8 <= _p_version.minor <= 10:
            self.patches.append('patches/py3.8.1.patch')

        if _p_version.minor >= 11:
            self.patches.append('patches/cpython-311-ctypes-find-library.patch')

        if _p_version.minor >= 14:
            self.patches.append('patches/3.14_armv7l_fix.patch')
            self.patches.append('patches/3.14_fix_remote_debug.patch')

        if shutil.which('lld') is not None:
            if _p_version.minor == 7:
                self.patches.append("patches/py3.7.1_fix_cortex_a8.patch")
            elif _p_version.minor >= 8:
                self.patches.append("patches/py3.8.1_fix_cortex_a8.patch")

        self.patches = list(dict.fromkeys(self.patches))  # preserve order for reproducibility
        super().apply_patches(arch, build_dir)

    def include_root(self, arch_name):
        _p_version = Version(self.version)
        return join(
            self.get_build_dir(arch_name), 'android-build', 'android-root',
            'include', f'python{_p_version.major}.{_p_version.minor}'
        )

    def link_root(self, arch_name):
        return join(self.get_build_dir(arch_name), 'android-build')

    def get_python_root(self, arch):
        return join(self.get_build_dir(arch.arch), 'android-build', 'android-root')

    def get_android_python_exe(self, arch):
        return join(self.get_python_root(arch), 'bin', self.name)

    def should_build(self, arch):
        return not isfile(join(self.link_root(arch.arch), self._libpython))

    def prebuild_arch(self, arch):
        super().prebuild_arch(arch)
        self.ctx.python_recipe = self

    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
        env = super().get_recipe_env(arch)
        env['HOSTARCH'] = arch.command_prefix
        env['CC'] = arch.get_clang_exe(with_target=True)
        env['PATH'] = '{hostpython_dir}:{old_path}'.format(
            hostpython_dir=self.get_recipe(
                'host' + self.name, self.ctx
            ).get_path_to_python(),
            old_path=env['PATH']
        )
        env['CFLAGS'] = ' '.join(
            [
                '-ffunction-sections',
                '-fdata-sections',
                '-fPIC'
            ]
        )

        env['LDFLAGS'] = env.get('LDFLAGS', '')
        if shutil.which('lld') is not None:
            # Note: The -L. is to fix a bug in python 3.7.
            # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
            env['LDFLAGS'] += ' -L. -fuse-ld=lld'
        else:
            warning('lld not found, linking without it. '
                    'Consider installing lld if linker errors occur.')

        return env

    def set_libs_flags(self, env, arch):
        '''Takes care to properly link libraries with python depending on our
        requirements and the attribute :attr:`opt_depends`.
        '''
        def add_flags(include_flags, link_dirs, link_libs):
            env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
            env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
            env['LIBS'] = env.get('LIBS', '') + link_libs

        info('Activating flags for sqlite3')
        recipe = Recipe.get_recipe('sqlite3', self.ctx)
        add_flags(' -I' + recipe.get_build_dir(arch.arch),
                  ' -L' + recipe.get_build_dir(arch.arch), ' -lsqlite3')

        info('Activating flags for libffi')
        recipe = Recipe.get_recipe('libffi', self.ctx)
        # In order to force the correct linkage for our libffi library, we
        # set the following variable to point where is our libffi.pc file,
        # because the python build system uses pkg-config to configure it.
        env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)
        add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
                  ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
                  ' -lffi')

        info('Activating flags for openssl')
        recipe = Recipe.get_recipe('openssl', self.ctx)
        self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))
        add_flags(recipe.include_flags(arch),
                  recipe.link_dirs_flags(arch), recipe.link_libs_flags())

        for library_name in {'libbz2', 'liblzma'}:
            if library_name in self.ctx.recipe_build_order:
                info(f'Activating flags for {library_name}')
                recipe = Recipe.get_recipe(library_name, self.ctx)
                add_flags(recipe.get_library_includes(arch),
                          recipe.get_library_ldflags(arch),
                          recipe.get_library_libs_flag())

        # python build system contains hardcoded zlib version which prevents
        # the build of zlib module, here we search for android's zlib version
        # and sets the right flags, so python can be build with android's zlib
        info("Activating flags for android's zlib")
        zlib_lib_path = arch.ndk_lib_dir_versioned
        zlib_includes = self.ctx.ndk.sysroot_include_dir
        zlib_h = join(zlib_includes, 'zlib.h')
        try:
            with open(zlib_h) as fileh:
                zlib_data = fileh.read()
        except IOError:
            raise BuildInterruptingException(
                "Could not determine android's zlib version, no zlib.h ({}) in"
                " the NDK dir includes".format(zlib_h)
            )
        for line in zlib_data.split('\n'):
            if line.startswith('#define ZLIB_VERSION '):
                break
        else:
            raise BuildInterruptingException(
                'Could not parse zlib.h...so we cannot find zlib version,'
                'required by python build,'
            )
        env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
        add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')

        _p_version = Version(self.version)
        if _p_version.minor >= 11:
            self.configure_args.append('--with-build-python={python_host_bin}')

        if _p_version.minor >= 13 and self.disable_gil:
            self.configure_args.append("--disable-gil")

        self.configure_args = list(dict.fromkeys(self.configure_args))  # preserve order for reproducibility

        return env

    def build_arch(self, arch):
        if self.ctx.ndk_api < self.MIN_NDK_API:
            raise BuildInterruptingException(
                NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
                    ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
                ),
            )

        recipe_build_dir = self.get_build_dir(arch.arch)

        # Create a subdirectory to actually perform the build
        build_dir = join(recipe_build_dir, 'android-build')
        ensure_dir(build_dir)

        sys_prefix = join(build_dir, "android-root")
        ensure_dir(sys_prefix)

        env = self.get_recipe_env(arch)
        env = self.set_libs_flags(env, arch)

        android_build = sh.Command(
            join(recipe_build_dir,
                 'config.guess'))().strip()

        with current_directory(build_dir):
            if not exists('config.status'):
                shprint(
                    sh.Command(join(recipe_build_dir, 'configure')),
                    *(' '.join(self.configure_args).format(
                                    android_host=env['HOSTARCH'],
                                    android_build=android_build,
                                    python_host_bin=self.get_recipe(
                                        'host' + self.name, self.ctx
                                    ).python_exe,
                                    prefix=sys_prefix).split(' ')),
                    _env=env)

            shprint(
                sh.make,
                'all',
                'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
                _env=env
            )
            shprint(sh.make, 'install', _env=env)

            # rename executable
            if isfile("python"):
                sh.cp('python', 'libpythonbin.so')
            elif isfile("python.exe"):  # for macos
                sh.cp('python.exe', 'libpythonbin.so')

            # TODO: Look into passing the path to pyconfig.h in a
            # better way, although this is probably acceptable
            sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))

    def compile_python_files(self, dir):
        '''
        Compile the python files (recursively) for the python files inside
        a given folder.

        .. note:: python2 compiles the files into extension .pyo, but in
            python3, and as of Python 3.5, the .pyo filename extension is no
            longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
        '''
        args = [self.ctx.hostpython]
        args += ['-OO', '-m', 'compileall', '-b', '-f', '-q', dir]
        subprocess.call(args)

    def create_python_bundle(self, dirn, arch):
        """
        Create a packaged python bundle in the target directory, by
        copying all the modules and standard library to the right
        place.
        """
        modules_build_dir = glob.glob(join(
            self.get_build_dir(arch.arch),
            'android-build',
            'build',
            'lib.*'
        ))[0]
        # Compile to *.pyc the python modules
        self.compile_python_files(modules_build_dir)
        # Compile to *.pyc the standard python library
        self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
        # Compile to *.pyc the other python packages (site-packages)
        self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))

        # Bundle compiled python modules to a folder
        modules_dir = join(dirn, 'modules')
        c_ext = self.compiled_extension
        ensure_dir(modules_dir)
        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
                         glob.glob(join(modules_build_dir, '*' + c_ext)))
        info("Copy {} files into the bundle".format(len(module_filens)))
        for filen in module_filens:
            info(" - copy {}".format(filen))
            shutil.copy2(filen, modules_dir)

        # zip up the standard library
        stdlib_zip = join(dirn, 'stdlib.zip')
        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
            stdlib_filens = list(walk_valid_filens(
                '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
            if 'SOURCE_DATE_EPOCH' in environ:
                # for reproducible builds
                stdlib_filens.sort()
                timestamp = int(environ['SOURCE_DATE_EPOCH'])
                for filen in stdlib_filens:
                    utime(filen, (timestamp, timestamp))
            info("Zip {} files into the bundle".format(len(stdlib_filens)))
            shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)

        # copy the site-packages into place
        ensure_dir(join(dirn, 'site-packages'))
        ensure_dir(self.ctx.get_python_install_dir(arch.arch))
        # TODO: Improve the API around walking and copying the files
        with current_directory(self.ctx.get_python_install_dir(arch.arch)):
            filens = list(walk_valid_filens(
                '.', self.site_packages_dir_blacklist,
                self.site_packages_filen_blacklist,
                excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
            info("Copy {} files into the site-packages".format(len(filens)))
            for filen in filens:
                info(" - copy {}".format(filen))
                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
                shutil.copy2(filen, join(dirn, 'site-packages', filen))

        # copy the python .so files into place
        python_build_dir = join(self.get_build_dir(arch.arch),
                                'android-build')
        python_lib_name = 'libpython' + self.link_version
        shprint(
            sh.cp,
            join(python_build_dir, python_lib_name + '.so'),
            join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
        )

        info('Renaming .so files to reflect cross-compile')
        self.reduce_object_file_names(join(dirn, 'site-packages'))

        return join(dirn, 'site-packages')


recipe = Python3Recipe()

1	import glob	6✔
2	import sh	6✔
3	import subprocess	6✔
4
5	from os import environ, utime	6✔
6	from os.path import dirname, exists, join, isfile	6✔
7	import shutil	6✔
8
9	from packaging.version import Version	6✔
10	from pythonforandroid.logger import info, shprint, warning	6✔
11	from pythonforandroid.recipe import Recipe, TargetPythonRecipe	6✔
12	from pythonforandroid.util import (	6✔
13	current_directory,
14	ensure_dir,
15	walk_valid_filens,
16	BuildInterruptingException,
17	)
18
19	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (	6✔
20	'Target ndk-api is {ndk_api}, '
21	'but the python3 recipe supports only {min_ndk_api}+'
22	)
23
24
25	class Python3Recipe(TargetPythonRecipe):	6✔
26	'''
27	The python3's recipe
28	^^^^^^^^^^^^^^^^^^^^
29
30	The python 3 recipe can be built with some extra python modules, but to do
31	so, we need some libraries. By default, we ship the python3 recipe with
32	some common libraries, defined in ``depends``. We also support some optional
33	libraries, which are less common that the ones defined in ``depends``, so
34	we added them as optional dependencies (``opt_depends``).
35
36	Below you have a relationship between the python modules and the recipe
37	libraries::
38
39	- _ctypes: you must add the recipe for ``libffi``.
40	- _sqlite3: you must add the recipe for ``sqlite3``.
41	- _ssl: you must add the recipe for ``openssl``.
42	- _bz2: you must add the recipe for ``libbz2`` (optional).
43	- _lzma: you must add the recipe for ``liblzma`` (optional).
44
45	.. note:: This recipe can be built only against API 21+.
46
47	.. versionchanged:: 2019.10.06.post0
48	- Refactored from deleted class ``python.GuestPythonRecipe`` into here
49	- Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
50	and :mod:`~pythonforandroid.recipes.liblzma`
51
52	.. versionchanged:: 0.6.0
53	Refactored into class
54	:class:`~pythonforandroid.python.GuestPythonRecipe`
55	'''
56
57	version = '3.14.2'	6✔
58	url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'	6✔
59	name = 'python3'	6✔
60
61	patches = [	6✔
62	'patches/pyconfig_detection.patch',
63	'patches/reproducible-buildinfo.diff',
64	]
65
66	depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']	6✔
67	# those optional depends allow us to build python compression modules:
68	# - _bz2.so
69	# - _lzma.so
70	opt_depends = ['libbz2', 'liblzma']	6✔
71	'''The optional libraries which we would like to get our python linked'''	6✔
72
73	configure_args = [	6✔
74	'--host={android_host}',
75	'--build={android_build}',
76	'--enable-ipv6',
77	'--enable-loadable-sqlite-extensions',
78	'--enable-shared',
79
80	# Attempt on making the builds lighter
81	'--disable-test-modules',
82	'--without-c-locale-coercion',
83	'--without-decimal-contextvar',
84	'--without-doc-strings',
85	'--without-ensurepip',
86	'--without-readline',
87	'--without-static-libpython',
88
89	# Android prefix
90	'--prefix={prefix}',
91
92	# Special cross compile args
93	'ac_cv_header_bzlib_h=no',
94	'ac_cv_header_sys_eventfd_h=no',
95	'py_cv_module__curses=n/a',
96	'py_cv_module__curses_panel=n/a',
97	'py_cv_module__tkinter=n/a'
98	]
99
100	'''The configure arguments needed to build the python recipe. Those are	6✔
101	used in method :meth:`build_arch` (if not overwritten like python3's
102	recipe does).
103	'''
104
105	MIN_NDK_API = 21	6✔
106	'''Sets the minimal ndk api number needed to use the recipe.	6✔
107
108	.. warning:: Starting from Python 3.14 this recipe can only be built
109	against API 21+, so it means that any class which inherits from
110	class:`GuestPythonRecipe` will have this limitation.
111	'''
112
113	stdlib_dir_blacklist = {	6✔
114	'__pycache__',
115	'curses',
116	'ensurepip',
117	'idlelib',
118	'lib2to3',
119	'msilib',
120	'multiprocessing',
121	'pydoc_data',
122	'test',
123	'tests',
124	'tkinter',
125	'turtledemo',
126	'venv'
127	}
128	'''The directories that we want to omit for our python bundle'''	6✔
129
130	stdlib_filen_blacklist = [	6✔
131	'*.exe',
132	'*.py',
133	'*.whl',
134	'turtle.pyc'
135	]
136	'''The file extensions that we want to blacklist for our python bundle'''	6✔
137
138	site_packages_dir_blacklist = {	6✔
139	'__pycache__',
140	'*.dist-info',
141	'bin',
142	'tests',
143	'setuptools',
144	'_distutils_hack'
145	}
146	'''The directories from site packages dir that we don't want to be included	6✔
147	in our python bundle.'''
148
149	site_packages_excluded_dir_exceptions = [	6✔
150	# 'numpy' is excluded here because importing with `import numpy as np`
151	# can fail if the `tests` directory inside the numpy package is excluded.
152	'numpy',
153	]
154	'''Directories from `site_packages_dir_blacklist` will not be excluded	6✔
155	if the full path contains any of these exceptions.'''
156
157	site_packages_filen_blacklist = [	6✔
158	'*.py',
159	'*.pyx'
160	]
161	'''The file extensions from site packages dir that we don't want to be	6✔
162	included in our python bundle.'''
163
164	compiled_extension = '.pyc'	6✔
165	'''the default extension for compiled python files.	6✔
166
167	.. note:: the default extension for compiled python files has been .pyo for
168	python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
169	longer used and has been removed in favour of extension .pyc
170	'''
171
172	disable_gil = False	6✔
173	'''python3.13 experimental free-threading build'''	6✔
174
175	built_libraries = {"libpythonbin.so": "./android-build/"}	6✔
176
177	def __init__(self, args, *kwargs):	6✔
178	self._ctx = None	6✔
179	super().__init__(args, *kwargs)	6✔
180
181	@property	6✔
182	def _libpython(self):	6✔
183	'''return the python's library name (with extension)'''
184	return 'libpython{link_version}.so'.format(	6✔
185	link_version=self.link_version
186	)
187
188	@property	6✔
189	def link_version(self):	6✔
190	'''return the python's library link version e.g. 3.7m, 3.8'''
191	major, minor = self.major_minor_version_string.split('.')	6✔
192	flags = ''	6✔
193	if major == '3' and int(minor) < 8:	6!
UNCOV 194	flags += 'm'	×
195	return '{major}.{minor}{flags}'.format(	6✔
196	major=major,
197	minor=minor,
198	flags=flags
199	)
200
201	def apply_patches(self, arch, build_dir=None):	6✔
202
UNCOV 203	_p_version = Version(self.version)	×
204	if _p_version.major == 3 and _p_version.minor == 7:	×
205	self.patches += [	×
206	'patches/py3.7.1_fix-ctypes-util-find-library.patch',
207	'patches/py3.7.1_fix-zlib-version.patch',
208	]
209
210	if 8 <= _p_version.minor <= 10:	×
211	self.patches.append('patches/py3.8.1.patch')	×
212
UNCOV 213	if _p_version.minor >= 11:	×
214	self.patches.append('patches/cpython-311-ctypes-find-library.patch')	×
215
216	if _p_version.minor >= 14:	×
217	self.patches.append('patches/3.14_armv7l_fix.patch')	×
218	self.patches.append('patches/3.14_fix_remote_debug.patch')	×
219
220	if shutil.which('lld') is not None:	×
221	if _p_version.minor == 7:	×
UNCOV 222	self.patches.append("patches/py3.7.1_fix_cortex_a8.patch")	×
UNCOV 223	elif _p_version.minor >= 8:	×
UNCOV 224	self.patches.append("patches/py3.8.1_fix_cortex_a8.patch")	×
225
UNCOV 226	self.patches = list(dict.fromkeys(self.patches)) # preserve order for reproducibility	×
UNCOV 227	super().apply_patches(arch, build_dir)	×
228
229	def include_root(self, arch_name):	6✔
230	_p_version = Version(self.version)	6✔
231	return join(	6✔
232	self.get_build_dir(arch_name), 'android-build', 'android-root',
233	'include', f'python{_p_version.major}.{_p_version.minor}'
234	)
235
236	def link_root(self, arch_name):	6✔
237	return join(self.get_build_dir(arch_name), 'android-build')	6✔
238
239	def get_python_root(self, arch):	6✔
240	return join(self.get_build_dir(arch.arch), 'android-build', 'android-root')	×
241
242	def get_android_python_exe(self, arch):	6✔
243	return join(self.get_python_root(arch), 'bin', self.name)	×
244
245	def should_build(self, arch):	6✔
UNCOV 246	return not isfile(join(self.link_root(arch.arch), self._libpython))	×
247
248	def prebuild_arch(self, arch):	6✔
UNCOV 249	super().prebuild_arch(arch)	×
NEW 250	self.ctx.python_recipe = self	×
251
252	def get_recipe_env(self, arch=None, with_flags_in_cc=True):	6✔
253	env = super().get_recipe_env(arch)	6✔
254	env['HOSTARCH'] = arch.command_prefix	6✔
255	env['CC'] = arch.get_clang_exe(with_target=True)	6✔
256	env['PATH'] = '{hostpython_dir}:{old_path}'.format(	6✔
257	hostpython_dir=self.get_recipe(
258	'host' + self.name, self.ctx
259	).get_path_to_python(),
260	old_path=env['PATH']
261	)
262	env['CFLAGS'] = ' '.join(	6✔
263	[
264	'-ffunction-sections',
265	'-fdata-sections',
266	'-fPIC'
267	]
268	)
269
270	env['LDFLAGS'] = env.get('LDFLAGS', '')	6✔
271	if shutil.which('lld') is not None:	6!
272	# Note: The -L. is to fix a bug in python 3.7.
273	# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
274	env['LDFLAGS'] += ' -L. -fuse-ld=lld'	6✔
275	else:
UNCOV 276	warning('lld not found, linking without it. '	×
277	'Consider installing lld if linker errors occur.')
278
279	return env	6✔
280
281	def set_libs_flags(self, env, arch):	6✔
282	'''Takes care to properly link libraries with python depending on our
283	requirements and the attribute :attr:`opt_depends`.
284	'''
285	def add_flags(include_flags, link_dirs, link_libs):	6✔
286	env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags	6✔
287	env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs	6✔
288	env['LIBS'] = env.get('LIBS', '') + link_libs	6✔
289
290	info('Activating flags for sqlite3')	6✔
291	recipe = Recipe.get_recipe('sqlite3', self.ctx)	6✔
292	add_flags(' -I' + recipe.get_build_dir(arch.arch),	6✔
293	' -L' + recipe.get_build_dir(arch.arch), ' -lsqlite3')
294
295	info('Activating flags for libffi')	6✔
296	recipe = Recipe.get_recipe('libffi', self.ctx)	6✔
297	# In order to force the correct linkage for our libffi library, we
298	# set the following variable to point where is our libffi.pc file,
299	# because the python build system uses pkg-config to configure it.
300	env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)	6✔
301	add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),	6✔
302	' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
303	' -lffi')
304
305	info('Activating flags for openssl')	6✔
306	recipe = Recipe.get_recipe('openssl', self.ctx)	6✔
307	self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))	6✔
308	add_flags(recipe.include_flags(arch),	6✔
309	recipe.link_dirs_flags(arch), recipe.link_libs_flags())
310
311	for library_name in {'libbz2', 'liblzma'}:	6✔
312	if library_name in self.ctx.recipe_build_order:	6!
UNCOV 313	info(f'Activating flags for {library_name}')	×
UNCOV 314	recipe = Recipe.get_recipe(library_name, self.ctx)	×
UNCOV 315	add_flags(recipe.get_library_includes(arch),	×
316	recipe.get_library_ldflags(arch),
317	recipe.get_library_libs_flag())
318
319	# python build system contains hardcoded zlib version which prevents
320	# the build of zlib module, here we search for android's zlib version
321	# and sets the right flags, so python can be build with android's zlib
322	info("Activating flags for android's zlib")	6✔
323	zlib_lib_path = arch.ndk_lib_dir_versioned	6✔
324	zlib_includes = self.ctx.ndk.sysroot_include_dir	6✔
325	zlib_h = join(zlib_includes, 'zlib.h')	6✔
326	try:	6✔
327	with open(zlib_h) as fileh:	6✔
328	zlib_data = fileh.read()	6✔
UNCOV 329	except IOError:	×
UNCOV 330	raise BuildInterruptingException(	×
331	"Could not determine android's zlib version, no zlib.h ({}) in"
332	" the NDK dir includes".format(zlib_h)
333	)
334	for line in zlib_data.split('\n'):	6!
335	if line.startswith('#define ZLIB_VERSION '):	6!
336	break	6✔
337	else:
UNCOV 338	raise BuildInterruptingException(	×
339	'Could not parse zlib.h...so we cannot find zlib version,'
340	'required by python build,'
341	)
342	env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')	6✔
343	add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')	6✔
344
345	_p_version = Version(self.version)	6✔
346	if _p_version.minor >= 11:	6!
347	self.configure_args.append('--with-build-python={python_host_bin}')	6✔
348
349	if _p_version.minor >= 13 and self.disable_gil:	6!
UNCOV 350	self.configure_args.append("--disable-gil")	×
351
352	self.configure_args = list(dict.fromkeys(self.configure_args)) # preserve order for reproducibility	6✔
353
354	return env	6✔
355
356	def build_arch(self, arch):	6✔
357	if self.ctx.ndk_api < self.MIN_NDK_API:	6✔
358	raise BuildInterruptingException(	6✔
359	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
360	ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
361	),
362	)
363
364	recipe_build_dir = self.get_build_dir(arch.arch)	6✔
365
366	# Create a subdirectory to actually perform the build
367	build_dir = join(recipe_build_dir, 'android-build')	6✔
368	ensure_dir(build_dir)	6✔
369
370	sys_prefix = join(build_dir, "android-root")	6✔
371	ensure_dir(sys_prefix)	6✔
372
373	env = self.get_recipe_env(arch)	6✔
374	env = self.set_libs_flags(env, arch)	6✔
375
376	android_build = sh.Command(	6✔
377	join(recipe_build_dir,
378	'config.guess'))().strip()
379
380	with current_directory(build_dir):	6✔
381	if not exists('config.status'):	6!
382	shprint(	6✔
383	sh.Command(join(recipe_build_dir, 'configure')),
384	*(' '.join(self.configure_args).format(
385	android_host=env['HOSTARCH'],
386	android_build=android_build,
387	python_host_bin=self.get_recipe(
388	'host' + self.name, self.ctx
389	).python_exe,
390	prefix=sys_prefix).split(' ')),
391	_env=env)
392
393	shprint(	6✔
394	sh.make,
395	'all',
396	'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
397	_env=env
398	)
399	shprint(sh.make, 'install', _env=env)	6✔
400
401	# rename executable
402	if isfile("python"):	6!
UNCOV 403	sh.cp('python', 'libpythonbin.so')	×
404	elif isfile("python.exe"): # for macos	6!
UNCOV 405	sh.cp('python.exe', 'libpythonbin.so')	×
406
407	# TODO: Look into passing the path to pyconfig.h in a
408	# better way, although this is probably acceptable
409	sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))	6✔
410
411	def compile_python_files(self, dir):	6✔
412	'''
413	Compile the python files (recursively) for the python files inside
414	a given folder.
415
416	.. note:: python2 compiles the files into extension .pyo, but in
417	python3, and as of Python 3.5, the .pyo filename extension is no
418	longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
419	'''
420	args = [self.ctx.hostpython]	6✔
421	args += ['-OO', '-m', 'compileall', '-b', '-f', '-q', dir]	6✔
422	subprocess.call(args)	6✔
423
424	def create_python_bundle(self, dirn, arch):	6✔
425	"""
426	Create a packaged python bundle in the target directory, by
427	copying all the modules and standard library to the right
428	place.
429	"""
UNCOV 430	modules_build_dir = glob.glob(join(	×
431	self.get_build_dir(arch.arch),
432	'android-build',
433	'build',
434	'lib.*'
435	))[0]
436	# Compile to *.pyc the python modules
UNCOV 437	self.compile_python_files(modules_build_dir)	×
438	# Compile to *.pyc the standard python library
439	self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))	×
440	# Compile to *.pyc the other python packages (site-packages)
441	self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))	×
442
443	# Bundle compiled python modules to a folder
444	modules_dir = join(dirn, 'modules')	×
445	c_ext = self.compiled_extension	×
446	ensure_dir(modules_dir)	×
UNCOV 447	module_filens = (glob.glob(join(modules_build_dir, '*.so')) +	×
448	glob.glob(join(modules_build_dir, '*' + c_ext)))
449	info("Copy {} files into the bundle".format(len(module_filens)))	×
450	for filen in module_filens:	×
451	info(" - copy {}".format(filen))	×
UNCOV 452	shutil.copy2(filen, modules_dir)	×
453
454	# zip up the standard library
455	stdlib_zip = join(dirn, 'stdlib.zip')	×
456	with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):	×
457	stdlib_filens = list(walk_valid_filens(	×
458	'.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
459	if 'SOURCE_DATE_EPOCH' in environ:	×
460	# for reproducible builds
UNCOV 461	stdlib_filens.sort()	×
UNCOV 462	timestamp = int(environ['SOURCE_DATE_EPOCH'])	×
463	for filen in stdlib_filens:	×
464	utime(filen, (timestamp, timestamp))	×
UNCOV 465	info("Zip {} files into the bundle".format(len(stdlib_filens)))	×
466	shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)	×
467
468	# copy the site-packages into place
UNCOV 469	ensure_dir(join(dirn, 'site-packages'))	×
UNCOV 470	ensure_dir(self.ctx.get_python_install_dir(arch.arch))	×
471	# TODO: Improve the API around walking and copying the files
472	with current_directory(self.ctx.get_python_install_dir(arch.arch)):	×
473	filens = list(walk_valid_filens(	×
474	'.', self.site_packages_dir_blacklist,
475	self.site_packages_filen_blacklist,
476	excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
UNCOV 477	info("Copy {} files into the site-packages".format(len(filens)))	×
478	for filen in filens:	×
UNCOV 479	info(" - copy {}".format(filen))	×
480	ensure_dir(join(dirn, 'site-packages', dirname(filen)))	×
481	shutil.copy2(filen, join(dirn, 'site-packages', filen))	×
482
483	# copy the python .so files into place
UNCOV 484	python_build_dir = join(self.get_build_dir(arch.arch),	×
485	'android-build')
UNCOV 486	python_lib_name = 'libpython' + self.link_version	×
487	shprint(	×
488	sh.cp,
489	join(python_build_dir, python_lib_name + '.so'),
490	join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
491	)
492
UNCOV 493	info('Renaming .so files to reflect cross-compile')	×
UNCOV 494	self.reduce_object_file_names(join(dirn, 'site-packages'))	×
495
UNCOV 496	return join(dirn, 'site-packages')	×
497
498
499	recipe = Python3Recipe()	6✔

kivy / python-for-android / 36255542538

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous