24930372022

Committed 25 Apr 2026 11:58AM UTC coverage: 62.932% (-0.5%) from 63.382%

Build # 24930372022

Build Type

push

github

Committed by

web-flow

Commit Message

fix PYTHONPATH hacks (#3301)

* fix PYTHONPATH hacks

* fix android recipe

* fix numpy and matplotlib build

* no shebang patching required now

* fix kivy master build

* fix numpy build

* more wrappers for meson recipe

* flake8 fix

* add back pandas host preq

* fix bug in logger

* add back cython for pandas

Coverage Stats

1832 of 3175 branches covered (57.7%)

Branch coverage included in aggregate %.

61 of 150 new or added lines in 8 files covered. (40.67%)

9 existing lines in 2 files now uncovered.

5329 of 8204 relevant lines covered (64.96%)

5.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.45

/pythonforandroid/recipes/python3/__init__.py

import glob
import sh
import subprocess

from os import environ, utime
from os.path import dirname, exists, join, isfile
import shutil

from packaging.version import Version
from pythonforandroid.logger import info, shprint, warning
from pythonforandroid.recipe import Recipe, TargetPythonRecipe
from pythonforandroid.util import (
    current_directory,
    ensure_dir,
    walk_valid_filens,
    BuildInterruptingException,
)

NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (
    'Target ndk-api is {ndk_api}, '
    'but the python3 recipe supports only {min_ndk_api}+'
)


class Python3Recipe(TargetPythonRecipe):
    '''
    The python3's recipe
    ^^^^^^^^^^^^^^^^^^^^

    The python 3 recipe can be built with some extra python modules, but to do
    so, we need some libraries. By default, we ship the python3 recipe with
    some common libraries, defined in ``depends``. We also support some optional
    libraries, which are less common that the ones defined in ``depends``, so
    we added them as optional dependencies (``opt_depends``).

    Below you have a relationship between the python modules and the recipe
    libraries::

        - _ctypes: you must add the recipe for ``libffi``.
        - _sqlite3: you must add the recipe for ``sqlite3``.
        - _ssl: you must add the recipe for ``openssl``.
        - _bz2: you must add the recipe for ``libbz2`` (optional).
        - _lzma: you must add the recipe for ``liblzma`` (optional).

    .. note:: This recipe can be built only against API 21+.

    .. versionchanged:: 2019.10.06.post0
        - Refactored from deleted class ``python.GuestPythonRecipe`` into here
        - Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
          and :mod:`~pythonforandroid.recipes.liblzma`

    .. versionchanged:: 0.6.0
        Refactored into class
        :class:`~pythonforandroid.python.GuestPythonRecipe`
    '''

    version = '3.14.2'
    url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
    name = 'python3'

    patches = [
        'patches/pyconfig_detection.patch',
        'patches/reproducible-buildinfo.diff',
    ]

    depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
    # those optional depends allow us to build python compression modules:
    #   - _bz2.so
    #   - _lzma.so
    opt_depends = ['libbz2', 'liblzma']
    '''The optional libraries which we would like to get our python linked'''

    configure_args = [
        '--host={android_host}',
        '--build={android_build}',
        '--enable-shared',
        '--enable-ipv6',
        '--enable-loadable-sqlite-extensions',
        '--without-static-libpython',
        '--without-readline',
        '--without-ensurepip',

        # Android prefix
        '--prefix={prefix}',
        '--enable-loadable-sqlite-extensions',

        # Special cross compile args
        'ac_cv_file__dev_ptmx=yes',
        'ac_cv_file__dev_ptc=no',
        'ac_cv_header_sys_eventfd_h=no',
        'ac_cv_little_endian_double=yes',
        'ac_cv_header_bzlib_h=no',
    ]

    '''The configure arguments needed to build the python recipe. Those are
    used in method :meth:`build_arch` (if not overwritten like python3's
    recipe does).
    '''

    MIN_NDK_API = 21
    '''Sets the minimal ndk api number needed to use the recipe.

    .. warning:: This recipe can be built only against API 21+, so it means
        that any class which inherits from class:`GuestPythonRecipe` will have
        this limitation.
    '''

    stdlib_dir_blacklist = {
        '__pycache__',
        'test',
        'tests',
        'lib2to3',
        'ensurepip',
        'idlelib',
        'tkinter',
    }
    '''The directories that we want to omit for our python bundle'''

    stdlib_filen_blacklist = [
        '*.py',
        '*.exe',
        '*.whl',
    ]
    '''The file extensions that we want to blacklist for our python bundle'''

    site_packages_dir_blacklist = {
        '__pycache__',
        'tests'
    }
    '''The directories from site packages dir that we don't want to be included
    in our python bundle.'''

    site_packages_excluded_dir_exceptions = [
        # 'numpy' is excluded here because importing with `import numpy as np`
        # can fail if the `tests` directory inside the numpy package is excluded.
        'numpy',
    ]
    '''Directories from `site_packages_dir_blacklist` will not be excluded
    if the full path contains any of these exceptions.'''

    site_packages_filen_blacklist = [
        '*.py'
    ]
    '''The file extensions from site packages dir that we don't want to be
    included in our python bundle.'''

    compiled_extension = '.pyc'
    '''the default extension for compiled python files.

    .. note:: the default extension for compiled python files has been .pyo for
        python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
        longer used and has been removed in favour of extension .pyc
    '''

    disable_gil = False
    '''python3.13 experimental free-threading build'''

    built_libraries = {"libpythonbin.so": "./android-build/"}

    def __init__(self, *args, **kwargs):
        self._ctx = None
        super().__init__(*args, **kwargs)

    @property
    def _libpython(self):
        '''return the python's library name (with extension)'''
        return 'libpython{link_version}.so'.format(
            link_version=self.link_version
        )

    @property
    def link_version(self):
        '''return the python's library link version e.g. 3.7m, 3.8'''
        major, minor = self.major_minor_version_string.split('.')
        flags = ''
        if major == '3' and int(minor) < 8:
            flags += 'm'
        return '{major}.{minor}{flags}'.format(
            major=major,
            minor=minor,
            flags=flags
        )

    def apply_patches(self, arch, build_dir=None):

        _p_version = Version(self.version)
        if _p_version.major == 3 and _p_version.minor == 7:
            self.patches += [
                'patches/py3.7.1_fix-ctypes-util-find-library.patch',
                'patches/py3.7.1_fix-zlib-version.patch',
            ]

        if 8 <= _p_version.minor <= 10:
            self.patches.append('patches/py3.8.1.patch')

        if _p_version.minor >= 11:
            self.patches.append('patches/cpython-311-ctypes-find-library.patch')

        if _p_version.minor >= 14:
            self.patches.append('patches/3.14_armv7l_fix.patch')
            self.patches.append('patches/3.14_fix_remote_debug.patch')

        if shutil.which('lld') is not None:
            if _p_version.minor == 7:
                self.patches.append("patches/py3.7.1_fix_cortex_a8.patch")
            elif _p_version.minor >= 8:
                self.patches.append("patches/py3.8.1_fix_cortex_a8.patch")

        self.patches = list(set(self.patches))
        super().apply_patches(arch, build_dir)

    def include_root(self, arch_name):
        _p_version = Version(self.version)
        return join(
            self.get_build_dir(arch_name), 'android-build', 'android-root',
            'include', f'python{_p_version.major}.{_p_version.minor}'
        )

    def link_root(self, arch_name):
        return join(self.get_build_dir(arch_name), 'android-build')

    def get_python_root(self, arch):
        return join(self.get_build_dir(arch.arch), 'android-build', 'android-root')

    def get_android_python_exe(self, arch):
        return join(self.get_python_root(arch), 'bin', self.name)

    def should_build(self, arch):
        return not isfile(join(self.link_root(arch.arch), self._libpython))

    def prebuild_arch(self, arch):
        super().prebuild_arch(arch)
        self.ctx.python_recipe = self

    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
        env = super().get_recipe_env(arch)
        env['HOSTARCH'] = arch.command_prefix

        env['CC'] = arch.get_clang_exe(with_target=True)

        env['PATH'] = (
            '{hostpython_dir}:{old_path}').format(
                hostpython_dir=self.get_recipe(
                    'host' + self.name, self.ctx).get_path_to_python(),
                old_path=env['PATH'])

        env['CFLAGS'] = ' '.join(
            [
                '-fPIC',
                '-DANDROID'
            ]
        )

        env['LDFLAGS'] = env.get('LDFLAGS', '')
        if shutil.which('lld') is not None:
            # Note: The -L. is to fix a bug in python 3.7.
            # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
            env['LDFLAGS'] += ' -L. -fuse-ld=lld'
        else:
            warning('lld not found, linking without it. '
                    'Consider installing lld if linker errors occur.')

        return env

    def set_libs_flags(self, env, arch):
        '''Takes care to properly link libraries with python depending on our
        requirements and the attribute :attr:`opt_depends`.
        '''
        def add_flags(include_flags, link_dirs, link_libs):
            env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
            env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
            env['LIBS'] = env.get('LIBS', '') + link_libs

        info('Activating flags for sqlite3')
        recipe = Recipe.get_recipe('sqlite3', self.ctx)
        add_flags(' -I' + recipe.get_build_dir(arch.arch),
                  ' -L' + recipe.get_build_dir(arch.arch), ' -lsqlite3')

        info('Activating flags for libffi')
        recipe = Recipe.get_recipe('libffi', self.ctx)
        # In order to force the correct linkage for our libffi library, we
        # set the following variable to point where is our libffi.pc file,
        # because the python build system uses pkg-config to configure it.
        env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)
        add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
                  ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
                  ' -lffi')

        info('Activating flags for openssl')
        recipe = Recipe.get_recipe('openssl', self.ctx)
        self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))
        add_flags(recipe.include_flags(arch),
                  recipe.link_dirs_flags(arch), recipe.link_libs_flags())

        for library_name in {'libbz2', 'liblzma'}:
            if library_name in self.ctx.recipe_build_order:
                info(f'Activating flags for {library_name}')
                recipe = Recipe.get_recipe(library_name, self.ctx)
                add_flags(recipe.get_library_includes(arch),
                          recipe.get_library_ldflags(arch),
                          recipe.get_library_libs_flag())

        # python build system contains hardcoded zlib version which prevents
        # the build of zlib module, here we search for android's zlib version
        # and sets the right flags, so python can be build with android's zlib
        info("Activating flags for android's zlib")
        zlib_lib_path = arch.ndk_lib_dir_versioned
        zlib_includes = self.ctx.ndk.sysroot_include_dir
        zlib_h = join(zlib_includes, 'zlib.h')
        try:
            with open(zlib_h) as fileh:
                zlib_data = fileh.read()
        except IOError:
            raise BuildInterruptingException(
                "Could not determine android's zlib version, no zlib.h ({}) in"
                " the NDK dir includes".format(zlib_h)
            )
        for line in zlib_data.split('\n'):
            if line.startswith('#define ZLIB_VERSION '):
                break
        else:
            raise BuildInterruptingException(
                'Could not parse zlib.h...so we cannot find zlib version,'
                'required by python build,'
            )
        env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
        add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')

        _p_version = Version(self.version)
        if _p_version.minor >= 11:
            self.configure_args.append('--with-build-python={python_host_bin}')

        if _p_version.minor >= 13 and self.disable_gil:
            self.configure_args.append("--disable-gil")

        self.configure_args = list(set(self.configure_args))

        return env

    def build_arch(self, arch):
        if self.ctx.ndk_api < self.MIN_NDK_API:
            raise BuildInterruptingException(
                NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
                    ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
                ),
            )

        recipe_build_dir = self.get_build_dir(arch.arch)

        # Create a subdirectory to actually perform the build
        build_dir = join(recipe_build_dir, 'android-build')
        ensure_dir(build_dir)

        sys_prefix = join(build_dir, "android-root")
        ensure_dir(sys_prefix)

        env = self.get_recipe_env(arch)
        env = self.set_libs_flags(env, arch)

        android_build = sh.Command(
            join(recipe_build_dir,
                 'config.guess'))().strip()

        with current_directory(build_dir):
            if not exists('config.status'):
                shprint(
                    sh.Command(join(recipe_build_dir, 'configure')),
                    *(' '.join(self.configure_args).format(
                                    android_host=env['HOSTARCH'],
                                    android_build=android_build,
                                    python_host_bin=self.get_recipe(
                                        'host' + self.name, self.ctx
                                    ).python_exe,
                                    prefix=sys_prefix).split(' ')),
                    _env=env)

            shprint(
                sh.make,
                'all',
                'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
                _env=env
            )
            shprint(sh.make, 'install', _env=env)

            # rename executable
            if isfile("python"):
                sh.cp('python', 'libpythonbin.so')
            elif isfile("python.exe"):  # for macos
                sh.cp('python.exe', 'libpythonbin.so')

            # TODO: Look into passing the path to pyconfig.h in a
            # better way, although this is probably acceptable
            sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))

    def compile_python_files(self, dir):
        '''
        Compile the python files (recursively) for the python files inside
        a given folder.

        .. note:: python2 compiles the files into extension .pyo, but in
            python3, and as of Python 3.5, the .pyo filename extension is no
            longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
        '''
        args = [self.ctx.hostpython]
        args += ['-OO', '-m', 'compileall', '-b', '-f', dir]
        subprocess.call(args)

    def create_python_bundle(self, dirn, arch):
        """
        Create a packaged python bundle in the target directory, by
        copying all the modules and standard library to the right
        place.
        """
        modules_build_dir = glob.glob(join(
            self.get_build_dir(arch.arch),
            'android-build',
            'build',
            'lib.*'
        ))[0]
        # Compile to *.pyc the python modules
        self.compile_python_files(modules_build_dir)
        # Compile to *.pyc the standard python library
        self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
        # Compile to *.pyc the other python packages (site-packages)
        self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))

        # Bundle compiled python modules to a folder
        modules_dir = join(dirn, 'modules')
        c_ext = self.compiled_extension
        ensure_dir(modules_dir)
        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
                         glob.glob(join(modules_build_dir, '*' + c_ext)))
        info("Copy {} files into the bundle".format(len(module_filens)))
        for filen in module_filens:
            info(" - copy {}".format(filen))
            shutil.copy2(filen, modules_dir)

        # zip up the standard library
        stdlib_zip = join(dirn, 'stdlib.zip')
        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
            stdlib_filens = list(walk_valid_filens(
                '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
            if 'SOURCE_DATE_EPOCH' in environ:
                # for reproducible builds
                stdlib_filens.sort()
                timestamp = int(environ['SOURCE_DATE_EPOCH'])
                for filen in stdlib_filens:
                    utime(filen, (timestamp, timestamp))
            info("Zip {} files into the bundle".format(len(stdlib_filens)))
            shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)

        # copy the site-packages into place
        ensure_dir(join(dirn, 'site-packages'))
        ensure_dir(self.ctx.get_python_install_dir(arch.arch))
        # TODO: Improve the API around walking and copying the files
        with current_directory(self.ctx.get_python_install_dir(arch.arch)):
            filens = list(walk_valid_filens(
                '.', self.site_packages_dir_blacklist,
                self.site_packages_filen_blacklist,
                excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
            info("Copy {} files into the site-packages".format(len(filens)))
            for filen in filens:
                info(" - copy {}".format(filen))
                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
                shutil.copy2(filen, join(dirn, 'site-packages', filen))

        # copy the python .so files into place
        python_build_dir = join(self.get_build_dir(arch.arch),
                                'android-build')
        python_lib_name = 'libpython' + self.link_version
        shprint(
            sh.cp,
            join(python_build_dir, python_lib_name + '.so'),
            join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
        )

        info('Renaming .so files to reflect cross-compile')
        self.reduce_object_file_names(join(dirn, 'site-packages'))

        return join(dirn, 'site-packages')


recipe = Python3Recipe()

1	import glob	8✔
2	import sh	8✔
3	import subprocess	8✔
4
5	from os import environ, utime	8✔
6	from os.path import dirname, exists, join, isfile	8✔
7	import shutil	8✔
8
9	from packaging.version import Version	8✔
10	from pythonforandroid.logger import info, shprint, warning	8✔
11	from pythonforandroid.recipe import Recipe, TargetPythonRecipe	8✔
12	from pythonforandroid.util import (	8✔
13	current_directory,
14	ensure_dir,
15	walk_valid_filens,
16	BuildInterruptingException,
17	)
18
19	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (	8✔
20	'Target ndk-api is {ndk_api}, '
21	'but the python3 recipe supports only {min_ndk_api}+'
22	)
23
24
25	class Python3Recipe(TargetPythonRecipe):	8✔
26	'''
27	The python3's recipe
28	^^^^^^^^^^^^^^^^^^^^
29
30	The python 3 recipe can be built with some extra python modules, but to do
31	so, we need some libraries. By default, we ship the python3 recipe with
32	some common libraries, defined in ``depends``. We also support some optional
33	libraries, which are less common that the ones defined in ``depends``, so
34	we added them as optional dependencies (``opt_depends``).
35
36	Below you have a relationship between the python modules and the recipe
37	libraries::
38
39	- _ctypes: you must add the recipe for ``libffi``.
40	- _sqlite3: you must add the recipe for ``sqlite3``.
41	- _ssl: you must add the recipe for ``openssl``.
42	- _bz2: you must add the recipe for ``libbz2`` (optional).
43	- _lzma: you must add the recipe for ``liblzma`` (optional).
44
45	.. note:: This recipe can be built only against API 21+.
46
47	.. versionchanged:: 2019.10.06.post0
48	- Refactored from deleted class ``python.GuestPythonRecipe`` into here
49	- Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
50	and :mod:`~pythonforandroid.recipes.liblzma`
51
52	.. versionchanged:: 0.6.0
53	Refactored into class
54	:class:`~pythonforandroid.python.GuestPythonRecipe`
55	'''
56
57	version = '3.14.2'	8✔
58	url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'	8✔
59	name = 'python3'	8✔
60
61	patches = [	8✔
62	'patches/pyconfig_detection.patch',
63	'patches/reproducible-buildinfo.diff',
64	]
65
66	depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']	8✔
67	# those optional depends allow us to build python compression modules:
68	# - _bz2.so
69	# - _lzma.so
70	opt_depends = ['libbz2', 'liblzma']	8✔
71	'''The optional libraries which we would like to get our python linked'''	6✔
72
73	configure_args = [	8✔
74	'--host={android_host}',
75	'--build={android_build}',
76	'--enable-shared',
77	'--enable-ipv6',
78	'--enable-loadable-sqlite-extensions',
79	'--without-static-libpython',
80	'--without-readline',
81	'--without-ensurepip',
82
83	# Android prefix
84	'--prefix={prefix}',
85	'--enable-loadable-sqlite-extensions',
86
87	# Special cross compile args
88	'ac_cv_file__dev_ptmx=yes',
89	'ac_cv_file__dev_ptc=no',
90	'ac_cv_header_sys_eventfd_h=no',
91	'ac_cv_little_endian_double=yes',
92	'ac_cv_header_bzlib_h=no',
93	]
94
95	'''The configure arguments needed to build the python recipe. Those are	6✔
96	used in method :meth:`build_arch` (if not overwritten like python3's
97	recipe does).
98	'''
99
100	MIN_NDK_API = 21	8✔
101	'''Sets the minimal ndk api number needed to use the recipe.	6✔
102
103	.. warning:: This recipe can be built only against API 21+, so it means
104	that any class which inherits from class:`GuestPythonRecipe` will have
105	this limitation.
106	'''
107
108	stdlib_dir_blacklist = {	8✔
109	'__pycache__',
110	'test',
111	'tests',
112	'lib2to3',
113	'ensurepip',
114	'idlelib',
115	'tkinter',
116	}
117	'''The directories that we want to omit for our python bundle'''	6✔
118
119	stdlib_filen_blacklist = [	8✔
120	'*.py',
121	'*.exe',
122	'*.whl',
123	]
124	'''The file extensions that we want to blacklist for our python bundle'''	6✔
125
126	site_packages_dir_blacklist = {	8✔
127	'__pycache__',
128	'tests'
129	}
130	'''The directories from site packages dir that we don't want to be included	6✔
131	in our python bundle.'''
132
133	site_packages_excluded_dir_exceptions = [	8✔
134	# 'numpy' is excluded here because importing with `import numpy as np`
135	# can fail if the `tests` directory inside the numpy package is excluded.
136	'numpy',
137	]
138	'''Directories from `site_packages_dir_blacklist` will not be excluded	6✔
139	if the full path contains any of these exceptions.'''
140
141	site_packages_filen_blacklist = [	8✔
142	'*.py'
143	]
144	'''The file extensions from site packages dir that we don't want to be	6✔
145	included in our python bundle.'''
146
147	compiled_extension = '.pyc'	8✔
148	'''the default extension for compiled python files.	6✔
149
150	.. note:: the default extension for compiled python files has been .pyo for
151	python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
152	longer used and has been removed in favour of extension .pyc
153	'''
154
155	disable_gil = False	8✔
156	'''python3.13 experimental free-threading build'''	6✔
157
158	built_libraries = {"libpythonbin.so": "./android-build/"}	8✔
159
160	def __init__(self, args, *kwargs):	8✔
161	self._ctx = None	8✔
162	super().__init__(args, *kwargs)	8✔
163
164	@property	8✔
165	def _libpython(self):	8✔
166	'''return the python's library name (with extension)'''
167	return 'libpython{link_version}.so'.format(	8✔
168	link_version=self.link_version
169	)
170
171	@property	8✔
172	def link_version(self):	8✔
173	'''return the python's library link version e.g. 3.7m, 3.8'''
174	major, minor = self.major_minor_version_string.split('.')	8✔
175	flags = ''	8✔
176	if major == '3' and int(minor) < 8:	8!
177	flags += 'm'	×
178	return '{major}.{minor}{flags}'.format(	8✔
179	major=major,
180	minor=minor,
181	flags=flags
182	)
183
184	def apply_patches(self, arch, build_dir=None):	8✔
185
186	_p_version = Version(self.version)	×
187	if _p_version.major == 3 and _p_version.minor == 7:	×
188	self.patches += [	×
189	'patches/py3.7.1_fix-ctypes-util-find-library.patch',
190	'patches/py3.7.1_fix-zlib-version.patch',
191	]
192
193	if 8 <= _p_version.minor <= 10:	×
194	self.patches.append('patches/py3.8.1.patch')	×
195
196	if _p_version.minor >= 11:	×
197	self.patches.append('patches/cpython-311-ctypes-find-library.patch')	×
198
199	if _p_version.minor >= 14:	×
200	self.patches.append('patches/3.14_armv7l_fix.patch')	×
201	self.patches.append('patches/3.14_fix_remote_debug.patch')	×
202
203	if shutil.which('lld') is not None:	×
204	if _p_version.minor == 7:	×
205	self.patches.append("patches/py3.7.1_fix_cortex_a8.patch")	×
206	elif _p_version.minor >= 8:	×
207	self.patches.append("patches/py3.8.1_fix_cortex_a8.patch")	×
208
209	self.patches = list(set(self.patches))	×
210	super().apply_patches(arch, build_dir)	×
211
212	def include_root(self, arch_name):	8✔
213	_p_version = Version(self.version)	8✔
214	return join(	8✔
215	self.get_build_dir(arch_name), 'android-build', 'android-root',
216	'include', f'python{_p_version.major}.{_p_version.minor}'
217	)
218
219	def link_root(self, arch_name):	8✔
220	return join(self.get_build_dir(arch_name), 'android-build')	8✔
221
222	def get_python_root(self, arch):	8✔
NEW 223	return join(self.get_build_dir(arch.arch), 'android-build', 'android-root')	×
224
225	def get_android_python_exe(self, arch):	8✔
NEW 226	return join(self.get_python_root(arch), 'bin', self.name)	×
227
228	def should_build(self, arch):	8✔
229	return not isfile(join(self.link_root(arch.arch), self._libpython))	×
230
231	def prebuild_arch(self, arch):	8✔
232	super().prebuild_arch(arch)	×
233	self.ctx.python_recipe = self	×
234
235	def get_recipe_env(self, arch=None, with_flags_in_cc=True):	8✔
236	env = super().get_recipe_env(arch)	8✔
237	env['HOSTARCH'] = arch.command_prefix	8✔
238
239	env['CC'] = arch.get_clang_exe(with_target=True)	8✔
240
241	env['PATH'] = (	8✔
242	'{hostpython_dir}:{old_path}').format(
243	hostpython_dir=self.get_recipe(
244	'host' + self.name, self.ctx).get_path_to_python(),
245	old_path=env['PATH'])
246
247	env['CFLAGS'] = ' '.join(	8✔
248	[
249	'-fPIC',
250	'-DANDROID'
251	]
252	)
253
254	env['LDFLAGS'] = env.get('LDFLAGS', '')	8✔
255	if shutil.which('lld') is not None:	8!
256	# Note: The -L. is to fix a bug in python 3.7.
257	# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
258	env['LDFLAGS'] += ' -L. -fuse-ld=lld'	8✔
259	else:
260	warning('lld not found, linking without it. '	×
261	'Consider installing lld if linker errors occur.')
262
263	return env	8✔
264
265	def set_libs_flags(self, env, arch):	8✔
266	'''Takes care to properly link libraries with python depending on our
267	requirements and the attribute :attr:`opt_depends`.
268	'''
269	def add_flags(include_flags, link_dirs, link_libs):	8✔
270	env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags	8✔
271	env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs	8✔
272	env['LIBS'] = env.get('LIBS', '') + link_libs	8✔
273
274	info('Activating flags for sqlite3')	8✔
275	recipe = Recipe.get_recipe('sqlite3', self.ctx)	8✔
276	add_flags(' -I' + recipe.get_build_dir(arch.arch),	8✔
277	' -L' + recipe.get_build_dir(arch.arch), ' -lsqlite3')
278
279	info('Activating flags for libffi')	8✔
280	recipe = Recipe.get_recipe('libffi', self.ctx)	8✔
281	# In order to force the correct linkage for our libffi library, we
282	# set the following variable to point where is our libffi.pc file,
283	# because the python build system uses pkg-config to configure it.
284	env['PKG_CONFIG_LIBDIR'] = recipe.get_build_dir(arch.arch)	8✔
285	add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),	8✔
286	' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
287	' -lffi')
288
289	info('Activating flags for openssl')	8✔
290	recipe = Recipe.get_recipe('openssl', self.ctx)	8✔
291	self.configure_args.append('--with-openssl=' + recipe.get_build_dir(arch.arch))	8✔
292	add_flags(recipe.include_flags(arch),	8✔
293	recipe.link_dirs_flags(arch), recipe.link_libs_flags())
294
295	for library_name in {'libbz2', 'liblzma'}:	8✔
296	if library_name in self.ctx.recipe_build_order:	8!
297	info(f'Activating flags for {library_name}')	×
298	recipe = Recipe.get_recipe(library_name, self.ctx)	×
299	add_flags(recipe.get_library_includes(arch),	×
300	recipe.get_library_ldflags(arch),
301	recipe.get_library_libs_flag())
302
303	# python build system contains hardcoded zlib version which prevents
304	# the build of zlib module, here we search for android's zlib version
305	# and sets the right flags, so python can be build with android's zlib
306	info("Activating flags for android's zlib")	8✔
307	zlib_lib_path = arch.ndk_lib_dir_versioned	8✔
308	zlib_includes = self.ctx.ndk.sysroot_include_dir	8✔
309	zlib_h = join(zlib_includes, 'zlib.h')	8✔
310	try:	8✔
311	with open(zlib_h) as fileh:	8✔
312	zlib_data = fileh.read()	8✔
313	except IOError:	×
314	raise BuildInterruptingException(	×
315	"Could not determine android's zlib version, no zlib.h ({}) in"
316	" the NDK dir includes".format(zlib_h)
317	)
318	for line in zlib_data.split('\n'):	8!
319	if line.startswith('#define ZLIB_VERSION '):	8!
320	break	8✔
321	else:
322	raise BuildInterruptingException(	×
323	'Could not parse zlib.h...so we cannot find zlib version,'
324	'required by python build,'
325	)
326	env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')	8✔
327	add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')	8✔
328
329	_p_version = Version(self.version)	8✔
330	if _p_version.minor >= 11:	8!
331	self.configure_args.append('--with-build-python={python_host_bin}')	8✔
332
333	if _p_version.minor >= 13 and self.disable_gil:	8!
334	self.configure_args.append("--disable-gil")	×
335
336	self.configure_args = list(set(self.configure_args))	8✔
337
338	return env	8✔
339
340	def build_arch(self, arch):	8✔
341	if self.ctx.ndk_api < self.MIN_NDK_API:	8✔
342	raise BuildInterruptingException(	8✔
343	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
344	ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
345	),
346	)
347
348	recipe_build_dir = self.get_build_dir(arch.arch)	8✔
349
350	# Create a subdirectory to actually perform the build
351	build_dir = join(recipe_build_dir, 'android-build')	8✔
352	ensure_dir(build_dir)	8✔
353
354	sys_prefix = join(build_dir, "android-root")	8✔
355	ensure_dir(sys_prefix)	8✔
356
357	env = self.get_recipe_env(arch)	8✔
358	env = self.set_libs_flags(env, arch)	8✔
359
360	android_build = sh.Command(	8✔
361	join(recipe_build_dir,
362	'config.guess'))().strip()
363
364	with current_directory(build_dir):	8✔
365	if not exists('config.status'):	8!
366	shprint(	8✔
367	sh.Command(join(recipe_build_dir, 'configure')),
368	*(' '.join(self.configure_args).format(
369	android_host=env['HOSTARCH'],
370	android_build=android_build,
371	python_host_bin=self.get_recipe(
372	'host' + self.name, self.ctx
373	).python_exe,
374	prefix=sys_prefix).split(' ')),
375	_env=env)
376
377	shprint(	8✔
378	sh.make,
379	'all',
380	'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
381	_env=env
382	)
383	shprint(sh.make, 'install', _env=env)	8✔
384
385	# rename executable
386	if isfile("python"):	8!
387	sh.cp('python', 'libpythonbin.so')	×
388	elif isfile("python.exe"): # for macos	8!
389	sh.cp('python.exe', 'libpythonbin.so')	×
390
391	# TODO: Look into passing the path to pyconfig.h in a
392	# better way, although this is probably acceptable
393	sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))	8✔
394
395	def compile_python_files(self, dir):	8✔
396	'''
397	Compile the python files (recursively) for the python files inside
398	a given folder.
399
400	.. note:: python2 compiles the files into extension .pyo, but in
401	python3, and as of Python 3.5, the .pyo filename extension is no
402	longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
403	'''
404	args = [self.ctx.hostpython]	8✔
405	args += ['-OO', '-m', 'compileall', '-b', '-f', dir]	8✔
406	subprocess.call(args)	8✔
407
408	def create_python_bundle(self, dirn, arch):	8✔
409	"""
410	Create a packaged python bundle in the target directory, by
411	copying all the modules and standard library to the right
412	place.
413	"""
414	modules_build_dir = glob.glob(join(	×
415	self.get_build_dir(arch.arch),
416	'android-build',
417	'build',
418	'lib.*'
419	))[0]
420	# Compile to *.pyc the python modules
421	self.compile_python_files(modules_build_dir)	×
422	# Compile to *.pyc the standard python library
423	self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))	×
424	# Compile to *.pyc the other python packages (site-packages)
425	self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))	×
426
427	# Bundle compiled python modules to a folder
428	modules_dir = join(dirn, 'modules')	×
429	c_ext = self.compiled_extension	×
430	ensure_dir(modules_dir)	×
431	module_filens = (glob.glob(join(modules_build_dir, '*.so')) +	×
432	glob.glob(join(modules_build_dir, '*' + c_ext)))
433	info("Copy {} files into the bundle".format(len(module_filens)))	×
434	for filen in module_filens:	×
435	info(" - copy {}".format(filen))	×
436	shutil.copy2(filen, modules_dir)	×
437
438	# zip up the standard library
439	stdlib_zip = join(dirn, 'stdlib.zip')	×
440	with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):	×
441	stdlib_filens = list(walk_valid_filens(	×
442	'.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
443	if 'SOURCE_DATE_EPOCH' in environ:	×
444	# for reproducible builds
445	stdlib_filens.sort()	×
446	timestamp = int(environ['SOURCE_DATE_EPOCH'])	×
447	for filen in stdlib_filens:	×
448	utime(filen, (timestamp, timestamp))	×
449	info("Zip {} files into the bundle".format(len(stdlib_filens)))	×
450	shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)	×
451
452	# copy the site-packages into place
453	ensure_dir(join(dirn, 'site-packages'))	×
454	ensure_dir(self.ctx.get_python_install_dir(arch.arch))	×
455	# TODO: Improve the API around walking and copying the files
456	with current_directory(self.ctx.get_python_install_dir(arch.arch)):	×
457	filens = list(walk_valid_filens(	×
458	'.', self.site_packages_dir_blacklist,
459	self.site_packages_filen_blacklist,
460	excluded_dir_exceptions=self.site_packages_excluded_dir_exceptions))
461	info("Copy {} files into the site-packages".format(len(filens)))	×
462	for filen in filens:	×
463	info(" - copy {}".format(filen))	×
464	ensure_dir(join(dirn, 'site-packages', dirname(filen)))	×
465	shutil.copy2(filen, join(dirn, 'site-packages', filen))	×
466
467	# copy the python .so files into place
468	python_build_dir = join(self.get_build_dir(arch.arch),	×
469	'android-build')
470	python_lib_name = 'libpython' + self.link_version	×
471	shprint(	×
472	sh.cp,
473	join(python_build_dir, python_lib_name + '.so'),
474	join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
475	)
476
477	info('Renaming .so files to reflect cross-compile')	×
478	self.reduce_object_file_names(join(dirn, 'site-packages'))	×
479
480	return join(dirn, 'site-packages')	×
481
482
483	recipe = Python3Recipe()	8✔

kivy / python-for-android / 24930372022

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous