6774900670

Committed 06 Nov 2023 06:20PM UTC coverage: 59.187% (+0.08%) from 59.106%

Build # 6774900670

Build Type

push

github

Committed by

web-flow

Commit Message

Remove `distutils` usage, as is not available anymore on Python `3.12` (#2912)

* Remove distutils usage, as is not available anymore on Python 3.12

* Updated testapps to use setuptools instead of distutils

Run Details

943 of 2239 branches covered (0.0%)

Branch coverage included in aggregate %.

24 of 26 new or added lines in 6 files covered. (92.31%)

1 existing line in 1 file now uncovered.

4733 of 7351 relevant lines covered (64.39%)

2.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.95

/pythonforandroid/recipes/python3/__init__.py

import glob
import sh
import subprocess

from multiprocessing import cpu_count
from os import environ, utime
from os.path import dirname, exists, join
from pathlib import Path
import shutil

from pythonforandroid.logger import info, warning, shprint
from pythonforandroid.patching import version_starts_with
from pythonforandroid.recipe import Recipe, TargetPythonRecipe
from pythonforandroid.util import (
    current_directory,
    ensure_dir,
    walk_valid_filens,
    BuildInterruptingException,
)

NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (
    'Target ndk-api is {ndk_api}, '
    'but the python3 recipe supports only {min_ndk_api}+'
)


class Python3Recipe(TargetPythonRecipe):
    '''
    The python3's recipe
    ^^^^^^^^^^^^^^^^^^^^

    The python 3 recipe can be built with some extra python modules, but to do
    so, we need some libraries. By default, we ship the python3 recipe with
    some common libraries, defined in ``depends``. We also support some optional
    libraries, which are less common that the ones defined in ``depends``, so
    we added them as optional dependencies (``opt_depends``).

    Below you have a relationship between the python modules and the recipe
    libraries::

        - _ctypes: you must add the recipe for ``libffi``.
        - _sqlite3: you must add the recipe for ``sqlite3``.
        - _ssl: you must add the recipe for ``openssl``.
        - _bz2: you must add the recipe for ``libbz2`` (optional).
        - _lzma: you must add the recipe for ``liblzma`` (optional).

    .. note:: This recipe can be built only against API 21+.

    .. versionchanged:: 2019.10.06.post0
        - Refactored from deleted class ``python.GuestPythonRecipe`` into here
        - Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
          and :mod:`~pythonforandroid.recipes.liblzma`

    .. versionchanged:: 0.6.0
        Refactored into class
        :class:`~pythonforandroid.python.GuestPythonRecipe`
    '''

    version = '3.10.10'
    url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz'
    name = 'python3'

    patches = [
        'patches/pyconfig_detection.patch',
        'patches/reproducible-buildinfo.diff',

        # Python 3.7.1
        ('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")),
        ('patches/py3.7.1_fix-zlib-version.patch', version_starts_with("3.7")),

        # Python 3.8.1 & 3.9.X
        ('patches/py3.8.1.patch', version_starts_with("3.8")),
        ('patches/py3.8.1.patch', version_starts_with("3.9")),
        ('patches/py3.8.1.patch', version_starts_with("3.10"))
    ]

    if shutil.which('lld') is not None:
        patches = patches + [
            ("patches/py3.7.1_fix_cortex_a8.patch", version_starts_with("3.7")),
            ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.8")),
            ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.9")),
            ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10"))
        ]

    depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
    # those optional depends allow us to build python compression modules:
    #   - _bz2.so
    #   - _lzma.so
    opt_depends = ['libbz2', 'liblzma']
    '''The optional libraries which we would like to get our python linked'''

    configure_args = (
        '--host={android_host}',
        '--build={android_build}',
        '--enable-shared',
        '--enable-ipv6',
        'ac_cv_file__dev_ptmx=yes',
        'ac_cv_file__dev_ptc=no',
        '--without-ensurepip',
        'ac_cv_little_endian_double=yes',
        'ac_cv_header_sys_eventfd_h=no',
        '--prefix={prefix}',
        '--exec-prefix={exec_prefix}',
        '--enable-loadable-sqlite-extensions')
    '''The configure arguments needed to build the python recipe. Those are
    used in method :meth:`build_arch` (if not overwritten like python3's
    recipe does).
    '''

    MIN_NDK_API = 21
    '''Sets the minimal ndk api number needed to use the recipe.

    .. warning:: This recipe can be built only against API 21+, so it means
        that any class which inherits from class:`GuestPythonRecipe` will have
        this limitation.
    '''

    stdlib_dir_blacklist = {
        '__pycache__',
        'test',
        'tests',
        'lib2to3',
        'ensurepip',
        'idlelib',
        'tkinter',
    }
    '''The directories that we want to omit for our python bundle'''

    stdlib_filen_blacklist = [
        '*.py',
        '*.exe',
        '*.whl',
    ]
    '''The file extensions that we want to blacklist for our python bundle'''

    site_packages_dir_blacklist = {
        '__pycache__',
        'tests'
    }
    '''The directories from site packages dir that we don't want to be included
    in our python bundle.'''

    site_packages_filen_blacklist = [
        '*.py'
    ]
    '''The file extensions from site packages dir that we don't want to be
    included in our python bundle.'''

    compiled_extension = '.pyc'
    '''the default extension for compiled python files.

    .. note:: the default extension for compiled python files has been .pyo for
        python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
        longer used and has been removed in favour of extension .pyc
    '''

    def __init__(self, *args, **kwargs):
        self._ctx = None
        super().__init__(*args, **kwargs)

    @property
    def _libpython(self):
        '''return the python's library name (with extension)'''
        return 'libpython{link_version}.so'.format(
            link_version=self.link_version
        )

    @property
    def link_version(self):
        '''return the python's library link version e.g. 3.7m, 3.8'''
        major, minor = self.major_minor_version_string.split('.')
        flags = ''
        if major == '3' and int(minor) < 8:
            flags += 'm'
        return '{major}.{minor}{flags}'.format(
            major=major,
            minor=minor,
            flags=flags
        )

    def include_root(self, arch_name):
        return join(self.get_build_dir(arch_name), 'Include')

    def link_root(self, arch_name):
        return join(self.get_build_dir(arch_name), 'android-build')

    def should_build(self, arch):
        return not Path(self.link_root(arch.arch), self._libpython).is_file()

    def prebuild_arch(self, arch):
        super().prebuild_arch(arch)
        self.ctx.python_recipe = self

    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
        env = super().get_recipe_env(arch)
        env['HOSTARCH'] = arch.command_prefix

        env['CC'] = arch.get_clang_exe(with_target=True)

        env['PATH'] = (
            '{hostpython_dir}:{old_path}').format(
                hostpython_dir=self.get_recipe(
                    'host' + self.name, self.ctx).get_path_to_python(),
                old_path=env['PATH'])

        env['CFLAGS'] = ' '.join(
            [
                '-fPIC',
                '-DANDROID'
            ]
        )

        env['LDFLAGS'] = env.get('LDFLAGS', '')
        if shutil.which('lld') is not None:
            # Note: The -L. is to fix a bug in python 3.7.
            # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
            env['LDFLAGS'] += ' -L. -fuse-ld=lld'
        else:
            warning('lld not found, linking without it. '
                    'Consider installing lld if linker errors occur.')

        return env

    def set_libs_flags(self, env, arch):
        '''Takes care to properly link libraries with python depending on our
        requirements and the attribute :attr:`opt_depends`.
        '''
        def add_flags(include_flags, link_dirs, link_libs):
            env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
            env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
            env['LIBS'] = env.get('LIBS', '') + link_libs

        if 'sqlite3' in self.ctx.recipe_build_order:
            info('Activating flags for sqlite3')
            recipe = Recipe.get_recipe('sqlite3', self.ctx)
            add_flags(' -I' + recipe.get_build_dir(arch.arch),
                      ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')

        if 'libffi' in self.ctx.recipe_build_order:
            info('Activating flags for libffi')
            recipe = Recipe.get_recipe('libffi', self.ctx)
            # In order to force the correct linkage for our libffi library, we
            # set the following variable to point where is our libffi.pc file,
            # because the python build system uses pkg-config to configure it.
            env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch)
            add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
                      ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
                      ' -lffi')

        if 'openssl' in self.ctx.recipe_build_order:
            info('Activating flags for openssl')
            recipe = Recipe.get_recipe('openssl', self.ctx)
            self.configure_args += \
                ('--with-openssl=' + recipe.get_build_dir(arch.arch),)
            add_flags(recipe.include_flags(arch),
                      recipe.link_dirs_flags(arch), recipe.link_libs_flags())

        for library_name in {'libbz2', 'liblzma'}:
            if library_name in self.ctx.recipe_build_order:
                info(f'Activating flags for {library_name}')
                recipe = Recipe.get_recipe(library_name, self.ctx)
                add_flags(recipe.get_library_includes(arch),
                          recipe.get_library_ldflags(arch),
                          recipe.get_library_libs_flag())

        # python build system contains hardcoded zlib version which prevents
        # the build of zlib module, here we search for android's zlib version
        # and sets the right flags, so python can be build with android's zlib
        info("Activating flags for android's zlib")
        zlib_lib_path = arch.ndk_lib_dir_versioned
        zlib_includes = self.ctx.ndk.sysroot_include_dir
        zlib_h = join(zlib_includes, 'zlib.h')
        try:
            with open(zlib_h) as fileh:
                zlib_data = fileh.read()
        except IOError:
            raise BuildInterruptingException(
                "Could not determine android's zlib version, no zlib.h ({}) in"
                " the NDK dir includes".format(zlib_h)
            )
        for line in zlib_data.split('\n'):
            if line.startswith('#define ZLIB_VERSION '):
                break
        else:
            raise BuildInterruptingException(
                'Could not parse zlib.h...so we cannot find zlib version,'
                'required by python build,'
            )
        env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
        add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')

        return env

    def build_arch(self, arch):
        if self.ctx.ndk_api < self.MIN_NDK_API:
            raise BuildInterruptingException(
                NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
                    ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
                ),
            )

        recipe_build_dir = self.get_build_dir(arch.arch)

        # Create a subdirectory to actually perform the build
        build_dir = join(recipe_build_dir, 'android-build')
        ensure_dir(build_dir)

        # TODO: Get these dynamically, like bpo-30386 does
        sys_prefix = '/usr/local'
        sys_exec_prefix = '/usr/local'

        env = self.get_recipe_env(arch)
        env = self.set_libs_flags(env, arch)

        android_build = sh.Command(
            join(recipe_build_dir,
                 'config.guess'))().stdout.strip().decode('utf-8')

        with current_directory(build_dir):
            if not exists('config.status'):
                shprint(
                    sh.Command(join(recipe_build_dir, 'configure')),
                    *(' '.join(self.configure_args).format(
                                    android_host=env['HOSTARCH'],
                                    android_build=android_build,
                                    prefix=sys_prefix,
                                    exec_prefix=sys_exec_prefix)).split(' '),
                    _env=env)

            shprint(
                sh.make, 'all', '-j', str(cpu_count()),
                'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
                _env=env
            )

            # TODO: Look into passing the path to pyconfig.h in a
            # better way, although this is probably acceptable
            sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))

    def compile_python_files(self, dir):
        '''
        Compile the python files (recursively) for the python files inside
        a given folder.

        .. note:: python2 compiles the files into extension .pyo, but in
            python3, and as of Python 3.5, the .pyo filename extension is no
            longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
        '''
        args = [self.ctx.hostpython]
        args += ['-OO', '-m', 'compileall', '-b', '-f', dir]
        subprocess.call(args)

    def create_python_bundle(self, dirn, arch):
        """
        Create a packaged python bundle in the target directory, by
        copying all the modules and standard library to the right
        place.
        """
        # Todo: find a better way to find the build libs folder
        modules_build_dir = join(
            self.get_build_dir(arch.arch),
            'android-build',
            'build',
            'lib.linux{}-{}-{}'.format(
                '2' if self.version[0] == '2' else '',
                arch.command_prefix.split('-')[0],
                self.major_minor_version_string
            ))

        # Compile to *.pyc the python modules
        self.compile_python_files(modules_build_dir)
        # Compile to *.pyc the standard python library
        self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
        # Compile to *.pyc the other python packages (site-packages)
        self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))

        # Bundle compiled python modules to a folder
        modules_dir = join(dirn, 'modules')
        c_ext = self.compiled_extension
        ensure_dir(modules_dir)
        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
                         glob.glob(join(modules_build_dir, '*' + c_ext)))
        info("Copy {} files into the bundle".format(len(module_filens)))
        for filen in module_filens:
            info(" - copy {}".format(filen))
            shutil.copy2(filen, modules_dir)

        # zip up the standard library
        stdlib_zip = join(dirn, 'stdlib.zip')
        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
            stdlib_filens = list(walk_valid_filens(
                '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
            if 'SOURCE_DATE_EPOCH' in environ:
                # for reproducible builds
                stdlib_filens.sort()
                timestamp = int(environ['SOURCE_DATE_EPOCH'])
                for filen in stdlib_filens:
                    utime(filen, (timestamp, timestamp))
            info("Zip {} files into the bundle".format(len(stdlib_filens)))
            shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)

        # copy the site-packages into place
        ensure_dir(join(dirn, 'site-packages'))
        ensure_dir(self.ctx.get_python_install_dir(arch.arch))
        # TODO: Improve the API around walking and copying the files
        with current_directory(self.ctx.get_python_install_dir(arch.arch)):
            filens = list(walk_valid_filens(
                '.', self.site_packages_dir_blacklist,
                self.site_packages_filen_blacklist))
            info("Copy {} files into the site-packages".format(len(filens)))
            for filen in filens:
                info(" - copy {}".format(filen))
                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
                shutil.copy2(filen, join(dirn, 'site-packages', filen))

        # copy the python .so files into place
        python_build_dir = join(self.get_build_dir(arch.arch),
                                'android-build')
        python_lib_name = 'libpython' + self.link_version
        shprint(
            sh.cp,
            join(python_build_dir, python_lib_name + '.so'),
            join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
        )

        info('Renaming .so files to reflect cross-compile')
        self.reduce_object_file_names(join(dirn, 'site-packages'))

        return join(dirn, 'site-packages')


recipe = Python3Recipe()

1	import glob	4✔
2	import sh	4✔
3	import subprocess	4✔
4
5	from multiprocessing import cpu_count	4✔
6	from os import environ, utime	4✔
7	from os.path import dirname, exists, join	4✔
8	from pathlib import Path	4✔
9	import shutil	4✔
10
11	from pythonforandroid.logger import info, warning, shprint	4✔
12	from pythonforandroid.patching import version_starts_with	4✔
13	from pythonforandroid.recipe import Recipe, TargetPythonRecipe	4✔
14	from pythonforandroid.util import (	4✔
15	current_directory,
16	ensure_dir,
17	walk_valid_filens,
18	BuildInterruptingException,
19	)
20
21	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = (	4✔
22	'Target ndk-api is {ndk_api}, '
23	'but the python3 recipe supports only {min_ndk_api}+'
24	)
25
26
27	class Python3Recipe(TargetPythonRecipe):	4✔
28	'''
29	The python3's recipe
30	^^^^^^^^^^^^^^^^^^^^
31
32	The python 3 recipe can be built with some extra python modules, but to do
33	so, we need some libraries. By default, we ship the python3 recipe with
34	some common libraries, defined in ``depends``. We also support some optional
35	libraries, which are less common that the ones defined in ``depends``, so
36	we added them as optional dependencies (``opt_depends``).
37
38	Below you have a relationship between the python modules and the recipe
39	libraries::
40
41	- _ctypes: you must add the recipe for ``libffi``.
42	- _sqlite3: you must add the recipe for ``sqlite3``.
43	- _ssl: you must add the recipe for ``openssl``.
44	- _bz2: you must add the recipe for ``libbz2`` (optional).
45	- _lzma: you must add the recipe for ``liblzma`` (optional).
46
47	.. note:: This recipe can be built only against API 21+.
48
49	.. versionchanged:: 2019.10.06.post0
50	- Refactored from deleted class ``python.GuestPythonRecipe`` into here
51	- Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2`
52	and :mod:`~pythonforandroid.recipes.liblzma`
53
54	.. versionchanged:: 0.6.0
55	Refactored into class
56	:class:`~pythonforandroid.python.GuestPythonRecipe`
57	'''
58
59	version = '3.10.10'	4✔
60	url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz'	4✔
61	name = 'python3'	4✔
62
63	patches = [	4✔
64	'patches/pyconfig_detection.patch',
65	'patches/reproducible-buildinfo.diff',
66
67	# Python 3.7.1
68	('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")),
69	('patches/py3.7.1_fix-zlib-version.patch', version_starts_with("3.7")),
70
71	# Python 3.8.1 & 3.9.X
72	('patches/py3.8.1.patch', version_starts_with("3.8")),
73	('patches/py3.8.1.patch', version_starts_with("3.9")),
74	('patches/py3.8.1.patch', version_starts_with("3.10"))
75	]
76
77	if shutil.which('lld') is not None:	4✔
78	patches = patches + [	4✔
79	("patches/py3.7.1_fix_cortex_a8.patch", version_starts_with("3.7")),
80	("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.8")),
81	("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.9")),
82	("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10"))
83	]
84
85	depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']	4✔
86	# those optional depends allow us to build python compression modules:
87	# - _bz2.so
88	# - _lzma.so
89	opt_depends = ['libbz2', 'liblzma']	4✔
90	'''The optional libraries which we would like to get our python linked'''	2✔
91
92	configure_args = (	4✔
93	'--host={android_host}',
94	'--build={android_build}',
95	'--enable-shared',
96	'--enable-ipv6',
97	'ac_cv_file__dev_ptmx=yes',
98	'ac_cv_file__dev_ptc=no',
99	'--without-ensurepip',
100	'ac_cv_little_endian_double=yes',
101	'ac_cv_header_sys_eventfd_h=no',
102	'--prefix={prefix}',
103	'--exec-prefix={exec_prefix}',
104	'--enable-loadable-sqlite-extensions')
105	'''The configure arguments needed to build the python recipe. Those are	2✔
106	used in method :meth:`build_arch` (if not overwritten like python3's
107	recipe does).
108	'''
109
110	MIN_NDK_API = 21	4✔
111	'''Sets the minimal ndk api number needed to use the recipe.	2✔
112
113	.. warning:: This recipe can be built only against API 21+, so it means
114	that any class which inherits from class:`GuestPythonRecipe` will have
115	this limitation.
116	'''
117
118	stdlib_dir_blacklist = {	4✔
119	'__pycache__',
120	'test',
121	'tests',
122	'lib2to3',
123	'ensurepip',
124	'idlelib',
125	'tkinter',
126	}
127	'''The directories that we want to omit for our python bundle'''	2✔
128
129	stdlib_filen_blacklist = [	4✔
130	'*.py',
131	'*.exe',
132	'*.whl',
133	]
134	'''The file extensions that we want to blacklist for our python bundle'''	2✔
135
136	site_packages_dir_blacklist = {	4✔
137	'__pycache__',
138	'tests'
139	}
140	'''The directories from site packages dir that we don't want to be included	2✔
141	in our python bundle.'''
142
143	site_packages_filen_blacklist = [	4✔
144	'*.py'
145	]
146	'''The file extensions from site packages dir that we don't want to be	2✔
147	included in our python bundle.'''
148
149	compiled_extension = '.pyc'	4✔
150	'''the default extension for compiled python files.	2✔
151
152	.. note:: the default extension for compiled python files has been .pyo for
153	python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no
154	longer used and has been removed in favour of extension .pyc
155	'''
156
157	def __init__(self, args, *kwargs):	4✔
158	self._ctx = None	4✔
159	super().__init__(args, *kwargs)	4✔
160
161	@property	4✔
162	def _libpython(self):	4✔
163	'''return the python's library name (with extension)'''
164	return 'libpython{link_version}.so'.format(	4✔
165	link_version=self.link_version
166	)
167
168	@property	4✔
169	def link_version(self):	4✔
170	'''return the python's library link version e.g. 3.7m, 3.8'''
171	major, minor = self.major_minor_version_string.split('.')	4✔
172	flags = ''	4✔
173	if major == '3' and int(minor) < 8:	4!
174	flags += 'm'	×
175	return '{major}.{minor}{flags}'.format(	4✔
176	major=major,
177	minor=minor,
178	flags=flags
179	)
180
181	def include_root(self, arch_name):	4✔
182	return join(self.get_build_dir(arch_name), 'Include')	4✔
183
184	def link_root(self, arch_name):	4✔
185	return join(self.get_build_dir(arch_name), 'android-build')	4✔
186
187	def should_build(self, arch):	4✔
188	return not Path(self.link_root(arch.arch), self._libpython).is_file()	4✔
189
190	def prebuild_arch(self, arch):	4✔
191	super().prebuild_arch(arch)	×
192	self.ctx.python_recipe = self	×
193
194	def get_recipe_env(self, arch=None, with_flags_in_cc=True):	4✔
195	env = super().get_recipe_env(arch)	4✔
196	env['HOSTARCH'] = arch.command_prefix	4✔
197
198	env['CC'] = arch.get_clang_exe(with_target=True)	4✔
199
200	env['PATH'] = (	4✔
201	'{hostpython_dir}:{old_path}').format(
202	hostpython_dir=self.get_recipe(
203	'host' + self.name, self.ctx).get_path_to_python(),
204	old_path=env['PATH'])
205
206	env['CFLAGS'] = ' '.join(	4✔
207	[
208	'-fPIC',
209	'-DANDROID'
210	]
211	)
212
213	env['LDFLAGS'] = env.get('LDFLAGS', '')	4✔
214	if shutil.which('lld') is not None:	4!
215	# Note: The -L. is to fix a bug in python 3.7.
216	# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409
217	env['LDFLAGS'] += ' -L. -fuse-ld=lld'	4✔
218	else:
UNCOV 219	warning('lld not found, linking without it. '	×
220	'Consider installing lld if linker errors occur.')
221
222	return env	4✔
223
224	def set_libs_flags(self, env, arch):	4✔
225	'''Takes care to properly link libraries with python depending on our
226	requirements and the attribute :attr:`opt_depends`.
227	'''
228	def add_flags(include_flags, link_dirs, link_libs):	4✔
229	env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags	4✔
230	env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs	4✔
231	env['LIBS'] = env.get('LIBS', '') + link_libs	4✔
232
233	if 'sqlite3' in self.ctx.recipe_build_order:	4!
234	info('Activating flags for sqlite3')	×
235	recipe = Recipe.get_recipe('sqlite3', self.ctx)	×
236	add_flags(' -I' + recipe.get_build_dir(arch.arch),	×
237	' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')
238
239	if 'libffi' in self.ctx.recipe_build_order:	4!
240	info('Activating flags for libffi')	×
241	recipe = Recipe.get_recipe('libffi', self.ctx)	×
242	# In order to force the correct linkage for our libffi library, we
243	# set the following variable to point where is our libffi.pc file,
244	# because the python build system uses pkg-config to configure it.
245	env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch)	×
246	add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),	×
247	' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
248	' -lffi')
249
250	if 'openssl' in self.ctx.recipe_build_order:	4!
251	info('Activating flags for openssl')	×
252	recipe = Recipe.get_recipe('openssl', self.ctx)	×
253	self.configure_args += \	×
254	('--with-openssl=' + recipe.get_build_dir(arch.arch),)
255	add_flags(recipe.include_flags(arch),	×
256	recipe.link_dirs_flags(arch), recipe.link_libs_flags())
257
258	for library_name in {'libbz2', 'liblzma'}:	4✔
259	if library_name in self.ctx.recipe_build_order:	4!
260	info(f'Activating flags for {library_name}')	×
261	recipe = Recipe.get_recipe(library_name, self.ctx)	×
262	add_flags(recipe.get_library_includes(arch),	×
263	recipe.get_library_ldflags(arch),
264	recipe.get_library_libs_flag())
265
266	# python build system contains hardcoded zlib version which prevents
267	# the build of zlib module, here we search for android's zlib version
268	# and sets the right flags, so python can be build with android's zlib
269	info("Activating flags for android's zlib")	4✔
270	zlib_lib_path = arch.ndk_lib_dir_versioned	4✔
271	zlib_includes = self.ctx.ndk.sysroot_include_dir	4✔
272	zlib_h = join(zlib_includes, 'zlib.h')	4✔
273	try:	4✔
274	with open(zlib_h) as fileh:	4✔
275	zlib_data = fileh.read()	4✔
276	except IOError:	×
277	raise BuildInterruptingException(	×
278	"Could not determine android's zlib version, no zlib.h ({}) in"
279	" the NDK dir includes".format(zlib_h)
280	)
281	for line in zlib_data.split('\n'):	4!
282	if line.startswith('#define ZLIB_VERSION '):	4!
283	break	4✔
284	else:
285	raise BuildInterruptingException(	×
286	'Could not parse zlib.h...so we cannot find zlib version,'
287	'required by python build,'
288	)
289	env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')	4✔
290	add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')	4✔
291
292	return env	4✔
293
294	def build_arch(self, arch):	4✔
295	if self.ctx.ndk_api < self.MIN_NDK_API:	4✔
296	raise BuildInterruptingException(	4✔
297	NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
298	ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API
299	),
300	)
301
302	recipe_build_dir = self.get_build_dir(arch.arch)	4✔
303
304	# Create a subdirectory to actually perform the build
305	build_dir = join(recipe_build_dir, 'android-build')	4✔
306	ensure_dir(build_dir)	4✔
307
308	# TODO: Get these dynamically, like bpo-30386 does
309	sys_prefix = '/usr/local'	4✔
310	sys_exec_prefix = '/usr/local'	4✔
311
312	env = self.get_recipe_env(arch)	4✔
313	env = self.set_libs_flags(env, arch)	4✔
314
315	android_build = sh.Command(	4✔
316	join(recipe_build_dir,
317	'config.guess'))().stdout.strip().decode('utf-8')
318
319	with current_directory(build_dir):	4✔
320	if not exists('config.status'):	4!
321	shprint(	4✔
322	sh.Command(join(recipe_build_dir, 'configure')),
323	*(' '.join(self.configure_args).format(
324	android_host=env['HOSTARCH'],
325	android_build=android_build,
326	prefix=sys_prefix,
327	exec_prefix=sys_exec_prefix)).split(' '),
328	_env=env)
329
330	shprint(	4✔
331	sh.make, 'all', '-j', str(cpu_count()),
332	'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
333	_env=env
334	)
335
336	# TODO: Look into passing the path to pyconfig.h in a
337	# better way, although this is probably acceptable
338	sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))	4✔
339
340	def compile_python_files(self, dir):	4✔
341	'''
342	Compile the python files (recursively) for the python files inside
343	a given folder.
344
345	.. note:: python2 compiles the files into extension .pyo, but in
346	python3, and as of Python 3.5, the .pyo filename extension is no
347	longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488)
348	'''
349	args = [self.ctx.hostpython]	4✔
350	args += ['-OO', '-m', 'compileall', '-b', '-f', dir]	4✔
351	subprocess.call(args)	4✔
352
353	def create_python_bundle(self, dirn, arch):	4✔
354	"""
355	Create a packaged python bundle in the target directory, by
356	copying all the modules and standard library to the right
357	place.
358	"""
359	# Todo: find a better way to find the build libs folder
360	modules_build_dir = join(	4✔
361	self.get_build_dir(arch.arch),
362	'android-build',
363	'build',
364	'lib.linux{}-{}-{}'.format(
365	'2' if self.version[0] == '2' else '',
366	arch.command_prefix.split('-')[0],
367	self.major_minor_version_string
368	))
369
370	# Compile to *.pyc the python modules
371	self.compile_python_files(modules_build_dir)	4✔
372	# Compile to *.pyc the standard python library
373	self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))	4✔
374	# Compile to *.pyc the other python packages (site-packages)
375	self.compile_python_files(self.ctx.get_python_install_dir(arch.arch))	4✔
376
377	# Bundle compiled python modules to a folder
378	modules_dir = join(dirn, 'modules')	4✔
379	c_ext = self.compiled_extension	4✔
380	ensure_dir(modules_dir)	4✔
381	module_filens = (glob.glob(join(modules_build_dir, '*.so')) +	4✔
382	glob.glob(join(modules_build_dir, '*' + c_ext)))
383	info("Copy {} files into the bundle".format(len(module_filens)))	4✔
384	for filen in module_filens:	4!
385	info(" - copy {}".format(filen))	×
386	shutil.copy2(filen, modules_dir)	×
387
388	# zip up the standard library
389	stdlib_zip = join(dirn, 'stdlib.zip')	4✔
390	with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):	4✔
391	stdlib_filens = list(walk_valid_filens(	4✔
392	'.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
393	if 'SOURCE_DATE_EPOCH' in environ:	4!
394	# for reproducible builds
395	stdlib_filens.sort()	×
396	timestamp = int(environ['SOURCE_DATE_EPOCH'])	×
397	for filen in stdlib_filens:	×
398	utime(filen, (timestamp, timestamp))	×
399	info("Zip {} files into the bundle".format(len(stdlib_filens)))	4✔
400	shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)	4✔
401
402	# copy the site-packages into place
403	ensure_dir(join(dirn, 'site-packages'))	4✔
404	ensure_dir(self.ctx.get_python_install_dir(arch.arch))	4✔
405	# TODO: Improve the API around walking and copying the files
406	with current_directory(self.ctx.get_python_install_dir(arch.arch)):	4✔
407	filens = list(walk_valid_filens(	4✔
408	'.', self.site_packages_dir_blacklist,
409	self.site_packages_filen_blacklist))
410	info("Copy {} files into the site-packages".format(len(filens)))	4✔
411	for filen in filens:	4✔
412	info(" - copy {}".format(filen))	4✔
413	ensure_dir(join(dirn, 'site-packages', dirname(filen)))	4✔
414	shutil.copy2(filen, join(dirn, 'site-packages', filen))	4✔
415
416	# copy the python .so files into place
417	python_build_dir = join(self.get_build_dir(arch.arch),	4✔
418	'android-build')
419	python_lib_name = 'libpython' + self.link_version	4✔
420	shprint(	4✔
421	sh.cp,
422	join(python_build_dir, python_lib_name + '.so'),
423	join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)
424	)
425
426	info('Renaming .so files to reflect cross-compile')	4✔
427	self.reduce_object_file_names(join(dirn, 'site-packages'))	4✔
428
429	return join(dirn, 'site-packages')	4✔
430
431
432	recipe = Python3Recipe()	4✔

kivy / python-for-android / 6774900670

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous