5556572719

# treebuilder.py - handle arch-specific tree building stuff using templates
#
# Copyright (C) 2011-2015 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Author(s):  Will Woods <wwoods@redhat.com>

import logging
logger = logging.getLogger("pylorax.treebuilder")

import os, re
from os.path import basename
from shutil import copytree, copy2
from subprocess import CalledProcessError
from pathlib import Path
import itertools

from pylorax.sysutils import joinpaths, remove
from pylorax.base import DataHolder
from pylorax.ltmpl import LoraxTemplateRunner
import pylorax.imgutils as imgutils
from pylorax.imgutils import DracutChroot
from pylorax.executils import runcmd, runcmd_output, execWithCapture

templatemap = {
    'x86_64':  'x86.tmpl',
    'ppc64le': 'ppc64le.tmpl',
    's390x':   's390.tmpl',
    'aarch64': 'aarch64.tmpl',
}

def generate_module_info(moddir, outfile=None):
    def module_desc(mod):
        output = runcmd_output(["modinfo", "-F", "description", mod])
        return output.strip()
    def read_module_set(name):
        return set(l.strip() for l in open(joinpaths(moddir,name)) if ".ko" in l)
    modsets = {'scsi':read_module_set("modules.block"),
               'eth':read_module_set("modules.networking")}

    modinfo = list()
    for root, _dirs, files in os.walk(moddir):
        for modtype, modset in modsets.items():
            for mod in modset.intersection(files):  # modules in this dir
                (name, _ext) = os.path.splitext(mod) # foo.ko -> (foo, .ko)
                desc = module_desc(joinpaths(root,mod)) or "%s driver" % name
                modinfo.append(dict(name=name, type=modtype, desc=desc))

    out = open(outfile or joinpaths(moddir,"module-info"), "w")
    out.write("Version 0\n")
    for mod in sorted(modinfo, key=lambda m: m.get('name')):
        out.write('{name}\n\t{type}\n\t"{desc:.65}"\n'.format(**mod))

class RuntimeBuilder(object):
    '''Builds the anaconda runtime image.'''
    def __init__(self, product, arch, dbo, templatedir=None,
                 installpkgs=None, excludepkgs=None,
                 add_templates=None,
                 add_template_vars=None,
                 skip_branding=False):
        root = dbo.conf.installroot
        self.dbo = dbo
        self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
                                           dbo=dbo, templatedir=templatedir)
        self.add_templates = add_templates or []
        self.add_template_vars = add_template_vars or {}
        self._installpkgs = installpkgs or []
        self._excludepkgs = excludepkgs or []
        self.dbo.reset()

        # use a copy of product so we can modify it locally
        product = product.copy()
        product.name = product.name.lower()
        self._branding = self.get_branding(skip_branding, product)
        self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
                               basearch=arch.basearch, libdir=arch.libdir,
                               branding=self._branding)
        self._runner.defaults = self.vars

    def get_branding(self, skip, product):
        """Select the branding from the available 'system-release' packages
        The *best* way to control this is to have a single package in the repo provide 'system-release'
        When there are more than 1 package it will:
        - Make a list of the available packages
        - If variant is set look for a package ending with lower(variant) and use that
        - If there are one or more non-generic packages, use the first one after sorting

        Returns the package names of the system-release and release logos package
        """
        if skip:
            return DataHolder(release=None, logos=None)

        release = None
        q = self.dbo.sack.query()
        a = q.available()
        pkgs = sorted([p.name for p in a.filter(provides='system-release')
                                    if not p.name.startswith("generic")])
        if not pkgs:
            logger.error("No system-release packages found, could not get the release")
            return DataHolder(release=None, logos=None)

        logger.debug("system-release packages: %s", pkgs)
        if product.variant:
            variant = [p for p in pkgs if p.endswith("-"+product.variant.lower())]
            if variant:
                release = variant[0]
        if not release:
            release = pkgs[0]

        # release
        logger.info('got release: %s', release)

        # logos uses the basename from release (fedora, redhat, centos, ...)
        logos, _suffix = release.split('-', 1)
        return DataHolder(release=release, logos=logos+"-logos")

    def install(self):
        '''Install packages and do initial setup with runtime-install.tmpl'''
        if self._branding.release:
            self._runner.installpkg(self._branding.release)
        if self._branding.logos:
            self._runner.installpkg(self._branding.logos)

        if len(self._installpkgs) > 0:
            self._runner.installpkg(*self._installpkgs)
        if len(self._excludepkgs) > 0:
            self._runner.removepkg(*self._excludepkgs)

        self._runner.run("runtime-install.tmpl")

        for tmpl in self.add_templates:
            self._runner.run(tmpl, **self.add_template_vars)

    def writepkglists(self, pkglistdir):
        '''debugging data: write out lists of package contents'''
        if not os.path.isdir(pkglistdir):
            os.makedirs(pkglistdir)
        q = self.dbo.sack.query()
        for pkgobj in q.installed():
            with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
                for fname in pkgobj.files:
                    fobj.write("{0}\n".format(fname))

    def postinstall(self):
        '''Do some post-install setup work with runtime-postinstall.tmpl'''
        # copy configdir into runtime root beforehand
        configdir = joinpaths(self._runner.templatedir,"config_files")
        configdir_path = "tmp/config_files"
        fullpath = joinpaths(self.vars.root, configdir_path)
        if os.path.exists(fullpath):
            remove(fullpath)
        copytree(configdir, fullpath)
        self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)

    def cleanup(self):
        '''Remove unneeded packages and files with runtime-cleanup.tmpl'''
        self._runner.run("runtime-cleanup.tmpl")

    def verify(self):
        '''Ensure that contents of the installroot can run'''
        status = True

        ELF_MAGIC = b'\x7fELF'

        # Iterate over all files in /usr/bin and /usr/sbin
        # For ELF files, gather them into a list and we'll check them all at
        # the end. For files with a #!, check them as we go
        elf_files = []
        usr_bin = Path(self.vars.root + '/usr/bin')
        usr_sbin = Path(self.vars.root + '/usr/sbin')
        for path in (str(x) for x in itertools.chain(usr_bin.iterdir(), usr_sbin.iterdir()) \
                     if x.is_file()):
            with open(path, "rb") as f:
                magic = f.read(4)
                if magic == ELF_MAGIC:
                    # Save the path, minus the chroot prefix
                    elf_files.append(path[len(self.vars.root):])
                elif magic[:2] == b'#!':
                    # Reopen the file as text and read the first line.
                    # Open as latin-1 so that stray 8-bit characters don't make
                    # things blow up. We only really care about ASCII parts.
                    with open(path, "rt", encoding="latin-1") as f_text:
                        # Remove the #!, split on space, and take the first part
                        shabang = f_text.readline()[2:].split()[0]

                    # Does the path exist?
                    if not os.path.exists(self.vars.root + shabang):
                        logger.error('%s, needed by %s, does not exist', shabang, path)
                        status = False

        # Now, run ldd on all the ELF files
        # Just run ldd once on everything so it isn't logged a million times.
        # At least one thing in the list isn't going to be a dynamic executable,
        # so use execWithCapture to ignore the exit code.
        filename = ''
        for line in execWithCapture('ldd', elf_files, root=self.vars.root,
                log_output=False, filter_stderr=True).split('\n'):
            if line and not line[0].isspace():
                # New filename header, strip the : at the end and save
                filename = line[:-1]
            elif 'not found' in line:
                logger.error('%s, needed by %s, not found', line.split()[0], filename)
                status = False

        return status

    def writepkgsizes(self, pkgsizefile):
        '''debugging data: write a big list of pkg sizes'''
        fobj = open(pkgsizefile, "w")
        getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
        q = self.dbo.sack.query()
        for p in sorted(q.installed()):
            pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
            fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))

    def generate_module_data(self):
        root = self.vars.root
        moddir = joinpaths(root, "lib/modules/")
        for kernel in findkernels(root=root):
            ksyms = joinpaths(root, "boot/System.map-%s" % kernel.version)
            logger.info("doing depmod and module-info for %s", kernel.version)
            runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kernel.version])
            generate_module_info(moddir+kernel.version, outfile=moddir+"module-info")

    def create_squashfs_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
        """Create a plain squashfs runtime"""
        compressargs = compressargs or []
        os.makedirs(os.path.dirname(outfile))

        # squash the rootfs
        return imgutils.mksquashfs(self.vars.root, outfile, compression, compressargs)

    def create_ext4_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
        """Create a squashfs compressed ext4 runtime"""
        # make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
        compressargs = compressargs or []
        workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
        os.makedirs(joinpaths(workdir, "LiveOS"))

        # Catch problems with the rootfs being too small and clearly log them
        try:
            imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
                                 "Anaconda", size=size)
        except CalledProcessError as e:
            if e.stdout and "No space left on device" in e.stdout:
                logger.error("The rootfs ran out of space with size=%d", size)
            raise

        # squash the live rootfs and clean up workdir
        rc = imgutils.mksquashfs(workdir, outfile, compression, compressargs)
        remove(workdir)
        return rc

    def finished(self):
        """ Done using RuntimeBuilder

        Close the dnf base object
        """
        self.dbo.close()

class TreeBuilder(object):
    '''Builds the arch-specific boot images.
    inroot should be the installtree root (the newly-built runtime dir)'''
    def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=True, doupgrade=True,
                 templatedir=None, add_templates=None, add_template_vars=None, workdir=None, extra_boot_args=""):

        # NOTE: if you pass an arg named "runtime" to a mako template it'll
        # clobber some mako internal variables - hence "runtime_img".
        self.vars = DataHolder(arch=arch, product=product, runtime_img=runtime,
                               runtime_base=basename(runtime),
                               inroot=inroot, outroot=outroot,
                               basearch=arch.basearch, libdir=arch.libdir,
                               isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade,
                               workdir=workdir, lower=string_lower,
                               extra_boot_args=extra_boot_args)
        self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir)
        self._runner.defaults = self.vars
        self.add_templates = add_templates or []
        self.add_template_vars = add_template_vars or {}
        self.templatedir = templatedir
        self.treeinfo_data = None

    @property
    def kernels(self):
        return findkernels(root=self.vars.inroot)

    def rebuild_initrds(self, add_args=None, backup="", prefix=""):
        '''Rebuild all the initrds in the tree. If backup is specified, each
        initrd will be renamed with backup as a suffix before rebuilding.
        If backup is empty, the existing initrd files will be overwritten.
        If suffix is specified, the existing initrd is untouched and a new
        image is built with the filename "${prefix}-${kernel.version}.img"

        If the initrd doesn't exist its name will be created based on the
        name of the kernel.
        '''
        add_args = add_args or []
        args = ["--nomdadmconf", "--nolvmconf"] + add_args
        if not backup:
            args.append("--force")

        if not self.kernels:
            raise RuntimeError("No kernels found, cannot rebuild_initrds")

        with DracutChroot(self.vars.inroot) as dracut:
            for kernel in self.kernels:
                if prefix:
                    idir = os.path.dirname(kernel.path)
                    outfile = joinpaths(idir, prefix+'-'+kernel.version+'.img')
                elif hasattr(kernel, "initrd"):
                    # If there is an existing initrd, use that
                    outfile = kernel.initrd.path
                else:
                    # Construct an initrd from the kernel name
                    outfile = kernel.path.replace("vmlinuz-", "initrd-") + ".img"
                logger.info("rebuilding %s", outfile)

                if backup:
                    initrd = joinpaths(self.vars.inroot, outfile)
                    if os.path.exists(initrd):
                        os.rename(initrd, initrd + backup)
                dracut.Run(args + [outfile, kernel.version])

    def build(self):
        templatefile = templatemap[self.vars.arch.basearch]
        for tmpl in self.add_templates:
            self._runner.run(tmpl, **self.add_template_vars)
        self._runner.run(templatefile, kernels=self.kernels)
        self.treeinfo_data = self._runner.results.treeinfo
        self.implantisomd5()

    def implantisomd5(self):
        for _section, data in self.treeinfo_data.items():
            if 'boot.iso' in data:
                iso = joinpaths(self.vars.outroot, data['boot.iso'])
                runcmd(["implantisomd5", iso])

    @property
    def dracut_hooks_path(self):
        """ Return the path to the lorax dracut hooks scripts

            Use the configured share dir if it is setup,
            otherwise default to /usr/share/lorax/dracut_hooks
        """
        if self.templatedir:
            return joinpaths(self.templatedir, "dracut_hooks")
        else:
            return "/usr/share/lorax/dracut_hooks"

    def copy_dracut_hooks(self, hooks):
        """ Copy the hook scripts in hooks into the installroot's /tmp/
        and return a list of commands to pass to dracut when creating the
        initramfs

        hooks is a list of tuples with the name of the hook script and the
        target dracut hook directory
        (eg. [("99anaconda-copy-ks.sh", "/lib/dracut/hooks/pre-pivot")])
        """
        dracut_commands = []
        for hook_script, dracut_path in hooks:
            src = joinpaths(self.dracut_hooks_path, hook_script)
            if not os.path.exists(src):
                logger.error("Missing lorax dracut hook script %s", (src))
                continue
            dst = joinpaths(self.vars.inroot, "/tmp/", hook_script)
            copy2(src, dst)
            dracut_commands += ["--include", joinpaths("/tmp/", hook_script),
                                dracut_path]
        return dracut_commands

#### TreeBuilder helper functions

def findkernels(root="/", kdir="boot"):
    # To find possible flavors, awk '/BuildKernel/ { print $4 }' kernel.spec
    flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen', 'lpae')
    kre = re.compile(r"vmlinuz-(?P<version>.+?\.(?P<arch>[a-z0-9_]+)"
                     r"(.(?P<flavor>{0}))?)$".format("|".join(flavors)))
    kernels = []
    bootfiles = os.listdir(joinpaths(root, kdir))
    for f in bootfiles:
        match = kre.match(f)
        if match:
            kernel = DataHolder(path=joinpaths(kdir, f))
            kernel.update(match.groupdict()) # sets version, arch, flavor
            kernels.append(kernel)

    # look for associated initrd/initramfs/etc.
    for kernel in kernels:
        for f in bootfiles:
            if f.endswith('-'+kernel.version+'.img'):
                imgtype, _rest = f.split('-',1)
                # special backwards-compat case
                if imgtype == 'initramfs':
                    imgtype = 'initrd'
                kernel[imgtype] = DataHolder(path=joinpaths(kdir, f))

    logger.debug("kernels=%s", kernels)
    return kernels

# udev whitelist: 'a-zA-Z0-9#+.:=@_-' (see is_whitelisted in libudev-util.c)
udev_blacklist=' !"$%&\'()*,/;<>?[\\]^`{|}~' # ASCII printable, minus whitelist
udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable
def udev_escape(label):
    out = ''
    for ch in label:
        out += ch if ch not in udev_blacklist else '\\x%02x' % ord(ch)
    return out

def string_lower(string):
    """ Return a lowercase string.

    :param string: String to lowercase

    This is used as a filter in the templates.
    """
    return string.lower()

1	# treebuilder.py - handle arch-specific tree building stuff using templates
2	#
3	# Copyright (C) 2011-2015 Red Hat, Inc.
4	#
5	# This program is free software; you can redistribute it and/or modify
6	# it under the terms of the GNU General Public License as published by
7	# the Free Software Foundation; either version 2 of the License, or
8	# (at your option) any later version.
9	#
10	# This program is distributed in the hope that it will be useful,
11	# but WITHOUT ANY WARRANTY; without even the implied warranty of
12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	# GNU General Public License for more details.
14	#
15	# You should have received a copy of the GNU General Public License
16	# along with this program. If not, see <http://www.gnu.org/licenses/>.
17	#
18	# Author(s): Will Woods <wwoods@redhat.com>
19
20	import logging	1✔
21	logger = logging.getLogger("pylorax.treebuilder")	1✔
22
23	import os, re	1✔
24	from os.path import basename	1✔
25	from shutil import copytree, copy2	1✔
26	from subprocess import CalledProcessError	1✔
27	from pathlib import Path	1✔
28	import itertools	1✔
29
30	from pylorax.sysutils import joinpaths, remove	1✔
31	from pylorax.base import DataHolder	1✔
32	from pylorax.ltmpl import LoraxTemplateRunner	1✔
33	import pylorax.imgutils as imgutils	1✔
34	from pylorax.imgutils import DracutChroot	1✔
35	from pylorax.executils import runcmd, runcmd_output, execWithCapture	1✔
36
37	templatemap = {	1✔
38	'x86_64': 'x86.tmpl',
39	'ppc64le': 'ppc64le.tmpl',
40	's390x': 's390.tmpl',
41	'aarch64': 'aarch64.tmpl',
42	}
43
44	def generate_module_info(moddir, outfile=None):	1✔
45	def module_desc(mod):	×
46	output = runcmd_output(["modinfo", "-F", "description", mod])	×
47	return output.strip()	×
48	def read_module_set(name):	×
49	return set(l.strip() for l in open(joinpaths(moddir,name)) if ".ko" in l)	×
50	modsets = {'scsi':read_module_set("modules.block"),	×
51	'eth':read_module_set("modules.networking")}
52
53	modinfo = list()	×
54	for root, _dirs, files in os.walk(moddir):	×
55	for modtype, modset in modsets.items():	×
56	for mod in modset.intersection(files): # modules in this dir	×
57	(name, _ext) = os.path.splitext(mod) # foo.ko -> (foo, .ko)	×
58	desc = module_desc(joinpaths(root,mod)) or "%s driver" % name	×
59	modinfo.append(dict(name=name, type=modtype, desc=desc))	×
60
61	out = open(outfile or joinpaths(moddir,"module-info"), "w")	×
62	out.write("Version 0\n")	×
63	for mod in sorted(modinfo, key=lambda m: m.get('name')):	×
64	out.write('{name}\n\t{type}\n\t"{desc:.65}"\n'.format(**mod))	×
65
66	class RuntimeBuilder(object):	1✔
67	'''Builds the anaconda runtime image.'''
68	def __init__(self, product, arch, dbo, templatedir=None,	1✔
69	installpkgs=None, excludepkgs=None,
70	add_templates=None,
71	add_template_vars=None,
72	skip_branding=False):
73	root = dbo.conf.installroot	1✔
74	self.dbo = dbo	1✔
75	self._runner = LoraxTemplateRunner(inroot=root, outroot=root,	1✔
76	dbo=dbo, templatedir=templatedir)
77	self.add_templates = add_templates or []	1✔
78	self.add_template_vars = add_template_vars or {}	1✔
79	self._installpkgs = installpkgs or []	1✔
80	self._excludepkgs = excludepkgs or []	1✔
81	self.dbo.reset()	1✔
82
83	# use a copy of product so we can modify it locally
84	product = product.copy()	1✔
85	product.name = product.name.lower()	1✔
86	self._branding = self.get_branding(skip_branding, product)	1✔
87	self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,	1✔
88	basearch=arch.basearch, libdir=arch.libdir,
89	branding=self._branding)
90	self._runner.defaults = self.vars	1✔
91
92	def get_branding(self, skip, product):	1✔
93	"""Select the branding from the available 'system-release' packages
94	The best way to control this is to have a single package in the repo provide 'system-release'
95	When there are more than 1 package it will:
96	- Make a list of the available packages
97	- If variant is set look for a package ending with lower(variant) and use that
98	- If there are one or more non-generic packages, use the first one after sorting
99
100	Returns the package names of the system-release and release logos package
101	"""
102	if skip:	1✔
103	return DataHolder(release=None, logos=None)	1✔
104
105	release = None	1✔
106	q = self.dbo.sack.query()	1✔
107	a = q.available()	1✔
108	pkgs = sorted([p.name for p in a.filter(provides='system-release')	1!
109	if not p.name.startswith("generic")])
110	if not pkgs:	1✔
111	logger.error("No system-release packages found, could not get the release")	1✔
112	return DataHolder(release=None, logos=None)	1✔
113
114	logger.debug("system-release packages: %s", pkgs)	1✔
115	if product.variant:	1✔
116	variant = [p for p in pkgs if p.endswith("-"+product.variant.lower())]	1!
117	if variant:	1✔
118	release = variant[0]	1✔
119	if not release:	1✔
120	release = pkgs[0]	1✔
121
122	# release
123	logger.info('got release: %s', release)	1✔
124
125	# logos uses the basename from release (fedora, redhat, centos, ...)
126	logos, _suffix = release.split('-', 1)	1✔
127	return DataHolder(release=release, logos=logos+"-logos")	1✔
128
129	def install(self):	1✔
130	'''Install packages and do initial setup with runtime-install.tmpl'''
131	if self._branding.release:	×
132	self._runner.installpkg(self._branding.release)	×
133	if self._branding.logos:	×
134	self._runner.installpkg(self._branding.logos)	×
135
136	if len(self._installpkgs) > 0:	×
137	self._runner.installpkg(*self._installpkgs)	×
138	if len(self._excludepkgs) > 0:	×
139	self._runner.removepkg(*self._excludepkgs)	×
140
141	self._runner.run("runtime-install.tmpl")	×
142
143	for tmpl in self.add_templates:	×
144	self._runner.run(tmpl, **self.add_template_vars)	×
145
146	def writepkglists(self, pkglistdir):	1✔
147	'''debugging data: write out lists of package contents'''
148	if not os.path.isdir(pkglistdir):	×
149	os.makedirs(pkglistdir)	×
150	q = self.dbo.sack.query()	×
151	for pkgobj in q.installed():	×
152	with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:	×
153	for fname in pkgobj.files:	×
154	fobj.write("{0}\n".format(fname))	×
155
156	def postinstall(self):	1✔
157	'''Do some post-install setup work with runtime-postinstall.tmpl'''
158	# copy configdir into runtime root beforehand
159	configdir = joinpaths(self._runner.templatedir,"config_files")	×
160	configdir_path = "tmp/config_files"	×
161	fullpath = joinpaths(self.vars.root, configdir_path)	×
162	if os.path.exists(fullpath):	×
163	remove(fullpath)	×
164	copytree(configdir, fullpath)	×
165	self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)	×
166
167	def cleanup(self):	1✔
168	'''Remove unneeded packages and files with runtime-cleanup.tmpl'''
169	self._runner.run("runtime-cleanup.tmpl")	×
170
171	def verify(self):	1✔
172	'''Ensure that contents of the installroot can run'''
173	status = True	×
174
175	ELF_MAGIC = b'\x7fELF'	×
176
177	# Iterate over all files in /usr/bin and /usr/sbin
178	# For ELF files, gather them into a list and we'll check them all at
179	# the end. For files with a #!, check them as we go
180	elf_files = []	×
181	usr_bin = Path(self.vars.root + '/usr/bin')	×
182	usr_sbin = Path(self.vars.root + '/usr/sbin')	×
183	for path in (str(x) for x in itertools.chain(usr_bin.iterdir(), usr_sbin.iterdir()) \	×
184	if x.is_file()):
185	with open(path, "rb") as f:	×
186	magic = f.read(4)	×
187	if magic == ELF_MAGIC:	×
188	# Save the path, minus the chroot prefix
189	elf_files.append(path[len(self.vars.root):])	×
190	elif magic[:2] == b'#!':	×
191	# Reopen the file as text and read the first line.
192	# Open as latin-1 so that stray 8-bit characters don't make
193	# things blow up. We only really care about ASCII parts.
194	with open(path, "rt", encoding="latin-1") as f_text:	×
195	# Remove the #!, split on space, and take the first part
196	shabang = f_text.readline()[2:].split()[0]	×
197
198	# Does the path exist?
199	if not os.path.exists(self.vars.root + shabang):	×
200	logger.error('%s, needed by %s, does not exist', shabang, path)	×
201	status = False	×
202
203	# Now, run ldd on all the ELF files
204	# Just run ldd once on everything so it isn't logged a million times.
205	# At least one thing in the list isn't going to be a dynamic executable,
206	# so use execWithCapture to ignore the exit code.
207	filename = ''	×
208	for line in execWithCapture('ldd', elf_files, root=self.vars.root,	×
209	log_output=False, filter_stderr=True).split('\n'):
210	if line and not line[0].isspace():	×
211	# New filename header, strip the : at the end and save
212	filename = line[:-1]	×
213	elif 'not found' in line:	×
214	logger.error('%s, needed by %s, not found', line.split()[0], filename)	×
215	status = False	×
216
217	return status	×
218
219	def writepkgsizes(self, pkgsizefile):	1✔
220	'''debugging data: write a big list of pkg sizes'''
221	fobj = open(pkgsizefile, "w")	×
222	getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0	×
223	q = self.dbo.sack.query()	×
224	for p in sorted(q.installed()):	×
225	pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)	×
226	fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))	×
227
228	def generate_module_data(self):	1✔
229	root = self.vars.root	×
230	moddir = joinpaths(root, "lib/modules/")	×
231	for kernel in findkernels(root=root):	×
232	ksyms = joinpaths(root, "boot/System.map-%s" % kernel.version)	×
233	logger.info("doing depmod and module-info for %s", kernel.version)	×
234	runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kernel.version])	×
235	generate_module_info(moddir+kernel.version, outfile=moddir+"module-info")	×
236
237	def create_squashfs_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):	1✔
238	"""Create a plain squashfs runtime"""
239	compressargs = compressargs or []	1✔
240	os.makedirs(os.path.dirname(outfile))	1✔
241
242	# squash the rootfs
243	return imgutils.mksquashfs(self.vars.root, outfile, compression, compressargs)	1✔
244
245	def create_ext4_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):	1✔
246	"""Create a squashfs compressed ext4 runtime"""
247	# make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
248	compressargs = compressargs or []	×
249	workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")	×
250	os.makedirs(joinpaths(workdir, "LiveOS"))	×
251
252	# Catch problems with the rootfs being too small and clearly log them
253	try:	×
254	imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),	×
255	"Anaconda", size=size)
256	except CalledProcessError as e:	×
257	if e.stdout and "No space left on device" in e.stdout:	×
258	logger.error("The rootfs ran out of space with size=%d", size)	×
259	raise	×
260
261	# squash the live rootfs and clean up workdir
262	rc = imgutils.mksquashfs(workdir, outfile, compression, compressargs)	×
263	remove(workdir)	×
264	return rc	×
265
266	def finished(self):	1✔
267	""" Done using RuntimeBuilder
268
269	Close the dnf base object
270	"""
271	self.dbo.close()	×
272
273	class TreeBuilder(object):	1✔
274	'''Builds the arch-specific boot images.
275	inroot should be the installtree root (the newly-built runtime dir)'''
276	def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=True, doupgrade=True,	1✔
277	templatedir=None, add_templates=None, add_template_vars=None, workdir=None, extra_boot_args=""):
278
279	# NOTE: if you pass an arg named "runtime" to a mako template it'll
280	# clobber some mako internal variables - hence "runtime_img".
281	self.vars = DataHolder(arch=arch, product=product, runtime_img=runtime,	1✔
282	runtime_base=basename(runtime),
283	inroot=inroot, outroot=outroot,
284	basearch=arch.basearch, libdir=arch.libdir,
285	isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade,
286	workdir=workdir, lower=string_lower,
287	extra_boot_args=extra_boot_args)
288	self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir)	1✔
289	self._runner.defaults = self.vars	1✔
290	self.add_templates = add_templates or []	1✔
291	self.add_template_vars = add_template_vars or {}	1✔
292	self.templatedir = templatedir	1✔
293	self.treeinfo_data = None	1✔
294
295	@property	1✔
296	def kernels(self):	1✔
297	return findkernels(root=self.vars.inroot)	×
298
299	def rebuild_initrds(self, add_args=None, backup="", prefix=""):	1✔
300	'''Rebuild all the initrds in the tree. If backup is specified, each
301	initrd will be renamed with backup as a suffix before rebuilding.
302	If backup is empty, the existing initrd files will be overwritten.
303	If suffix is specified, the existing initrd is untouched and a new
304	image is built with the filename "${prefix}-${kernel.version}.img"
305
306	If the initrd doesn't exist its name will be created based on the
307	name of the kernel.
308	'''
309	add_args = add_args or []	×
310	args = ["--nomdadmconf", "--nolvmconf"] + add_args	×
311	if not backup:	×
312	args.append("--force")	×
313
314	if not self.kernels:	×
315	raise RuntimeError("No kernels found, cannot rebuild_initrds")	×
316
317	with DracutChroot(self.vars.inroot) as dracut:	×
318	for kernel in self.kernels:	×
319	if prefix:	×
320	idir = os.path.dirname(kernel.path)	×
321	outfile = joinpaths(idir, prefix+'-'+kernel.version+'.img')	×
322	elif hasattr(kernel, "initrd"):	×
323	# If there is an existing initrd, use that
324	outfile = kernel.initrd.path	×
325	else:
326	# Construct an initrd from the kernel name
327	outfile = kernel.path.replace("vmlinuz-", "initrd-") + ".img"	×
328	logger.info("rebuilding %s", outfile)	×
329
330	if backup:	×
331	initrd = joinpaths(self.vars.inroot, outfile)	×
332	if os.path.exists(initrd):	×
333	os.rename(initrd, initrd + backup)	×
334	dracut.Run(args + [outfile, kernel.version])	×
335
336	def build(self):	1✔
337	templatefile = templatemap[self.vars.arch.basearch]	×
338	for tmpl in self.add_templates:	×
339	self._runner.run(tmpl, **self.add_template_vars)	×
340	self._runner.run(templatefile, kernels=self.kernels)	×
341	self.treeinfo_data = self._runner.results.treeinfo	×
342	self.implantisomd5()	×
343
344	def implantisomd5(self):	1✔
345	for _section, data in self.treeinfo_data.items():	×
346	if 'boot.iso' in data:	×
347	iso = joinpaths(self.vars.outroot, data['boot.iso'])	×
348	runcmd(["implantisomd5", iso])	×
349
350	@property	1✔
351	def dracut_hooks_path(self):	1✔
352	""" Return the path to the lorax dracut hooks scripts
353
354	Use the configured share dir if it is setup,
355	otherwise default to /usr/share/lorax/dracut_hooks
356	"""
357	if self.templatedir:	×
358	return joinpaths(self.templatedir, "dracut_hooks")	×
359	else:
360	return "/usr/share/lorax/dracut_hooks"	×
361
362	def copy_dracut_hooks(self, hooks):	1✔
363	""" Copy the hook scripts in hooks into the installroot's /tmp/
364	and return a list of commands to pass to dracut when creating the
365	initramfs
366
367	hooks is a list of tuples with the name of the hook script and the
368	target dracut hook directory
369	(eg. [("99anaconda-copy-ks.sh", "/lib/dracut/hooks/pre-pivot")])
370	"""
371	dracut_commands = []	×
372	for hook_script, dracut_path in hooks:	×
373	src = joinpaths(self.dracut_hooks_path, hook_script)	×
374	if not os.path.exists(src):	×
375	logger.error("Missing lorax dracut hook script %s", (src))	×
376	continue	×
377	dst = joinpaths(self.vars.inroot, "/tmp/", hook_script)	×
378	copy2(src, dst)	×
379	dracut_commands += ["--include", joinpaths("/tmp/", hook_script),	×
380	dracut_path]
381	return dracut_commands	×
382
383	#### TreeBuilder helper functions
384
385	def findkernels(root="/", kdir="boot"):	1✔
386	# To find possible flavors, awk '/BuildKernel/ { print $4 }' kernel.spec
387	flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen', 'lpae')	1✔
388	kre = re.compile(r"vmlinuz-(?P<version>.+?\.(?P<arch>[a-z0-9_]+)"	1✔
389	r"(.(?P<flavor>{0}))?)$".format("\|".join(flavors)))
390	kernels = []	1✔
391	bootfiles = os.listdir(joinpaths(root, kdir))	1✔
392	for f in bootfiles:	1✔
393	match = kre.match(f)	1✔
394	if match:	1✔
395	kernel = DataHolder(path=joinpaths(kdir, f))	1✔
396	kernel.update(match.groupdict()) # sets version, arch, flavor	1✔
397	kernels.append(kernel)	1✔
398
399	# look for associated initrd/initramfs/etc.
400	for kernel in kernels:	1✔
401	for f in bootfiles:	1✔
402	if f.endswith('-'+kernel.version+'.img'):	1✔
403	imgtype, _rest = f.split('-',1)	1✔
404	# special backwards-compat case
405	if imgtype == 'initramfs':	1!
406	imgtype = 'initrd'	1✔
407	kernel[imgtype] = DataHolder(path=joinpaths(kdir, f))	1✔
408
409	logger.debug("kernels=%s", kernels)	1✔
410	return kernels	1✔
411
412	# udev whitelist: 'a-zA-Z0-9#+.:=@_-' (see is_whitelisted in libudev-util.c)
413	udev_blacklist=' !"$%&\'()*,/;<>?[\\]^`{\|}~' # ASCII printable, minus whitelist	1✔
414	udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable	1✔
415	def udev_escape(label):	1✔
416	out = ''	×
417	for ch in label:	×
418	out += ch if ch not in udev_blacklist else '\\x%02x' % ord(ch)	×
419	return out	×
420
421	def string_lower(string):	1✔
422	""" Return a lowercase string.
423
424	:param string: String to lowercase
425
426	This is used as a filter in the templates.
427	"""
428	return string.lower()	×

weldr / lorax / 5556572719

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous