18208443738

Committed 02 Oct 2025 11:36PM UTC coverage: 85.362% (-0.003%) from 85.365%

Build # 18208443738

Build Type

Pull #7308

github

Committed by

web-flow

Commit Message

Merge ad66d2b87 into 7860313eb

Pull Request Pull Request #7308: Update to Python 3.13.7 (#6429)

Run Details

19338 of 22654 relevant lines covered (85.36%)

0.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

src/azul/lambda_layer.py

from collections import (
    defaultdict,
)
import hashlib
import logging
from pathlib import (
    Path,
)
import shutil
import subprocess
from zipfile import (
    ZipFile,
    ZipInfo,
)

from azul import (
    cached_property,
    config,
)
from azul.deployment import (
    aws,
)
from azul.files import (
    file_sha1,
)

log = logging.getLogger(__name__)


class DependenciesLayer:

    @property
    def s3(self):
        return aws.s3

    def _update_required(self) -> bool:
        log.info('Checking for dependencies layer package at s3://%s/%s.',
                 aws.shared_bucket, self.object_key)
        try:
            # Since the object is content-addressed, just checking for the
            # object's presence is sufficient
            self.s3.head_object(Bucket=aws.shared_bucket, Key=self.object_key)
        except self.s3.exceptions.ClientError as e:
            if e.response['Error']['Code'] == '404':
                return True
            else:
                raise
        else:
            return False

    layer_dir = Path(config.project_root) / 'lambdas' / 'layer'

    def update_layer(self):
        if self._update_required():
            log.info('Generating new layer package ...')
            out_dir = self.layer_dir / '.chalice' / 'terraform'
            self._build_package(out_dir)
            input_zip = out_dir / 'deployment.zip'
            output_zip = out_dir / 'layer.zip'
            self._filter_package(input_zip, output_zip)
            self._validate_layer(output_zip)
            log.info('Uploading layer package to S3 ...')
            self.s3.upload_file(str(output_zip), aws.shared_bucket, self.object_key)
            log.info('Successfully staged updated layer package.')
        else:
            log.info('Layer package already up-to-date.')

    def _build_package(self, out_dir):
        # Delete Chalice's build cache because our layer cache eviction rules
        # are stricter and we want a full rebuild.
        try:
            cache_dir = self.layer_dir / '.chalice' / 'deployments'
            log.info('Removing deployment cache at %r', str(cache_dir))
            shutil.rmtree(cache_dir)
        except FileNotFoundError:
            pass
        command = ['chalice', 'package', out_dir]
        log.info('Running %r', command)
        subprocess.run(command, cwd=self.layer_dir).check_returncode()

    def _filter_package(self, input_zip_path: Path, output_zip_path: Path):
        """
        Filter a ZIP file, removing `app.py` and prefixing other archive member
        paths with `python/`.
        """
        log.info('Filtering %r to %r', str(input_zip_path), str(output_zip_path))
        with ZipFile(input_zip_path, 'r') as input_zip:
            with ZipFile(output_zip_path, 'w') as output_zip:
                for input in input_zip.infolist():
                    if input.filename != 'app.py':
                        # ZipFile doesn't copy permissions. Setting permissions
                        # manually also requires setting other fields.
                        output = ZipInfo(filename='python/' + input.filename)
                        output.external_attr = input.external_attr
                        output.date_time = input.date_time
                        output.compress_type = input.compress_type
                        with input_zip.open(input, 'r') as rf:
                            with output_zip.open(output, 'w') as wf:
                                shutil.copyfileobj(rf, wf, length=1024 * 1024)

    def _validate_layer(self, layer_zip: Path):
        with ZipFile(layer_zip, 'r') as z:
            infos = z.infolist()
        files = defaultdict(list)
        for info in infos:
            files[info.filename].append(info)
        duplicates = {k: v for k, v in files.items() if len(v) > 1}
        assert not duplicates, duplicates

    @cached_property
    def object_key(self):
        sha1 = hashlib.sha1()
        # The `chalice package` command automatically includes Chalice's own
        # __init__.py and app.py. We need to include them in the hash or else
        # we risk ignoring changes to those files.
        import chalice.app
        for module in [chalice, chalice.app]:
            sha1.update(file_sha1(getattr(module, '__file__')).encode())
        for path in Path(config.chalice_bin).iterdir():
            sha1.update(file_sha1(path).encode())
        return f'azul/{config.deployment_stage}/{config.lambda_layer_key}/{sha1.hexdigest()}.zip'

1	from collections import (	×
2	defaultdict,
3	)
4	import hashlib	×
5	import logging	×
6	from pathlib import (	×
7	Path,
8	)
9	import shutil	×
10	import subprocess	×
11	from zipfile import (	×
12	ZipFile,
13	ZipInfo,
14	)
15
16	from azul import (	×
17	cached_property,
18	config,
19	)
20	from azul.deployment import (	×
21	aws,
22	)
23	from azul.files import (	×
24	file_sha1,
25	)
26
27	log = logging.getLogger(__name__)	×
28
29
30	class DependenciesLayer:	×
31
32	@property	×
33	def s3(self):	×
34	return aws.s3	×
35
36	def _update_required(self) -> bool:	×
37	log.info('Checking for dependencies layer package at s3://%s/%s.',	×
38	aws.shared_bucket, self.object_key)
39	try:	×
40	# Since the object is content-addressed, just checking for the
41	# object's presence is sufficient
42	self.s3.head_object(Bucket=aws.shared_bucket, Key=self.object_key)	×
43	except self.s3.exceptions.ClientError as e:	×
44	if e.response['Error']['Code'] == '404':	×
45	return True	×
46	else:
47	raise	×
48	else:
49	return False	×
50
51	layer_dir = Path(config.project_root) / 'lambdas' / 'layer'	×
52
53	def update_layer(self):	×
54	if self._update_required():	×
55	log.info('Generating new layer package ...')	×
56	out_dir = self.layer_dir / '.chalice' / 'terraform'	×
57	self._build_package(out_dir)	×
58	input_zip = out_dir / 'deployment.zip'	×
59	output_zip = out_dir / 'layer.zip'	×
60	self._filter_package(input_zip, output_zip)	×
61	self._validate_layer(output_zip)	×
62	log.info('Uploading layer package to S3 ...')	×
63	self.s3.upload_file(str(output_zip), aws.shared_bucket, self.object_key)	×
64	log.info('Successfully staged updated layer package.')	×
65	else:
66	log.info('Layer package already up-to-date.')	×
67
68	def _build_package(self, out_dir):	×
69	# Delete Chalice's build cache because our layer cache eviction rules
70	# are stricter and we want a full rebuild.
71	try:	×
72	cache_dir = self.layer_dir / '.chalice' / 'deployments'	×
73	log.info('Removing deployment cache at %r', str(cache_dir))	×
74	shutil.rmtree(cache_dir)	×
75	except FileNotFoundError:	×
76	pass	×
77	command = ['chalice', 'package', out_dir]	×
78	log.info('Running %r', command)	×
79	subprocess.run(command, cwd=self.layer_dir).check_returncode()	×
80
81	def _filter_package(self, input_zip_path: Path, output_zip_path: Path):	×
82	"""
83	Filter a ZIP file, removing `app.py` and prefixing other archive member
84	paths with `python/`.
85	"""
86	log.info('Filtering %r to %r', str(input_zip_path), str(output_zip_path))	×
87	with ZipFile(input_zip_path, 'r') as input_zip:	×
88	with ZipFile(output_zip_path, 'w') as output_zip:	×
89	for input in input_zip.infolist():	×
90	if input.filename != 'app.py':	×
91	# ZipFile doesn't copy permissions. Setting permissions
92	# manually also requires setting other fields.
93	output = ZipInfo(filename='python/' + input.filename)	×
94	output.external_attr = input.external_attr	×
95	output.date_time = input.date_time	×
96	output.compress_type = input.compress_type	×
97	with input_zip.open(input, 'r') as rf:	×
98	with output_zip.open(output, 'w') as wf:	×
99	shutil.copyfileobj(rf, wf, length=1024 * 1024)	×
100
101	def _validate_layer(self, layer_zip: Path):	×
102	with ZipFile(layer_zip, 'r') as z:	×
103	infos = z.infolist()	×
104	files = defaultdict(list)	×
105	for info in infos:	×
106	files[info.filename].append(info)	×
107	duplicates = {k: v for k, v in files.items() if len(v) > 1}	×
108	assert not duplicates, duplicates	×
109
110	@cached_property	×
111	def object_key(self):	×
112	sha1 = hashlib.sha1()	×
113	# The `chalice package` command automatically includes Chalice's own
114	# __init__.py and app.py. We need to include them in the hash or else
115	# we risk ignoring changes to those files.
116	import chalice.app	×
117	for module in [chalice, chalice.app]:	×
118	sha1.update(file_sha1(getattr(module, '__file__')).encode())	×
119	for path in Path(config.chalice_bin).iterdir():	×
120	sha1.update(file_sha1(path).encode())	×
121	return f'azul/{config.deployment_stage}/{config.lambda_layer_key}/{sha1.hexdigest()}.zip'	×

DataBiosphere / azul / 18208443738

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous