7790158199

Committed 05 Feb 2024 08:01PM UTC coverage: 85.162% (+0.01%) from 85.151%

Build # 7790158199

Build Type

Pull #3690

github

Committed by

web-flow

Commit Message

Merge 23fad85cd into 33a29aa3f

Pull Request Pull Request #3690: fix(core): migrate Dockerfile after metadata

Run Details

27 of 29 new or added lines in 3 files covered. (93.1%)

5 existing lines in 4 files now uncovered.

26659 of 31304 relevant lines covered (85.16%)

3.67 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.0

/renku/core/migration/migrate.py

# Copyright Swiss Data Science Center (SDSC). A partnership between
# École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Renku migrations management.

Migrations files are put in renku/core/management/migrations directory. Name
of these files has ``m_1234__name.py`` format where 1234 is the migration version
and name can be any alphanumeric and underscore combination. Migration files
are sorted based on their lowercase name. Each migration file must define a
public ``migrate`` function that accepts a ``MigrationContext`` as its argument.

When executing a migration, the migration file is imported as a module and the
``migrate`` function is executed. Renku checks project's metadata version and
applies any migration that has a higher version to the project.
"""

import importlib
import re
import shutil
from pathlib import Path
from typing import Optional, Tuple

from packaging.version import Version

from renku.command.command_builder.command import inject
from renku.core.constant import RENKU_TMP
from renku.core.errors import (
    DockerfileUpdateError,
    MigrationError,
    MigrationRequired,
    ProjectNotSupported,
    TemplateUpdateError,
)
from renku.core.interface.project_gateway import IProjectGateway
from renku.core.migration.models.migration import MigrationContext, MigrationType
from renku.core.migration.utils import is_using_temporary_datasets_path, read_project_version
from renku.core.template.usecase import calculate_dockerfile_checksum, update_dockerfile_checksum
from renku.core.util import communication
from renku.core.util.metadata import (
    is_renku_project,
    read_renku_version_from_dockerfile,
    replace_renku_version_in_dockerfile,
)
from renku.domain_model.project import ProjectTemplateMetadata
from renku.domain_model.project_context import project_context

try:
    import importlib_resources
except ImportError:
    import importlib.resources as importlib_resources  # type: ignore

SUPPORTED_PROJECT_VERSION = 10


def check_for_migration():
    """Checks if migration is required."""
    if is_migration_required():
        raise MigrationRequired
    elif is_project_unsupported():
        raise ProjectNotSupported


def is_migration_required():
    """Check if project requires migration."""
    return is_renku_project() and get_project_version() < SUPPORTED_PROJECT_VERSION


def is_project_unsupported():
    """Check if this version of Renku cannot work with the project."""
    return is_renku_project() and get_project_version() > SUPPORTED_PROJECT_VERSION


def is_docker_update_possible() -> bool:
    """Check if the Dockerfile can be updated to a new version of renku-python."""
    return update_dockerfile(check_only=True)[0]


@inject.autoparams("project_gateway")
def migrate_project(
    project_gateway: IProjectGateway,
    force_template_update=False,
    skip_template_update=False,
    skip_docker_update=False,
    skip_migrations=False,
    project_version=None,
    max_version=None,
    strict=False,
    migration_type: MigrationType = MigrationType.ALL,
    preserve_identifiers=False,
):
    """Migrate all project's entities.

    NOTE: The project path must be pushed to the project_context before calling this function.

    Args:
        project_gateway(IProjectGateway): The injected project gateway.
        force_template_update: Whether to force update the template  (Default value = False).
        skip_template_update: Whether to skip updating the template (Default value = False).
        skip_docker_update: Whether to skip updating the Dockerfile (Default value = False).
        skip_migrations: Whether to skip migrating project metadata (Default value = False).
        project_version: Starting migration version (Default value = False).
        max_version: Apply migration up to the given version (Default value = False).
        strict: Whether to fail on errors (Default value = False).
        migration_type(MigrationType): Type of migration to perform (Default value = MigrationType.ALL).
        preserve_identifiers: Whether to preserve ids when migrating metadata (Default value = False).

    Returns:
        Dictionary of project migration status.
    """
    template_updated = docker_updated = False
    if not is_renku_project():
        return False, template_updated, docker_updated

    n_migrations_executed = 0

    if not skip_migrations:
        project_version = project_version or get_project_version()

        migration_context = MigrationContext(
            strict=strict, type=migration_type, preserve_identifiers=preserve_identifiers
        )

        version = 1
        for version, path in get_migrations():
            if max_version and version > max_version:
                break
            if version > project_version:
                module = importlib.import_module(path)
                module_name = module.__name__.split(".")[-1]
                communication.echo(f"Applying migration {module_name}...")
                try:
                    module.migrate(migration_context)
                except (Exception, BaseException) as e:
                    raise MigrationError("Couldn't execute migration") from e
                n_migrations_executed += 1

        if not is_using_temporary_datasets_path():
            if n_migrations_executed > 0:
                project_context.project.version = str(version)
                project_gateway.update_project(project_context.project)

                communication.echo(f"Successfully applied {n_migrations_executed} migrations.")

            _remove_untracked_renku_files(metadata_path=project_context.metadata_path)

        # we might not have been able to tell if a docker update is possible due to outstanding migrations.
        # so we need to check again here.
        skip_docker_update |= not is_docker_update_possible()

    try:
        project = project_context.project
    except ValueError:
        project = None

    if (
        not skip_template_update
        and project
        and hasattr(project, "template_metadata")
        and isinstance(project.template_metadata, ProjectTemplateMetadata)
        and project.template_metadata.template_source
    ):
        try:
            template_updated = _update_template()
        except TemplateUpdateError:
            raise
        except Exception as e:
            raise TemplateUpdateError("Couldn't update from template.") from e

    if (
        not skip_docker_update
        and project
        and hasattr(project, "template_metadata")
        and isinstance(project.template_metadata, ProjectTemplateMetadata)
    ):
        try:
            docker_updated, _, _ = update_dockerfile()
        except DockerfileUpdateError:
            raise
        except Exception as e:
            raise DockerfileUpdateError("Couldn't update renku version in Dockerfile.") from e

    return n_migrations_executed != 0, template_updated, docker_updated


def _remove_untracked_renku_files(metadata_path):
    from renku.core.constant import CACHE

    untracked_paths = [RENKU_TMP, CACHE, "vendors"]
    for path in untracked_paths:
        path = metadata_path / path
        shutil.rmtree(path, ignore_errors=True)


def _update_template() -> bool:
    """Update local files from the remote template."""
    from renku.core.template.usecase import update_template

    try:
        project = project_context.project
    except ValueError:
        # NOTE: Old project, we don't know the status until it is migrated
        return False

    if not hasattr(project, "template_metadata") or not project.template_metadata.template_version:
        return False

    return bool(update_template(interactive=False, force=False, dry_run=False))


def update_dockerfile(*, check_only=False) -> Tuple[bool, Optional[bool], Optional[str]]:
    """Update the dockerfile to the newest version of renku."""
    from renku import __version__

    if not project_context.dockerfile_path.exists():
        return False, None, None

    with open(project_context.dockerfile_path) as f:
        dockerfile_content = f.read()

    docker_version = read_renku_version_from_dockerfile()
    if not docker_version:
        if check_only:
            return False, None, None
        raise DockerfileUpdateError(
            "Couldn't update renku-python version in Dockerfile, as it doesn't contain an 'ARG RENKU_VERSION=...' line."
        )

    current_version = Version(Version(__version__).base_version)
    if Version(docker_version.base_version) >= current_version:
        return True, False, str(docker_version)

    if check_only:
        return True, True, str(docker_version)

    communication.echo("Updating dockerfile...")

    new_content = replace_renku_version_in_dockerfile(dockerfile_content=dockerfile_content, version=__version__)
    new_checksum = calculate_dockerfile_checksum(dockerfile_content=new_content)

    try:
        update_dockerfile_checksum(new_checksum=new_checksum)
    except DockerfileUpdateError:
        pass

    with open(project_context.dockerfile_path, "w") as f:
        f.write(new_content)

    communication.echo("Updated dockerfile.")

    return True, False, str(current_version)


def get_project_version():
    """Get the metadata version the renku project is on."""
    try:
        return int(read_project_version())
    except ValueError:
        return 1


def get_migrations():
    """Return a sorted list of versions and migration modules."""
    migrations = []
    for entry in importlib_resources.files("renku.core.migration").iterdir():
        match = re.search(r"^m_([0-9]{4})__[a-zA-Z0-9_-]*.py$", entry.name)

        if match is None:  # migration files match m_0000__[name].py format
            continue

        version = int(match.groups()[0])
        path = f"renku.core.migration.{Path(entry.name).stem}"
        migrations.append((version, path))

    migrations = sorted(migrations, key=lambda v: v[1].lower())
    return migrations

1	# Copyright Swiss Data Science Center (SDSC). A partnership between
2	# École Polytechnique Fédérale de Lausanne (EPFL) and
3	# Eidgenössische Technische Hochschule Zürich (ETHZ).
4	#
5	# Licensed under the Apache License, Version 2.0 (the "License");
6	# you may not use this file except in compliance with the License.
7	# You may obtain a copy of the License at
8	#
9	# http://www.apache.org/licenses/LICENSE-2.0
10	#
11	# Unless required by applicable law or agreed to in writing, software
12	# distributed under the License is distributed on an "AS IS" BASIS,
13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	# See the License for the specific language governing permissions and
15	# limitations under the License.
16	"""Renku migrations management.	8✔
17
18	Migrations files are put in renku/core/management/migrations directory. Name
19	of these files has ``m_1234__name.py`` format where 1234 is the migration version
20	and name can be any alphanumeric and underscore combination. Migration files
21	are sorted based on their lowercase name. Each migration file must define a
22	public ``migrate`` function that accepts a ``MigrationContext`` as its argument.
23
24	When executing a migration, the migration file is imported as a module and the
25	``migrate`` function is executed. Renku checks project's metadata version and
26	applies any migration that has a higher version to the project.
27	"""
28
29	import importlib	8✔
30	import re	8✔
31	import shutil	8✔
32	from pathlib import Path	8✔
33	from typing import Optional, Tuple	8✔
34
35	from packaging.version import Version	8✔
36
37	from renku.command.command_builder.command import inject	8✔
38	from renku.core.constant import RENKU_TMP	8✔
39	from renku.core.errors import (	8✔
40	DockerfileUpdateError,
41	MigrationError,
42	MigrationRequired,
43	ProjectNotSupported,
44	TemplateUpdateError,
45	)
46	from renku.core.interface.project_gateway import IProjectGateway	8✔
47	from renku.core.migration.models.migration import MigrationContext, MigrationType	8✔
48	from renku.core.migration.utils import is_using_temporary_datasets_path, read_project_version	8✔
49	from renku.core.template.usecase import calculate_dockerfile_checksum, update_dockerfile_checksum	8✔
50	from renku.core.util import communication	8✔
51	from renku.core.util.metadata import (	8✔
52	is_renku_project,
53	read_renku_version_from_dockerfile,
54	replace_renku_version_in_dockerfile,
55	)
56	from renku.domain_model.project import ProjectTemplateMetadata	8✔
57	from renku.domain_model.project_context import project_context	8✔
58
59	try:	8✔
60	import importlib_resources	8✔
61	except ImportError:	×
62	import importlib.resources as importlib_resources # type: ignore	×
63
64	SUPPORTED_PROJECT_VERSION = 10	8✔
65
66
67	def check_for_migration():	8✔
68	"""Checks if migration is required."""
69	if is_migration_required():	7✔
70	raise MigrationRequired	2✔
71	elif is_project_unsupported():	7✔
72	raise ProjectNotSupported	×
73
74
75	def is_migration_required():	8✔
76	"""Check if project requires migration."""
77	return is_renku_project() and get_project_version() < SUPPORTED_PROJECT_VERSION	7✔
78
79
80	def is_project_unsupported():	8✔
81	"""Check if this version of Renku cannot work with the project."""
82	return is_renku_project() and get_project_version() > SUPPORTED_PROJECT_VERSION	8✔
83
84
85	def is_docker_update_possible() -> bool:	8✔
86	"""Check if the Dockerfile can be updated to a new version of renku-python."""
87	return update_dockerfile(check_only=True)[0]	4✔
88
89
90	@inject.autoparams("project_gateway")	8✔
91	def migrate_project(	8✔
92	project_gateway: IProjectGateway,
93	force_template_update=False,
94	skip_template_update=False,
95	skip_docker_update=False,
96	skip_migrations=False,
97	project_version=None,
98	max_version=None,
99	strict=False,
100	migration_type: MigrationType = MigrationType.ALL,
101	preserve_identifiers=False,
102	):
103	"""Migrate all project's entities.
104
105	NOTE: The project path must be pushed to the project_context before calling this function.
106
107	Args:
108	project_gateway(IProjectGateway): The injected project gateway.
109	force_template_update: Whether to force update the template (Default value = False).
110	skip_template_update: Whether to skip updating the template (Default value = False).
111	skip_docker_update: Whether to skip updating the Dockerfile (Default value = False).
112	skip_migrations: Whether to skip migrating project metadata (Default value = False).
113	project_version: Starting migration version (Default value = False).
114	max_version: Apply migration up to the given version (Default value = False).
115	strict: Whether to fail on errors (Default value = False).
116	migration_type(MigrationType): Type of migration to perform (Default value = MigrationType.ALL).
117	preserve_identifiers: Whether to preserve ids when migrating metadata (Default value = False).
118
119	Returns:
120	Dictionary of project migration status.
121	"""
122	template_updated = docker_updated = False	4✔
123	if not is_renku_project():	4✔
124	return False, template_updated, docker_updated	×
125
126	n_migrations_executed = 0	4✔
127
128	if not skip_migrations:	4✔
129	project_version = project_version or get_project_version()	4✔
130
131	migration_context = MigrationContext(	4✔
132	strict=strict, type=migration_type, preserve_identifiers=preserve_identifiers
133	)
134
135	version = 1	4✔
136	for version, path in get_migrations():	4✔
137	if max_version and version > max_version:	4✔
138	break	3✔
139	if version > project_version:	4✔
140	module = importlib.import_module(path)	4✔
141	module_name = module.__name__.split(".")[-1]	4✔
142	communication.echo(f"Applying migration {module_name}...")	4✔
143	try:	4✔
144	module.migrate(migration_context)	4✔
NEW 145	except (Exception, BaseException) as e:	×
NEW 146	raise MigrationError("Couldn't execute migration") from e	×
147	n_migrations_executed += 1	4✔
148
149	if not is_using_temporary_datasets_path():	4✔
150	if n_migrations_executed > 0:	4✔
151	project_context.project.version = str(version)	4✔
152	project_gateway.update_project(project_context.project)	4✔
153
154	communication.echo(f"Successfully applied {n_migrations_executed} migrations.")	4✔
155
156	_remove_untracked_renku_files(metadata_path=project_context.metadata_path)	4✔
157
158	# we might not have been able to tell if a docker update is possible due to outstanding migrations.
159	# so we need to check again here.
160	skip_docker_update \|= not is_docker_update_possible()	4✔
161
162	try:	4✔
163	project = project_context.project	4✔
164	except ValueError:	3✔
165	project = None	3✔
166
167	if (	4✔
168	not skip_template_update
169	and project
170	and hasattr(project, "template_metadata")
171	and isinstance(project.template_metadata, ProjectTemplateMetadata)
172	and project.template_metadata.template_source
173	):
174	try:	1✔
175	template_updated = _update_template()	1✔
176	except TemplateUpdateError:	×
177	raise	×
178	except Exception as e:	×
179	raise TemplateUpdateError("Couldn't update from template.") from e	×
180
181	if (	4✔
182	not skip_docker_update
183	and project
184	and hasattr(project, "template_metadata")
185	and isinstance(project.template_metadata, ProjectTemplateMetadata)
186	):
187	try:	3✔
188	docker_updated, _, _ = update_dockerfile()	3✔
189	except DockerfileUpdateError:	×
190	raise	×
191	except Exception as e:	×
192	raise DockerfileUpdateError("Couldn't update renku version in Dockerfile.") from e	×
193
194	return n_migrations_executed != 0, template_updated, docker_updated	4✔
195
196
197	def _remove_untracked_renku_files(metadata_path):	8✔
198	from renku.core.constant import CACHE	4✔
199
200	untracked_paths = [RENKU_TMP, CACHE, "vendors"]	4✔
201	for path in untracked_paths:	4✔
202	path = metadata_path / path	4✔
203	shutil.rmtree(path, ignore_errors=True)	4✔
204
205
206	def _update_template() -> bool:	8✔
207	"""Update local files from the remote template."""
208	from renku.core.template.usecase import update_template	1✔
209
210	try:	1✔
211	project = project_context.project	1✔
212	except ValueError:	×
213	# NOTE: Old project, we don't know the status until it is migrated
214	return False	×
215
216	if not hasattr(project, "template_metadata") or not project.template_metadata.template_version:	1✔
217	return False	×
218
219	return bool(update_template(interactive=False, force=False, dry_run=False))	1✔
220
221
222	def update_dockerfile(*, check_only=False) -> Tuple[bool, Optional[bool], Optional[str]]:	8✔
223	"""Update the dockerfile to the newest version of renku."""
224	from renku import __version__	4✔
225
226	if not project_context.dockerfile_path.exists():	4✔
227	return False, None, None	×
228
229	with open(project_context.dockerfile_path) as f:	4✔
230	dockerfile_content = f.read()	4✔
231
232	docker_version = read_renku_version_from_dockerfile()	4✔
233	if not docker_version:	4✔
234	if check_only:	3✔
235	return False, None, None	3✔
236	raise DockerfileUpdateError(	×
237	"Couldn't update renku-python version in Dockerfile, as it doesn't contain an 'ARG RENKU_VERSION=...' line."
238	)
239
240	current_version = Version(Version(__version__).base_version)	4✔
241	if Version(docker_version.base_version) >= current_version:	4✔
242	return True, False, str(docker_version)	1✔
243
244	if check_only:	4✔
245	return True, True, str(docker_version)	4✔
246
247	communication.echo("Updating dockerfile...")	3✔
248
249	new_content = replace_renku_version_in_dockerfile(dockerfile_content=dockerfile_content, version=__version__)	3✔
250	new_checksum = calculate_dockerfile_checksum(dockerfile_content=new_content)	3✔
251
252	try:	3✔
253	update_dockerfile_checksum(new_checksum=new_checksum)	3✔
254	except DockerfileUpdateError:	2✔
255	pass	2✔
256
257	with open(project_context.dockerfile_path, "w") as f:	3✔
258	f.write(new_content)	3✔
259
260	communication.echo("Updated dockerfile.")	3✔
261
262	return True, False, str(current_version)	3✔
263
264
265	def get_project_version():	8✔
266	"""Get the metadata version the renku project is on."""
267	try:	8✔
268	return int(read_project_version())	8✔
269	except ValueError:	×
270	return 1	×
271
272
273	def get_migrations():	8✔
274	"""Return a sorted list of versions and migration modules."""
275	migrations = []	4✔
276	for entry in importlib_resources.files("renku.core.migration").iterdir():	4✔
277	match = re.search(r"^m_([0-9]{4})__[a-zA-Z0-9_-]*.py$", entry.name)	4✔
278
279	if match is None: # migration files match m_0000__[name].py format	4✔
280	continue	4✔
281
282	version = int(match.groups()[0])	4✔
283	path = f"renku.core.migration.{Path(entry.name).stem}"	4✔
284	migrations.append((version, path))	4✔
285
286	migrations = sorted(migrations, key=lambda v: v[1].lower())	4✔
287	return migrations	4✔

SwissDataScienceCenter / renku-python / 7790158199

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous