9058668052

Committed 13 May 2024 07:05AM UTC coverage: 77.713% (-8.4%) from 86.115%

Build # 9058668052

Build Type

Pull #3727

github

Committed by

web-flow

Commit Message

Merge 128d38387 into 050ed61bf

Pull Request Pull Request #3727: fix: don't fail session launch when gitlab couldn't be reached

Run Details

15 of 29 new or added lines in 3 files covered. (51.72%)

2594 existing lines in 125 files now uncovered.

23893 of 30745 relevant lines covered (77.71%)

3.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.17

/renku/command/checks/workflow.py

# Copyright Swiss Data Science Center (SDSC). A partnership between
# École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Checks needed to determine integrity of workflows."""

from datetime import timedelta
from typing import List, Optional, Tuple, cast

from renku.command.command_builder import inject
from renku.command.util import WARNING
from renku.core.interface.plan_gateway import IPlanGateway
from renku.core.util import communication
from renku.domain_model.project_context import project_context
from renku.domain_model.workflow.plan import AbstractPlan
from renku.infrastructure.gateway.activity_gateway import reindex_catalog


def check_activity_catalog(fix, force, **_) -> Tuple[bool, bool, Optional[str]]:
    """Check if the activity-catalog needs to be rebuilt.

    Args:
        fix: Whether to fix found issues.
        force: Whether to force rebuild the activity catalog.
        _: keyword arguments.

    Returns:
        Tuple of whether the activity-catalog needs to be rebuilt, if an automated fix is available and a string of
            found problems.
    """
    database = project_context.database
    activity_catalog = database["activity-catalog"]
    relations = database["_downstream_relations"]

    # NOTE: If len(activity_catalog) > 0 then either the project is fixed or it used a fixed Renku version but still has
    # broken metadata. ``force`` allows to rebuild the metadata in the latter case.
    if (len(relations) == 0 or len(activity_catalog) > 0) and not (force and fix):
        return True, False, None

    if not fix:
        problems = (
            WARNING + "The project's workflow metadata needs to be rebuilt (use 'renku doctor --fix' to rebuild it).\n"
        )

        return False, True, problems

    with communication.busy("Rebuilding workflow metadata ..."):
        reindex_catalog(database=database)

    communication.info("Workflow metadata was rebuilt")

    return True, False, None


@inject.autoparams("plan_gateway")
def check_plan_modification_date(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, bool, Optional[str]]:
    """Check if all plans have modification date set for them.

    Args:
        fix(bool): Whether to fix found issues.
        plan_gateway(IPlanGateway): Injected PlanGateway.
        _: keyword arguments.

    Returns:
        Tuple[bool, Optional[str]]: Tuple of whether there are plans without modification date, if an automated fix is
            available and a string of their IDs
    """
    plans: List[AbstractPlan] = plan_gateway.get_all_plans()

    to_be_processed = []
    for plan in plans:
        if not hasattr(plan, "date_modified") or plan.date_modified is None:
            to_be_processed.append(plan)

    if not to_be_processed:
        return True, False, None
    if not fix:
        ids = [plan.id for plan in to_be_processed]
        message = (
            WARNING
            + "The following workflows have incorrect modification date (use 'renku doctor --fix' to fix them):\n\t"
            + "\n\t".join(ids)
        )
        return False, True, message

    fix_plan_dates(plans=to_be_processed, plan_gateway=plan_gateway)
    project_context.database.commit()
    communication.info("Workflow modification dates were fixed")

    return True, False, None


def fix_plan_dates(plans: List[AbstractPlan], plan_gateway):
    """Set modification date on a list of plans and fix their creation date."""
    processed = set()
    # NOTE: switch creation date for modification date
    for tail in plans:
        to_be_processed: List[AbstractPlan] = []
        if tail not in processed:
            processed.add(tail)
            to_be_processed.append(tail)
        creation_date = tail.date_created
        plan = tail

        while plan.is_derivation():
            plan = cast(AbstractPlan, plan_gateway.get_by_id(plan.derived_from))
            creation_date = plan.date_created
            if plan not in processed:
                processed.add(plan)
                to_be_processed.append(plan)

        while to_be_processed:
            plan = to_be_processed.pop()
            plan.unfreeze()
            plan.date_modified = plan.date_created
            plan.date_created = creation_date
            if plan.date_removed and plan.date_removed < plan.date_created:
                plan.date_removed = plan.date_created + timedelta(seconds=1)
            plan.freeze()


@inject.autoparams("plan_gateway")
def check_plan_id(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, bool, Optional[str]]:
    """Check if all plans have correct IDs.

    Args:
        fix(bool): Whether to fix found issues.
        plan_gateway(IPlanGateway): Injected PlanGateway.
        _: keyword arguments.

    Returns:
        Tuple[bool, Optional[str]]: Tuple of whether there are plans with invalid IDs, if an automated fix is
            available and a string of their IDs
    """
    plans: List[AbstractPlan] = plan_gateway.get_all_plans()

    to_be_processed = []
    to_be_processed_derived = []
    for plan in plans:
        if isinstance(plan.id, str) and plan.id.startswith("/plans//plans"):
            to_be_processed.append(plan)
        if isinstance(plan.derived_from, str) and plan.derived_from.startswith("/plans//plans"):
            to_be_processed_derived.append(plan)

    if not to_be_processed and not to_be_processed_derived:
        return True, False, None
    if not fix:
        ids = [plan.id for plan in to_be_processed + to_be_processed_derived]
        message = (
            WARNING
            + "The following workflows have incorrect IDs (use 'renku doctor --fix' to fix them):\n\t"
            + "\n\t".join(ids)
        )
        return False, True, message

    for plan in to_be_processed:
        plan.unfreeze()
        plan.id = plan.id.replace("//plans/", "/")
        plan.reassign_oid()
        plan._p_changed = True
        plan.freeze()

    for plan in to_be_processed_derived:
        if plan.derived_from is not None:
            plan.unfreeze()
            plan.derived_from = plan.derived_from.replace("//plans/", "/")
            plan._p_changed = True
            plan.freeze()

    project_context.database.commit()
    communication.info("Workflow IDs were fixed")

    return True, False, None

1	# Copyright Swiss Data Science Center (SDSC). A partnership between
2	# École Polytechnique Fédérale de Lausanne (EPFL) and
3	# Eidgenössische Technische Hochschule Zürich (ETHZ).
4	#
5	# Licensed under the Apache License, Version 2.0 (the "License");
6	# you may not use this file except in compliance with the License.
7	# You may obtain a copy of the License at
8	#
9	# http://www.apache.org/licenses/LICENSE-2.0
10	#
11	# Unless required by applicable law or agreed to in writing, software
12	# distributed under the License is distributed on an "AS IS" BASIS,
13	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14	# See the License for the specific language governing permissions and
15	# limitations under the License.
16	"""Checks needed to determine integrity of workflows."""	5✔
17
18	from datetime import timedelta	5✔
19	from typing import List, Optional, Tuple, cast	5✔
20
21	from renku.command.command_builder import inject	5✔
22	from renku.command.util import WARNING	5✔
23	from renku.core.interface.plan_gateway import IPlanGateway	5✔
24	from renku.core.util import communication	5✔
25	from renku.domain_model.project_context import project_context	5✔
26	from renku.domain_model.workflow.plan import AbstractPlan	5✔
27	from renku.infrastructure.gateway.activity_gateway import reindex_catalog	5✔
28
29
30	def check_activity_catalog(fix, force, **_) -> Tuple[bool, bool, Optional[str]]:	5✔
31	"""Check if the activity-catalog needs to be rebuilt.
32
33	Args:
34	fix: Whether to fix found issues.
35	force: Whether to force rebuild the activity catalog.
36	_: keyword arguments.
37
38	Returns:
39	Tuple of whether the activity-catalog needs to be rebuilt, if an automated fix is available and a string of
40	found problems.
41	"""
42	database = project_context.database	2✔
43	activity_catalog = database["activity-catalog"]	2✔
44	relations = database["_downstream_relations"]	2✔
45
46	# NOTE: If len(activity_catalog) > 0 then either the project is fixed or it used a fixed Renku version but still has
47	# broken metadata. ``force`` allows to rebuild the metadata in the latter case.
48	if (len(relations) == 0 or len(activity_catalog) > 0) and not (force and fix):	2✔
49	return True, False, None	2✔
50
51	if not fix:	1✔
52	problems = (	1✔
53	WARNING + "The project's workflow metadata needs to be rebuilt (use 'renku doctor --fix' to rebuild it).\n"
54	)
55
56	return False, True, problems	1✔
57
58	with communication.busy("Rebuilding workflow metadata ..."):	1✔
59	reindex_catalog(database=database)	1✔
60
61	communication.info("Workflow metadata was rebuilt")	1✔
62
63	return True, False, None	1✔
64
65
66	@inject.autoparams("plan_gateway")	5✔
67	def check_plan_modification_date(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, bool, Optional[str]]:	5✔
68	"""Check if all plans have modification date set for them.
69
70	Args:
71	fix(bool): Whether to fix found issues.
72	plan_gateway(IPlanGateway): Injected PlanGateway.
73	_: keyword arguments.
74
75	Returns:
76	Tuple[bool, Optional[str]]: Tuple of whether there are plans without modification date, if an automated fix is
77	available and a string of their IDs
78	"""
79	plans: List[AbstractPlan] = plan_gateway.get_all_plans()	2✔
80
81	to_be_processed = []	2✔
82	for plan in plans:	2✔
83	if not hasattr(plan, "date_modified") or plan.date_modified is None:	1✔
84	to_be_processed.append(plan)	1✔
85
86	if not to_be_processed:	2✔
87	return True, False, None	2✔
88	if not fix:	1✔
UNCOV 89	ids = [plan.id for plan in to_be_processed]	×
90	message = (	×
91	WARNING
92	+ "The following workflows have incorrect modification date (use 'renku doctor --fix' to fix them):\n\t"
93	+ "\n\t".join(ids)
94	)
UNCOV 95	return False, True, message	×
96
97	fix_plan_dates(plans=to_be_processed, plan_gateway=plan_gateway)	1✔
98	project_context.database.commit()	1✔
99	communication.info("Workflow modification dates were fixed")	1✔
100
101	return True, False, None	1✔
102
103
104	def fix_plan_dates(plans: List[AbstractPlan], plan_gateway):	5✔
105	"""Set modification date on a list of plans and fix their creation date."""
106	processed = set()	4✔
107	# NOTE: switch creation date for modification date
108	for tail in plans:	4✔
109	to_be_processed: List[AbstractPlan] = []	2✔
110	if tail not in processed:	2✔
111	processed.add(tail)	2✔
112	to_be_processed.append(tail)	2✔
113	creation_date = tail.date_created	2✔
114	plan = tail	2✔
115
116	while plan.is_derivation():	2✔
117	plan = cast(AbstractPlan, plan_gateway.get_by_id(plan.derived_from))	2✔
118	creation_date = plan.date_created	2✔
119	if plan not in processed:	2✔
120	processed.add(plan)	2✔
121	to_be_processed.append(plan)	2✔
122
123	while to_be_processed:	2✔
124	plan = to_be_processed.pop()	2✔
125	plan.unfreeze()	2✔
126	plan.date_modified = plan.date_created	2✔
127	plan.date_created = creation_date	2✔
128	if plan.date_removed and plan.date_removed < plan.date_created:	2✔
UNCOV 129	plan.date_removed = plan.date_created + timedelta(seconds=1)	×
130	plan.freeze()	2✔
131
132
133	@inject.autoparams("plan_gateway")	5✔
134	def check_plan_id(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, bool, Optional[str]]:	5✔
135	"""Check if all plans have correct IDs.
136
137	Args:
138	fix(bool): Whether to fix found issues.
139	plan_gateway(IPlanGateway): Injected PlanGateway.
140	_: keyword arguments.
141
142	Returns:
143	Tuple[bool, Optional[str]]: Tuple of whether there are plans with invalid IDs, if an automated fix is
144	available and a string of their IDs
145	"""
146	plans: List[AbstractPlan] = plan_gateway.get_all_plans()	2✔
147
148	to_be_processed = []	2✔
149	to_be_processed_derived = []	2✔
150	for plan in plans:	2✔
151	if isinstance(plan.id, str) and plan.id.startswith("/plans//plans"):	1✔
152	to_be_processed.append(plan)	1✔
153	if isinstance(plan.derived_from, str) and plan.derived_from.startswith("/plans//plans"):	1✔
NEW 154	to_be_processed_derived.append(plan)	×
155
156	if not to_be_processed and not to_be_processed_derived:	2✔
157	return True, False, None	2✔
158	if not fix:	1✔
NEW 159	ids = [plan.id for plan in to_be_processed + to_be_processed_derived]	×
UNCOV 160	message = (	×
161	WARNING
162	+ "The following workflows have incorrect IDs (use 'renku doctor --fix' to fix them):\n\t"
163	+ "\n\t".join(ids)
164	)
UNCOV 165	return False, True, message	×
166
167	for plan in to_be_processed:	1✔
168	plan.unfreeze()	1✔
169	plan.id = plan.id.replace("//plans/", "/")	1✔
170	plan.reassign_oid()	1✔
171	plan._p_changed = True	1✔
172	plan.freeze()	1✔
173
174	for plan in to_be_processed_derived:	1✔
NEW 175	if plan.derived_from is not None:	×
NEW 176	plan.unfreeze()	×
NEW 177	plan.derived_from = plan.derived_from.replace("//plans/", "/")	×
NEW 178	plan._p_changed = True	×
NEW 179	plan.freeze()	×
180
181	project_context.database.commit()	1✔
182	communication.info("Workflow IDs were fixed")	1✔
183
184	return True, False, None	1✔

SwissDataScienceCenter / renku-python / 9058668052

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous