thht / plus_sync / 158

Committed 28 Jan 2025 10:05AM UTC coverage: 91.476%. Remained the same

Build # 158

Build Type

push

gitlab-ci

Committed by

Commit Message

Merge branch 'fix_gitlab_lfs' into 'main'

Fix LFS

See merge request thht_science_software/plus_sync!29

Run Details

0 of 1 new or added line in 1 file covered. (0.0%)

880 of 962 relevant lines covered (91.48%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.26

/src/plus_sync/sync/gitlab.py

import datetime
import typing
from fnmatch import fnmatch
from functools import cache
from pathlib import Path
from typing import TYPE_CHECKING, Any

import gitlab
import gitlab.v4.objects
import joblib
from gitlab.utils import EncodedId
from requests.structures import CaseInsensitiveDict

from .base import BaseSync, FileInformation

if TYPE_CHECKING:
    import plus_sync.config


def _fetch_metadata(file_path: str, gl: gitlab.Gitlab, project_id: int, branch: str) -> CaseInsensitiveDict[Any]:
    return gl.http_head(
        f'/projects/{project_id}/repository/files/{EncodedId(file_path)}', query_parameters={'ref': branch}
    )


class GitlabAccess(BaseSync):
    def __init__(self, gitlab_project: 'plus_sync.config.GitlabConfig'):
        self.gitlab_project = gitlab_project
        if self.gitlab_project.host is None:
            raise ValueError('Host must be set for Gitlab project.')
        token = Path(gitlab_project.token_file).read_text().strip()
        self.gl = gitlab.Gitlab(self.gitlab_project.host, private_token=token)
        self.project = self.gl.projects.get(self.gitlab_project.slug)
        self.commits: list[gitlab.v4.objects.ProjectCommit] = typing.cast(
            list[gitlab.v4.objects.ProjectCommit], self.project.commits.list(all=True)
        )
        self.metadata_cache: dict[str, dict] = {}

    def _get_files(self, with_metadata: bool = True) -> list[FileInformation]:
        raw_all_files = self._get_files_local()
        if with_metadata:
            self._fetch_metadata(raw_all_files)

            all_files = [
                FileInformation(
                    path=x['path'],
                    size=self._get_file_size(x),
                    last_modified=self._get_file_lastmod(x),
                    hashes={'sha256': self._get_file_sha256(x)},
                )
                for x in raw_all_files
            ]
        else:
            all_files = [FileInformation(path=x['path']) for x in raw_all_files]
        return all_files

    def _get_files_local(self) -> list[dict]:
        all_files = []

        for cur_path in self.gitlab_project.paths:
            files_generator = self.project.repository_tree(
                path=cur_path, recursive=True, iterator=True, ref=self.gitlab_project.branch
            )
            file_list = [
                x
                for x in files_generator
                if x['type'] == 'blob' and any(fnmatch(x['path'], glob) for glob in self.gitlab_project.globs)
            ]
            all_files.extend(file_list)

        return all_files

    def _get_commit(self, id: str) -> gitlab.v4.objects.ProjectCommit:
        return [x for x in self.commits if x.id == id][0]

    def _fetch_metadata(self, file_paths: list[dict]) -> None:
        def d_fetch(f: dict) -> CaseInsensitiveDict[Any]:
            return _fetch_metadata(f['path'], self.gl, self.project.id, self.gitlab_project.branch)

        tmp = joblib.Parallel(n_jobs=30)(joblib.delayed(d_fetch)(f) for f in file_paths)
        self.metadata_cache = {x['x-gitlab-file-path']: x for x in tmp}

    @cache
    def _get_file_meta(self, file_path: str) -> dict[str, str]:
        return self.metadata_cache[file_path]

    def _get_file_sha256(self, file: dict[str, str]) -> str:
        return self._get_file_meta(file['path'])['x-gitlab-content-sha256']

    def _get_file_size(self, file: dict[str, str]) -> str:
        return self._get_file_meta(file['path'])['x-gitlab-size']

    def _get_file_last_commit(self, file: dict[str, str]) -> str:
        return self._get_file_meta(file['path'])['x-gitlab-last-commit-id']

    def _get_file_lastmod(self, file: dict[str, str]) -> datetime.datetime:
        commit_id = self._get_file_last_commit(file)
        return datetime.datetime.fromisoformat(self._get_commit(commit_id).committed_date)

    def get_content(self, file: FileInformation) -> bytes:
        return self.project.files.raw(file.path, ref=self.gitlab_project.branch, lfs=True)

1	import datetime	1✔
2	import typing	1✔
3	from fnmatch import fnmatch	1✔
4	from functools import cache	1✔
5	from pathlib import Path	1✔
6	from typing import TYPE_CHECKING, Any	1✔
7
8	import gitlab	1✔
9	import gitlab.v4.objects	1✔
10	import joblib	1✔
11	from gitlab.utils import EncodedId	1✔
12	from requests.structures import CaseInsensitiveDict	1✔
13
14	from .base import BaseSync, FileInformation	1✔
15
16	if TYPE_CHECKING:	1✔
17	import plus_sync.config	×
18
19
20	def _fetch_metadata(file_path: str, gl: gitlab.Gitlab, project_id: int, branch: str) -> CaseInsensitiveDict[Any]:	1✔
21	return gl.http_head(	×
22	f'/projects/{project_id}/repository/files/{EncodedId(file_path)}', query_parameters={'ref': branch}
23	)
24
25
26	class GitlabAccess(BaseSync):	1✔
27	def __init__(self, gitlab_project: 'plus_sync.config.GitlabConfig'):	1✔
28	self.gitlab_project = gitlab_project	×
29	if self.gitlab_project.host is None:	×
30	raise ValueError('Host must be set for Gitlab project.')	×
31	token = Path(gitlab_project.token_file).read_text().strip()	×
32	self.gl = gitlab.Gitlab(self.gitlab_project.host, private_token=token)	×
33	self.project = self.gl.projects.get(self.gitlab_project.slug)	×
34	self.commits: list[gitlab.v4.objects.ProjectCommit] = typing.cast(	×
35	list[gitlab.v4.objects.ProjectCommit], self.project.commits.list(all=True)
36	)
37	self.metadata_cache: dict[str, dict] = {}	×
38
39	def _get_files(self, with_metadata: bool = True) -> list[FileInformation]:	1✔
40	raw_all_files = self._get_files_local()	×
41	if with_metadata:	×
42	self._fetch_metadata(raw_all_files)	×
43
44	all_files = [	×
45	FileInformation(
46	path=x['path'],
47	size=self._get_file_size(x),
48	last_modified=self._get_file_lastmod(x),
49	hashes={'sha256': self._get_file_sha256(x)},
50	)
51	for x in raw_all_files
52	]
53	else:
54	all_files = [FileInformation(path=x['path']) for x in raw_all_files]	×
55	return all_files	×
56
57	def _get_files_local(self) -> list[dict]:	1✔
58	all_files = []	×
59
60	for cur_path in self.gitlab_project.paths:	×
61	files_generator = self.project.repository_tree(	×
62	path=cur_path, recursive=True, iterator=True, ref=self.gitlab_project.branch
63	)
64	file_list = [	×
65	x
66	for x in files_generator
67	if x['type'] == 'blob' and any(fnmatch(x['path'], glob) for glob in self.gitlab_project.globs)
68	]
69	all_files.extend(file_list)	×
70
71	return all_files	×
72
73	def _get_commit(self, id: str) -> gitlab.v4.objects.ProjectCommit:	1✔
74	return [x for x in self.commits if x.id == id][0]	×
75
76	def _fetch_metadata(self, file_paths: list[dict]) -> None:	1✔
77	def d_fetch(f: dict) -> CaseInsensitiveDict[Any]:	×
78	return _fetch_metadata(f['path'], self.gl, self.project.id, self.gitlab_project.branch)	×
79
80	tmp = joblib.Parallel(n_jobs=30)(joblib.delayed(d_fetch)(f) for f in file_paths)	×
81	self.metadata_cache = {x['x-gitlab-file-path']: x for x in tmp}	×
82
83	@cache	1✔
84	def _get_file_meta(self, file_path: str) -> dict[str, str]:	1✔
85	return self.metadata_cache[file_path]	×
86
87	def _get_file_sha256(self, file: dict[str, str]) -> str:	1✔
88	return self._get_file_meta(file['path'])['x-gitlab-content-sha256']	×
89
90	def _get_file_size(self, file: dict[str, str]) -> str:	1✔
91	return self._get_file_meta(file['path'])['x-gitlab-size']	×
92
93	def _get_file_last_commit(self, file: dict[str, str]) -> str:	1✔
94	return self._get_file_meta(file['path'])['x-gitlab-last-commit-id']	×
95
96	def _get_file_lastmod(self, file: dict[str, str]) -> datetime.datetime:	1✔
97	commit_id = self._get_file_last_commit(file)	×
98	return datetime.datetime.fromisoformat(self._get_commit(commit_id).committed_date)	×
99
100	def get_content(self, file: FileInformation) -> bytes:	1✔
NEW 101	return self.project.files.raw(file.path, ref=self.gitlab_project.branch, lfs=True)	×

thht / plus_sync / 158

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous