19471672015

Committed 18 Nov 2025 03:32PM UTC coverage: 79.472% (+0.02%) from 79.449%

Build # 19471672015

Build Type

push

github

Committed by

AGrigis

Commit Message

doc: fix documentation.

Coverage Stats

1444 of 1817 relevant lines covered (79.47%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.7

/brainprep/datasets/multi_modal.py

##########################################################################
# NSAp - Copyright (C) CEA, 2025
# Distributed under the terms of the CeCILL-B license, as published by
# the CEA-CNRS-INRIA. Refer to the LICENSE file or to
# http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
# for details.
##########################################################################

"""
Fetcher to download anatomical data.
"""

import gzip
import json
import os
import shutil
import tarfile
from pathlib import Path

import pandas as pd
import requests

from ..typing import (
    Directory,
)
from ..utils import (
    Bunch,
    print_info,
)


class MultiModalDataset:
    """
    Multi-modal dataset.

    This `dataset <https://www.nitrc.org/projects/multimodal>`_
    :footcite:p:`landman2011data` contains Magnetic Resonance (MR) images of
    21 healthy volunteers: scan-rescan imaging sessions including MPRAGE,
    FLAIR, DTI, resting state fMRI, B0 and B1 field maps, ...

    Parameters
    ----------
    datadir : Directory
        Directory where data will be stored.

    Attributes
    ----------
    _url : str
        Internal URL used to fetch data.

    Examples
    --------
    >>> from brainprep.config import Config
    >>> from pathlib import Path
    >>> from brainprep.datasets import MultiModalDataset
    >>>
    >>> datadir = Path("/tmp/brainprep-data")
    >>> datadir.mkdir(parents=True, exist_ok=True)
    >>> dataset = MultiModalDataset(datadir)
    >>> with Config(verbose=False):
    ...     data = dataset.fetch(
    ...         subject="01",
    ...         modality="func",
    ...         session="01",
    ...     )
    >>> data
    Bunch(
      description: PosixPath('...')
      anat: PosixPath('...')
      fmap1: PosixPath('...')
      fmap2: PosixPath('...')
      func: PosixPath('...')
    )

    References
    ----------

    .. footbibliography::
    """

    _url: str = "https://www.nitrc.org/frs/downloadlink.php/22{visit}"

    def __init__(
            self,
            datadir: Directory) -> None:
        self.datadir = datadir
        self.allowed_subjects = [
            f"{str(subject).zfill(2)}" for subject in range(1, 22)
        ]
        self.allowed_modalities = [
            "func",
            "dwi",
        ]
        self.allowed_sessions = [
            "01",
            "02",
        ]
        self.meta = pd.read_csv(
            Path(__file__).parent.parent / "resources" / "kirby21.tsv",
            sep="\t",
        )
        self.meta["Subject"] = self.meta["Subject"].map(
            {
                val: str(idx + 1).zfill(2)
                for idx, val in enumerate(pd.unique(self.meta["Subject"]))
            }
        )
        self.meta["Visit"] = self.meta["Visit"].astype(str).apply(
            lambda x: x.zfill(2)
        )

    def fetch(
            self,
            subject: str,
            modality: str,
            session: str) -> Bunch:
        """ Fetch data.

        Parameters
        ----------
        subject : str
            Subject identifier: ['01' - '21'].
        modality : str
            Modality to be fetched: 'func' or 'dwi'.
        session : str
            Session: '01' or '02'.

        Returns
        -------
        dataset: Bunch
            Fetched data path. A 'description' entry is also available.

        Raises
        ------
        ValueError
            If data could not be extracted from the fetched archive.
        """
        dataset = Bunch()
        self.sanity_check(subject, modality, session)

        description_file = (
            self.datadir /
            "rawdata" /
            "dataset_description.json"
        )
        if not description_file.is_file():
            description_file.parent.mkdir(parents=True, exist_ok=True)
            with open(description_file, "w") as of:
                description = {
                    "Name": "Example dataset",
                    "BIDSVersion": "1.0.2",
                }
                json.dump(description, of, indent=4)
        dataset["description"] = description_file

        selection = self.meta[self.meta["Subject"] == subject]
        visit = selection.iloc[int(session) - 1]["Visit"]
        url = self._url.format(
            visit=visit,
        )
        archive_file = (
                self.datadir /
                "sourcedata" /
                f"KKI2009-{visit}.tar.bz2"
        )
        if not archive_file.is_file():
            print_info(f"downloading: {url}")
            archive_file.parent.mkdir(parents=True, exist_ok=True)
            response = requests.get(url)
            response.raise_for_status()
            with open(archive_file, "wb") as of:
                of.write(response.content)

        to_extract = [
            ("anat", "T1w.nii", "MPRAGE.nii"),
            ("anat", "T1w.json", "MPRAGE.par"),
            ("fmap", "magnitude1.nii", "B08MS.nii"),
            ("fmap", "magnitude1.json", "B08MS.par"),
            ("fmap", "magnitude2.nii", "B09MS.nii"),
            ("fmap", "magnitude2.json", "B09MS.par"),
        ]
        if modality == "dwi":
            to_extract += [
                ("dwi", "dwi.nii", "DTI.nii"),
                ("dwi", "dwi.json", "DTI.par"),
                ("dwi", "dwi.bvec", "DTI.grad"),
                ("dwi", "dwi.bval", "DTI.b"),
            ]
        else:
            to_extract += [
                ("func", "task-rest_bold.nii", "fMRI.nii"),
                ("func", "task-rest_bold.json", "fMRI.par"),
            ]

        with tarfile.open(archive_file, "r:bz2") as tar:
            fmap_counter = 1
            for dtype, dest_suffix, src_suffix in to_extract:
                destination_file = (
                    self.datadir /
                    "rawdata" /
                    f"sub-{subject}" /
                    f"ses-{session}" /
                    dtype /
                    f"sub-{subject}_ses-{session}_{dest_suffix}"
                )
                if dest_suffix.endswith(".nii"):
                    if dtype == "fmap":
                        dtype = f"{dtype}{fmap_counter}"
                        fmap_counter += 1
                    dataset[dtype] = destination_file
                    if destination_file.with_suffix(".nii.gz").is_file():
                        continue
                elif dest_suffix.endswith((".bvec", ".bval")):
                    dataset[dest_suffix.split(".")[1]] = destination_file
                if destination_file.is_file():
                    continue
                destination_file.parent.mkdir(parents=True, exist_ok=True)
                file_to_extract = f"KKI2009-{visit}-{src_suffix}"
                print_info(f"extracting: {file_to_extract}")
                fileobj = tar.extractfile(file_to_extract)
                if fileobj is not None:
                    with open(destination_file, "wb") as of:
                        of.write(fileobj.read())
                else:
                    raise ValueError(
                        f"Could not extract '{file_to_extract}' from the "
                        "archive."
                    )

        for key, src_file in dataset.items():
            if src_file.suffix != ".nii":
                continue
            dest_file = src_file.with_suffix(".nii.gz")
            dataset[key] = dest_file
            if dest_file.is_file():
                continue
            with (open(src_file, "rb") as f_in,
                  gzip.open(dest_file, "wb") as f_out):
                shutil.copyfileobj(f_in, f_out)
            os.remove(src_file)

        return dataset

    def sanity_check(
            self,
            subject: str,
            modality: str,
            session: str) -> None:
        """ Check that the fetch parameters are correct.

        Parameters
        ----------
        subject : str
            the subject identifier. This identifier must lie in ['01' - '30'],
            ['01' - '20'], for cross sectional or longitudinal data
            respectively.
        modality : str
            the modality to be fetched: 'dwi', or 'func'.
        session : str
            Session: '01' or '02'.

        Raises
        ------
        ValueError
            If the fetch input parameters are not correct.
        """
        if session not in self.allowed_sessions:
            raise ValueError(f"Unexpected session: {session}.")
        if modality not in self.allowed_modalities:
            raise ValueError(f"Unexpected modality: {modality}.")
        if subject not in self.allowed_subjects:
            raise ValueError(f"Unexpected subject: {subject}.")

1	##########################################################################
2	# NSAp - Copyright (C) CEA, 2025
3	# Distributed under the terms of the CeCILL-B license, as published by
4	# the CEA-CNRS-INRIA. Refer to the LICENSE file or to
5	# http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
6	# for details.
7	##########################################################################
8
9	"""
10	Fetcher to download anatomical data.
11	"""
12
13	import gzip	1✔
14	import json	1✔
15	import os	1✔
16	import shutil	1✔
17	import tarfile	1✔
18	from pathlib import Path	1✔
19
20	import pandas as pd	1✔
21	import requests	1✔
22
23	from ..typing import (	1✔
24	Directory,
25	)
26	from ..utils import (	1✔
27	Bunch,
28	print_info,
29	)
30
31
32	class MultiModalDataset:	1✔
33	"""
34	Multi-modal dataset.
35
36	This `dataset <https://www.nitrc.org/projects/multimodal>`_
37	:footcite:p:`landman2011data` contains Magnetic Resonance (MR) images of
38	21 healthy volunteers: scan-rescan imaging sessions including MPRAGE,
39	FLAIR, DTI, resting state fMRI, B0 and B1 field maps, ...
40
41	Parameters
42	----------
43	datadir : Directory
44	Directory where data will be stored.
45
46	Attributes
47	----------
48	_url : str
49	Internal URL used to fetch data.
50
51	Examples
52	--------
53	>>> from brainprep.config import Config
54	>>> from pathlib import Path
55	>>> from brainprep.datasets import MultiModalDataset
56	>>>
57	>>> datadir = Path("/tmp/brainprep-data")
58	>>> datadir.mkdir(parents=True, exist_ok=True)
59	>>> dataset = MultiModalDataset(datadir)
60	>>> with Config(verbose=False):
61	... data = dataset.fetch(
62	... subject="01",
63	... modality="func",
64	... session="01",
65	... )
66	>>> data
67	Bunch(
68	description: PosixPath('...')
69	anat: PosixPath('...')
70	fmap1: PosixPath('...')
71	fmap2: PosixPath('...')
72	func: PosixPath('...')
73	)
74
75	References
76	----------
77
78	.. footbibliography::
79	"""
80
81	_url: str = "https://www.nitrc.org/frs/downloadlink.php/22{visit}"	1✔
82
83	def __init__(	1✔
84	self,
85	datadir: Directory) -> None:
86	self.datadir = datadir	1✔
87	self.allowed_subjects = [	1✔
88	f"{str(subject).zfill(2)}" for subject in range(1, 22)
89	]
90	self.allowed_modalities = [	1✔
91	"func",
92	"dwi",
93	]
94	self.allowed_sessions = [	1✔
95	"01",
96	"02",
97	]
98	self.meta = pd.read_csv(	1✔
99	Path(__file__).parent.parent / "resources" / "kirby21.tsv",
100	sep="\t",
101	)
102	self.meta["Subject"] = self.meta["Subject"].map(	1✔
103	{
104	val: str(idx + 1).zfill(2)
105	for idx, val in enumerate(pd.unique(self.meta["Subject"]))
106	}
107	)
108	self.meta["Visit"] = self.meta["Visit"].astype(str).apply(	1✔
109	lambda x: x.zfill(2)
110	)
111
112	def fetch(	1✔
113	self,
114	subject: str,
115	modality: str,
116	session: str) -> Bunch:
117	""" Fetch data.
118
119	Parameters
120	----------
121	subject : str
122	Subject identifier: ['01' - '21'].
123	modality : str
124	Modality to be fetched: 'func' or 'dwi'.
125	session : str
126	Session: '01' or '02'.
127
128	Returns
129	-------
130	dataset: Bunch
131	Fetched data path. A 'description' entry is also available.
132
133	Raises
134	------
135	ValueError
136	If data could not be extracted from the fetched archive.
137	"""
138	dataset = Bunch()	1✔
139	self.sanity_check(subject, modality, session)	1✔
140
141	description_file = (	1✔
142	self.datadir /
143	"rawdata" /
144	"dataset_description.json"
145	)
146	if not description_file.is_file():	1✔
147	description_file.parent.mkdir(parents=True, exist_ok=True)	×
148	with open(description_file, "w") as of:	×
149	description = {	×
150	"Name": "Example dataset",
151	"BIDSVersion": "1.0.2",
152	}
153	json.dump(description, of, indent=4)	×
154	dataset["description"] = description_file	1✔
155
156	selection = self.meta[self.meta["Subject"] == subject]	1✔
157	visit = selection.iloc[int(session) - 1]["Visit"]	1✔
158	url = self._url.format(	1✔
159	visit=visit,
160	)
161	archive_file = (	1✔
162	self.datadir /
163	"sourcedata" /
164	f"KKI2009-{visit}.tar.bz2"
165	)
166	if not archive_file.is_file():	1✔
167	print_info(f"downloading: {url}")	1✔
168	archive_file.parent.mkdir(parents=True, exist_ok=True)	1✔
169	response = requests.get(url)	1✔
170	response.raise_for_status()	1✔
171	with open(archive_file, "wb") as of:	1✔
172	of.write(response.content)	1✔
173
174	to_extract = [	1✔
175	("anat", "T1w.nii", "MPRAGE.nii"),
176	("anat", "T1w.json", "MPRAGE.par"),
177	("fmap", "magnitude1.nii", "B08MS.nii"),
178	("fmap", "magnitude1.json", "B08MS.par"),
179	("fmap", "magnitude2.nii", "B09MS.nii"),
180	("fmap", "magnitude2.json", "B09MS.par"),
181	]
182	if modality == "dwi":	1✔
183	to_extract += [	1✔
184	("dwi", "dwi.nii", "DTI.nii"),
185	("dwi", "dwi.json", "DTI.par"),
186	("dwi", "dwi.bvec", "DTI.grad"),
187	("dwi", "dwi.bval", "DTI.b"),
188	]
189	else:
190	to_extract += [	1✔
191	("func", "task-rest_bold.nii", "fMRI.nii"),
192	("func", "task-rest_bold.json", "fMRI.par"),
193	]
194
195	with tarfile.open(archive_file, "r:bz2") as tar:	1✔
196	fmap_counter = 1	1✔
197	for dtype, dest_suffix, src_suffix in to_extract:	1✔
198	destination_file = (	1✔
199	self.datadir /
200	"rawdata" /
201	f"sub-{subject}" /
202	f"ses-{session}" /
203	dtype /
204	f"sub-{subject}_ses-{session}_{dest_suffix}"
205	)
206	if dest_suffix.endswith(".nii"):	1✔
207	if dtype == "fmap":	1✔
208	dtype = f"{dtype}{fmap_counter}"	1✔
209	fmap_counter += 1	1✔
210	dataset[dtype] = destination_file	1✔
211	if destination_file.with_suffix(".nii.gz").is_file():	1✔
212	continue	1✔
213	elif dest_suffix.endswith((".bvec", ".bval")):	1✔
214	dataset[dest_suffix.split(".")[1]] = destination_file	1✔
215	if destination_file.is_file():	1✔
216	continue	1✔
217	destination_file.parent.mkdir(parents=True, exist_ok=True)	1✔
218	file_to_extract = f"KKI2009-{visit}-{src_suffix}"	1✔
219	print_info(f"extracting: {file_to_extract}")	1✔
220	fileobj = tar.extractfile(file_to_extract)	1✔
221	if fileobj is not None:	1✔
222	with open(destination_file, "wb") as of:	1✔
223	of.write(fileobj.read())	1✔
224	else:
225	raise ValueError(	×
226	f"Could not extract '{file_to_extract}' from the "
227	"archive."
228	)
229
230	for key, src_file in dataset.items():	1✔
231	if src_file.suffix != ".nii":	1✔
232	continue	1✔
233	dest_file = src_file.with_suffix(".nii.gz")	1✔
234	dataset[key] = dest_file	1✔
235	if dest_file.is_file():	1✔
236	continue	1✔
237	with (open(src_file, "rb") as f_in,	1✔
238	gzip.open(dest_file, "wb") as f_out):
239	shutil.copyfileobj(f_in, f_out)	1✔
240	os.remove(src_file)	1✔
241
242	return dataset	1✔
243
244	def sanity_check(	1✔
245	self,
246	subject: str,
247	modality: str,
248	session: str) -> None:
249	""" Check that the fetch parameters are correct.
250
251	Parameters
252	----------
253	subject : str
254	the subject identifier. This identifier must lie in ['01' - '30'],
255	['01' - '20'], for cross sectional or longitudinal data
256	respectively.
257	modality : str
258	the modality to be fetched: 'dwi', or 'func'.
259	session : str
260	Session: '01' or '02'.
261
262	Raises
263	------
264	ValueError
265	If the fetch input parameters are not correct.
266	"""
267	if session not in self.allowed_sessions:	1✔
268	raise ValueError(f"Unexpected session: {session}.")	×
269	if modality not in self.allowed_modalities:	1✔
270	raise ValueError(f"Unexpected modality: {modality}.")	×
271	if subject not in self.allowed_subjects:	1✔
272	raise ValueError(f"Unexpected subject: {subject}.")	×

neurospin-deepinsight / brainprep / 19471672015

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous