• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

ISI-MIP / isimip-data / 11802746904

12 Nov 2024 05:41PM UTC coverage: 88.473% (-1.8%) from 90.254%
11802746904

Pull #62

github

web-flow
Merge c59388233 into 7b62566b8
Pull Request #62: Update download interface and refactor front-end

4 of 55 new or added lines in 5 files covered. (7.27%)

13 existing lines in 2 files now uncovered.

2172 of 2455 relevant lines covered (88.47%)

0.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.68
/isimip_data/metadata/models.py
1
from datetime import datetime
1✔
2
from pathlib import Path
1✔
3

4
from django.contrib.postgres.fields import ArrayField
1✔
5
from django.contrib.postgres.search import SearchVectorField
1✔
6
from django.db import models
1✔
7
from django.urls import reverse
1✔
8
from django.utils.functional import cached_property
1✔
9

10
from .constants import RIGHTS
1✔
11
from .managers import DatasetManager, IdentifierManager, ResourceManager
1✔
12
from .utils import get_json_ld_name, get_terms_of_use, merge_identifiers, merge_specifiers, prettify_specifiers
1✔
13

14

15
class Dataset(models.Model):
1✔
16

17
    objects = DatasetManager()
1✔
18

19
    id = models.UUIDField(primary_key=True)
1✔
20
    target = models.ForeignKey('Dataset', on_delete=models.CASCADE, related_name='links')
1✔
21

22
    name = models.TextField()
1✔
23
    path = models.TextField()
1✔
24
    version = models.TextField()
1✔
25
    size = models.BigIntegerField()
1✔
26
    specifiers = models.JSONField()
1✔
27
    rights = models.TextField()
1✔
28
    identifiers = ArrayField(models.TextField())
1✔
29
    public = models.BooleanField()
1✔
30
    restricted = models.BooleanField()
1✔
31
    tree_path = models.TextField()
1✔
32

33
    created = models.DateTimeField()
1✔
34
    updated = models.DateTimeField()
1✔
35
    published = models.DateTimeField()
1✔
36
    archived = models.DateTimeField()
1✔
37

38
    class Meta:
1✔
39
        db_table = 'datasets'
1✔
40
        managed = False
1✔
41
        ordering = ('path', )
1✔
42

43
    def __str__(self):
1✔
44
        return self.path
1✔
45

46
    @cached_property
1✔
47
    def is_link(self):
1✔
48
        return self.target is not None
1✔
49

50
    @cached_property
1✔
51
    def is_global(self):
1✔
52
        return self.specifiers.get('region') == 'global'
1✔
53

54
    @cached_property
1✔
55
    def is_netcdf(self):
1✔
56
        return all(Path(file.path).suffix.startswith('.nc') for file in self.files.all())
1✔
57

58
    @cached_property
1✔
59
    def paths(self):
1✔
60
        return [self.path] + [link.path for link in self.links.all()]
1✔
61

62
    @cached_property
1✔
63
    def pretty_specifiers(self):
1✔
64
        return prettify_specifiers(self.merged_specifiers, self.merged_identifiers)
1✔
65

66
    @cached_property
1✔
67
    def merged_specifiers(self):
1✔
68
        return merge_specifiers(self)
1✔
69

70
    @cached_property
1✔
71
    def merged_identifiers(self):
1✔
72
        return merge_identifiers(self)
1✔
73

74
    @cached_property
1✔
75
    def rights_dict(self):
1✔
76
        return RIGHTS.get(self.rights, {})
1✔
77

78
    @cached_property
1✔
79
    def rights_list(self):
1✔
80
        return [self.rights_dict] if self.rights_dict else []
1✔
81

82
    @cached_property
1✔
83
    def terms_of_use(self):
1✔
84
        return get_terms_of_use()
1✔
85

86
    @cached_property
1✔
87
    def current_resources(self):
1✔
88
        resources = self.resources.order_by('paths', 'doi')
1✔
89
        resource_dois = {resource.doi for resource in resources}
1✔
90
        return [resource for resource in resources if (
1✔
91
            (resource.new_version is None) or (resource.new_version not in resource_dois)
92
        )]
93

94
    @cached_property
1✔
95
    def json_ld(self):
1✔
96
        data = {
1✔
97
            '@context': 'https://schema.org/',
98
            '@type': 'Dataset',
99
            'name': self.path
100
        }
101

102
        resources = self.resources.order_by('-doi')
1✔
103
        if resources:
1✔
104
            data['isPartOf'] = [resource.json_ld for resource in resources]
×
105

106
        return data
1✔
107

108
    def get_absolute_url(self):
1✔
109
        return reverse('dataset', kwargs={'pk': self.pk})
×
110

111

112
class File(models.Model):
1✔
113

114
    id = models.UUIDField(primary_key=True)
1✔
115
    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='files')
1✔
116
    target = models.ForeignKey('File', on_delete=models.CASCADE, related_name='links')
1✔
117

118
    name = models.TextField()
1✔
119
    path = models.TextField()
1✔
120
    version = models.TextField()
1✔
121
    size = models.BigIntegerField()
1✔
122
    checksum = models.TextField()
1✔
123
    checksum_type = models.TextField()
1✔
124
    netcdf_header = models.JSONField()
1✔
125
    specifiers = models.JSONField()
1✔
126
    identifiers = ArrayField(models.TextField())
1✔
127

128
    created = models.DateTimeField()
1✔
129
    updated = models.DateTimeField()
1✔
130

131
    class Meta:
1✔
132
        db_table = 'files'
1✔
133
        managed = False
1✔
134
        ordering = ('path', )
1✔
135

136
    def __str__(self):
1✔
137
        return self.path
1✔
138

139
    @cached_property
1✔
140
    def public(self):
1✔
141
        return self.dataset.public
1✔
142

143
    @cached_property
1✔
144
    def resources(self):
1✔
145
        return self.dataset.resources
×
146

147
    @cached_property
1✔
148
    def json_path(self):
1✔
149
        return str(Path(self.path).with_suffix('.json'))
1✔
150

151
    @cached_property
1✔
152
    def is_link(self):
1✔
153
        return self.target is not None
1✔
154

155
    @cached_property
1✔
156
    def is_global(self):
1✔
157
        return self.specifiers.get('region') == 'global'
×
158

159
    @cached_property
1✔
160
    def is_netcdf(self):
1✔
161
        return Path(self.path).suffix.startswith('.nc')
×
162

163
    @cached_property
1✔
164
    def paths(self):
1✔
165
        return [self.path] + [link.path for link in self.links.all()]
1✔
166

167
    @cached_property
1✔
168
    def pretty_specifiers(self):
1✔
169
        return prettify_specifiers(self.merged_specifiers, self.merged_identifiers)
1✔
170

171
    @cached_property
1✔
172
    def merged_specifiers(self):
1✔
173
        return merge_specifiers(self)
1✔
174

175
    @cached_property
1✔
176
    def merged_identifiers(self):
1✔
177
        return merge_identifiers(self)
1✔
178

179
    @cached_property
1✔
180
    def rights_dict(self):
1✔
181
        return RIGHTS.get(self.dataset.rights, {})
1✔
182

183
    @cached_property
1✔
184
    def rights_list(self):
1✔
185
        return [self.rights_dict] if self.rights_dict else []
1✔
186

187
    @cached_property
1✔
188
    def terms_of_use(self):
1✔
189
        return get_terms_of_use()
1✔
190

191
    @cached_property
1✔
192
    def json_ld(self):
1✔
193
        return {
1✔
194
            '@context': 'https://schema.org/',
195
            '@type': 'Dataset',
196
            'name': self.path,
197
            'isPartOf': self.dataset.json_ld
198
        }
199

200
    def get_absolute_url(self):
1✔
201
        return reverse('file', kwargs={'pk': self.pk})
×
202

203

204
class Resource(models.Model):
1✔
205

206
    objects = ResourceManager()
1✔
207

208
    id = models.UUIDField(primary_key=True)
1✔
209
    doi = models.TextField()
1✔
210
    title = models.TextField()
1✔
211
    version = models.TextField()
1✔
212
    paths = ArrayField(models.TextField())
1✔
213
    datacite = models.JSONField()
1✔
214

215
    created = models.DateTimeField()
1✔
216
    updated = models.DateTimeField()
1✔
217

218
    datasets = models.ManyToManyField(Dataset, related_name='resources')
1✔
219

220
    class Meta:
1✔
221
        db_table = 'resources'
1✔
222
        managed = False
1✔
223
        ordering = ('paths', )
1✔
224

225
    def __str__(self):
1✔
226
        return self.doi
1✔
227

228
    @cached_property
1✔
229
    def major_version(self):
1✔
230
        version = self.datacite.get('version')
1✔
231
        if version:
1✔
232
            return '.'.join(version.split('.')[:2])
1✔
233

234
    @cached_property
1✔
235
    def previous_version(self):
1✔
236
        try:
1✔
237
            related_identifier = next(i for i in self.datacite.get('relatedIdentifiers', [])
1✔
238
                                      if i.get('relationType') == 'IsNewVersionOf')
239
            return related_identifier.get('relatedIdentifier').replace('https://doi.org/', '')
1✔
240
        except StopIteration:
×
241
            return None
×
242

243
    @cached_property
1✔
244
    def new_version(self):
1✔
245
        try:
1✔
246
            related_identifier = next(i for i in self.datacite.get('relatedIdentifiers', [])
1✔
247
                                      if i.get('relationType') == 'IsPreviousVersionOf')
248
            return related_identifier.get('relatedIdentifier').replace('https://doi.org/', '')
×
249
        except (AttributeError, StopIteration):
1✔
250
            return None
1✔
251

252
    @cached_property
1✔
253
    def citation(self):
1✔
254
        if self.is_external:
1✔
UNCOV
255
            return f'{self.title}. {self.doi_url}'
×
256
        else:
257
            return f'{self.creators_str} ({self.publication_year}): ' \
1✔
258
                   f'{self.title_with_version}. {self.publisher}. {self.doi_url}'
259

260
    @cached_property
1✔
261
    def creators_str(self):
1✔
262
        return ', '.join([creator.get('name', '') for creator in self.datacite.get('creators', [])])
1✔
263

264
    @cached_property
1✔
265
    def publication_year(self):
1✔
266
        return self.datacite.get('publicationYear', '')
1✔
267

268
    @cached_property
1✔
269
    def title_with_version(self):
1✔
270
        if not self.version or self.title.endswith(')') or self.title[-1].isdigit():
1✔
UNCOV
271
            return self.title
×
272
        else:
273
            return f'{self.title} (v{self.version})'
1✔
274

275
    @cached_property
1✔
276
    def doi_url(self):
1✔
277
        return f'https://doi.org/{self.doi}'
1✔
278

279
    @cached_property
1✔
280
    def publisher(self):
1✔
281
        return self.datacite.get('publisher', '')
1✔
282

283
    @cached_property
1✔
284
    def contact_persons(self):
1✔
285
        return [
1✔
286
            contributor for contributor in self.datacite.get('contributors', [])
287
            if contributor.get('contributorType') == 'ContactPerson'
288
        ]
289

290
    @cached_property
1✔
291
    def abstract(self):
1✔
292
        for item in self.datacite.get('descriptions', []):
1✔
293
            try:
1✔
294
                if item['descriptionType'] == 'Abstract':
1✔
295
                    return item['description']
1✔
UNCOV
296
            except (TypeError, KeyError):
×
UNCOV
297
                pass
×
298

299
    @cached_property
1✔
300
    def publication_date(self):
1✔
301
        date_string = None
1✔
302
        for item in self.datacite.get('dates', []):
1✔
303
            try:
1✔
304
                if item['dateType'] == 'Issued':
1✔
305
                    date_string = item['date']
1✔
UNCOV
306
            except (TypeError, KeyError):
×
UNCOV
307
                pass
×
308

309
        if date_string is not None:
1✔
310
            try:
1✔
311
                return datetime.strptime(date_string, '%Y-%m-%d').date()
1✔
UNCOV
312
            except ValueError:
×
UNCOV
313
                return datetime.strptime(date_string, '%Y').date()
×
314

315
    @cached_property
1✔
316
    def rights_list(self):
1✔
317
        rights_list = []
1✔
318
        for rights in self.datacite.get('rightsList', []):
1✔
319
            rights_uri = rights.get('rightsURI')
1✔
320
            if rights_uri:
1✔
321
                rights = filter(lambda item: item.get('rights_uri') == rights_uri, RIGHTS.values())
1✔
322
                try:
1✔
323
                    rights_list.append(next(rights))
1✔
UNCOV
324
                except StopIteration:
×
UNCOV
325
                    pass
×
326
        return rights_list
1✔
327

328
    @cached_property
1✔
329
    def terms_of_use(self):
1✔
UNCOV
330
        return get_terms_of_use()
×
331

332
    @cached_property
1✔
333
    def is_external(self):
1✔
334
        return self.datacite is None
1✔
335

336
    def get_absolute_url(self):
1✔
UNCOV
337
        return reverse('resource', kwargs={'doi': self.doi})
×
338

339
    @cached_property
1✔
340
    def json_ld(self):
1✔
341
        data = {
1✔
342
            '@context': 'https://schema.org/',
343
            '@type': 'Dataset',
344
            'name': self.title,
345
            'identifier': self.doi_url,
346
        }
347

348
        if self.datacite:
1✔
349
            data.update({
1✔
350
                'description': self.abstract,
351
                'version': self.datacite.get('version'),
352
                'keywords': [
353
                    subject['subject']
354
                    for subject in self.datacite.get('subjects', [])
355
                    if subject.get('subject')
356
                ],
357
                'publisher': {
358
                    '@type': 'Organization',
359
                    'name': self.datacite.get('publisher')
360
                },
361
                'datePublished': self.publication_date.isoformat(),
362
                'license': [
363
                    {
364
                        '@type': 'CreativeWork',
365
                        'name': rights.get('rights'),
366
                        'url': rights.get('rights_uri')
367
                    } for rights in self.rights_list
368
                ],
369
                'isAccessibleForFree': True,
370
                'creator': [
371
                    get_json_ld_name(creator)
372
                    for creator in self.datacite.get('creators', [])
373
                ],
374
                'contributor': [
375
                    get_json_ld_name(contributor)
376
                    for contributor in self.datacite.get('contributors', [])
377
                ]
378
            })
379

380
        return data
1✔
381

382

383
class Tree(models.Model):
1✔
384

385
    id = models.UUIDField(primary_key=True)
1✔
386
    tree_dict = models.JSONField()
1✔
387

388
    created = models.DateTimeField()
1✔
389
    updated = models.DateTimeField()
1✔
390

391
    class Meta:
1✔
392
        db_table = 'trees'
1✔
393
        managed = False
1✔
394
        ordering = ('id', )
1✔
395

396
    def __str__(self):
1✔
397
        return str(self.id)
1✔
398

399

400
class Identifier(models.Model):
1✔
401

402
    objects = IdentifierManager()
1✔
403

404
    identifier = models.TextField(primary_key=True)
1✔
405
    specifiers = ArrayField(models.TextField())
1✔
406

407
    class Meta:
1✔
408
        db_table = 'identifiers'
1✔
409
        managed = False
1✔
410
        ordering = ('identifier', )
1✔
411

412
    def __str__(self):
1✔
413
        return self.identifier
1✔
414

415

416
class Specifier(models.Model):
1✔
417

418
    specifier = models.TextField(primary_key=True)
1✔
419

420
    class Meta:
1✔
421
        db_table = 'specifiers'
1✔
422
        managed = False
1✔
423
        ordering = ('specifier', )
1✔
424

425
    def __str__(self):
1✔
426
        return self.specifier
1✔
427

428

429
class Search(models.Model):
1✔
430

431
    dataset = models.ForeignKey('Dataset', on_delete=models.CASCADE, related_name='search')
1✔
432
    vector = SearchVectorField()
1✔
433

434
    class Meta:
1✔
435
        db_table = 'search'
1✔
436
        managed = False
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc