1138

Build # 1138

Build Type

push

travis-ci

Committed by Alexis Wolfsdorf

Commit Message

<a href="https://github.com/datosgobar/pydatajson/commit/<a class=hub.com/datosgobar/pydatajson/commit/<a class="double-link" href="https://git"><a class=hub.com/datosgobar/pydatajson/commit/<a class="double-link" href="https://git"><a class=hub.com/datosgobar/pydatajson/commit/6efc10356668cf561b69dc39fcbe6f19d1b44e19">6efc10356">&lt;a href=&quot;https://github.com/datosgobar/pydatajson/commit/</a><a class="double-link" href="https://github.com/datosgobar/pydatajson/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datosgobar/pydatajson/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datosgobar/pydatajson/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datosgobar/pydatajson/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datosgobar/pydatajson/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/datosgobar/pydatajson/commit/6efc10356668cf561b69dc39fcbe6f19d1b44e19">6efc10356</a><a href="https://github.com/datosgobar/pydatajson/commit/6efc10356668cf561b69dc39fcbe6f19d1b44e19">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/datosgobar/pydatajson/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/datosgobar/pydatajson/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/datosgobar/pydatajson/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/datosgobar/pydatajson/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/datosgobar/pydatajson/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/datosgobar/pydatajson/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/datosgobar/pydatajson/commit/6efc10356668cf561b69dc39fcbe6f19d1b44e19&quot;&gt;6efc10356&lt;/a&gt;&lt;a href=&quot;https://gith... (continued)

Run Details

14 of 14 new or added lines in 1 file covered. (100.0%)

2314 of 2787 relevant lines covered (83.03%)

1.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.86

/pydatajson/ckan_utils.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals

import json
import re
import logging
from datetime import time

from dateutil import parser, tz
from .helpers import title_to_name
from . import custom_exceptions as ce

logger = logging.getLogger('pydatajson')


def append_attribute_to_extra(package, dataset, attribute, serialize=False):
    value = dataset.get(attribute)
    if value:
        if serialize:
            value = json.dumps(value)
        package['extras'].append({'key': attribute, 'value': value})


def map_dataset_to_package(catalog, dataset, owner_org, catalog_id=None,
                           demote_superThemes=True, demote_themes=True):
    package = dict()
    package['extras'] = []

    # Obligatorios
    package['id'] = catalog_id + '_' + \
        dataset['identifier'] if catalog_id else dataset['identifier']
    package['name'] = title_to_name(
        catalog_id + '-' +
        dataset['title'] if catalog_id else dataset['title'],
        max_len=100, use_complete_words=True
    )
    package['title'] = dataset['title']
    package['private'] = False
    package['notes'] = dataset['description']
    package['author'] = dataset['publisher']['name']
    package['owner_org'] = owner_org

    append_attribute_to_extra(package, dataset, 'issued')
    append_attribute_to_extra(package, dataset, 'accrualPeriodicity')

    distributions = dataset['distribution']
    package['resources'] = map_distributions_to_resources(
        distributions, catalog_id)

    super_themes = dataset['superTheme']
    append_attribute_to_extra(package, dataset, 'superTheme', serialize=True)
    if demote_superThemes:
        package['groups'] = [
            {'name': title_to_name(super_theme, decode=False)}
            for super_theme in super_themes
        ]

    # Recomendados y opcionales
    package['url'] = dataset.get('landingPage')
    package['author_email'] = dataset['publisher'].get('mbox')
    append_attribute_to_extra(package, dataset, 'modified')
    append_attribute_to_extra(package, dataset, 'temporal')
    append_attribute_to_extra(package, dataset, 'source')
    append_attribute_to_extra(package, dataset, 'language', serialize=True)

    spatial = dataset.get('spatial')
    if spatial:
        serializable = isinstance(spatial, list)
        append_attribute_to_extra(package, dataset, 'spatial', serializable)

    contact_point = dataset.get('contactPoint')
    if contact_point:
        package['maintainer'] = contact_point.get('fn')
        package['maintainer_email'] = contact_point.get('hasEmail')

    keywords = dataset.get('keyword')
    if keywords:
        package['tags'] = [{'name': keyword} for keyword in keywords]

    # Move themes to keywords
    themes = dataset.get('theme', [])
    if themes and demote_themes:
        package['tags'] = package.get('tags', [])
        for theme in themes:
            # si falla continúa sin agregar ese theme a los tags del dataset
            try:
                label = _get_theme_label(catalog, theme)
                package['tags'].append({'name': label})
            except Exception:
                logger.exception('Theme no presente en catálogo.')
                continue
    else:
        package.setdefault('groups', [])
        for theme in themes:
            theme_dict = catalog.get_theme(identifier=theme) or\
                         catalog.get_theme(label=theme)
            if theme_dict:
                package['groups'].append(map_theme_to_group(theme_dict))
    return package


def _get_theme_label(catalog, theme):
    """Intenta conseguir el theme por id o por label."""
    try:
        label = catalog.get_theme(identifier=theme)['label']
    except BaseException:
        try:
            label = catalog.get_theme(label=theme)['label']
        except BaseException:
            raise ce.ThemeNonExistentError(theme)

    label = re.sub(r'[^\wá-úÁ-ÚñÑ .-]+', '',
                   label, flags=re.UNICODE)
    return label


def convert_iso_string_to_utc(date_string):
    date_time = parser.parse(date_string)
    if date_time.time() == time(0):
        return date_string

    if date_time.tzinfo is not None:
        utc_date_time = date_time.astimezone(tz.tzutc())
    else:
        utc_date_time = date_time
    utc_date_time = utc_date_time.replace(tzinfo=None)
    return utc_date_time.isoformat()


def map_distributions_to_resources(distributions, catalog_id=None):
    resources = []
    for distribution in distributions:
        resource = dict()
#       Obligatorios
        resource['id'] = catalog_id + '_' + \
            distribution['identifier'] if catalog_id else distribution[
                'identifier']
        resource['name'] = distribution['title']
        resource['url'] = distribution['downloadURL']
        resource['created'] = convert_iso_string_to_utc(distribution['issued'])
#       Recomendados y opcionales
        resource['description'] = distribution.get('description')
        resource['format'] = distribution.get('format')
        last_modified = distribution.get('modified')
        if last_modified:
            resource['last_modified'] = convert_iso_string_to_utc(
                last_modified)
        resource['mimetype'] = distribution.get('mediaType')
        resource['size'] = distribution.get('byteSize')
        resource['accessURL'] = distribution.get('accessURL')
        resource['resource_type'] = distribution.get('type')
        fileName = distribution.get('fileName')
        if fileName:
            resource['fileName'] = fileName
        dist_fields = distribution.get('field')
        if dist_fields:
            resource['attributesDescription'] = json.dumps(dist_fields)
        resources.append(resource)

    return resources


def map_theme_to_group(theme):

    return {
        "name": title_to_name(theme.get('id') or theme['label']),
        "title": theme.get('label'),
        "description": theme.get('description'),
    }

1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	from __future__ import print_function, unicode_literals	2✔
4
5	import json	2✔
6	import re	2✔
7	import logging	2✔
8	from datetime import time	2✔
9
10	from dateutil import parser, tz	2✔
11	from .helpers import title_to_name	2✔
12	from . import custom_exceptions as ce	2✔
13
14	logger = logging.getLogger('pydatajson')	2✔
15
16
17	def append_attribute_to_extra(package, dataset, attribute, serialize=False):	2✔
18	value = dataset.get(attribute)	2✔
19	if value:	2✔
20	if serialize:	2✔
21	value = json.dumps(value)	2✔
22	package['extras'].append({'key': attribute, 'value': value})	2✔
23
24
25	def map_dataset_to_package(catalog, dataset, owner_org, catalog_id=None,	2✔
26	demote_superThemes=True, demote_themes=True):
27	package = dict()	2✔
28	package['extras'] = []	2✔
29
30	# Obligatorios
31	package['id'] = catalog_id + '_' + \	2✔
32	dataset['identifier'] if catalog_id else dataset['identifier']
33	package['name'] = title_to_name(	2✔
34	catalog_id + '-' +
35	dataset['title'] if catalog_id else dataset['title'],
36	max_len=100, use_complete_words=True
37	)
38	package['title'] = dataset['title']	2✔
39	package['private'] = False	2✔
40	package['notes'] = dataset['description']	2✔
41	package['author'] = dataset['publisher']['name']	2✔
42	package['owner_org'] = owner_org	2✔
43
44	append_attribute_to_extra(package, dataset, 'issued')	2✔
45	append_attribute_to_extra(package, dataset, 'accrualPeriodicity')	2✔
46
47	distributions = dataset['distribution']	2✔
48	package['resources'] = map_distributions_to_resources(	2✔
49	distributions, catalog_id)
50
51	super_themes = dataset['superTheme']	2✔
52	append_attribute_to_extra(package, dataset, 'superTheme', serialize=True)	2✔
53	if demote_superThemes:	2✔
54	package['groups'] = [	2✔
55	{'name': title_to_name(super_theme, decode=False)}
56	for super_theme in super_themes
57	]
58
59	# Recomendados y opcionales
60	package['url'] = dataset.get('landingPage')	2✔
61	package['author_email'] = dataset['publisher'].get('mbox')	2✔
62	append_attribute_to_extra(package, dataset, 'modified')	2✔
63	append_attribute_to_extra(package, dataset, 'temporal')	2✔
64	append_attribute_to_extra(package, dataset, 'source')	2✔
65	append_attribute_to_extra(package, dataset, 'language', serialize=True)	2✔
66
67	spatial = dataset.get('spatial')	2✔
68	if spatial:	2✔
69	serializable = isinstance(spatial, list)	2✔
70	append_attribute_to_extra(package, dataset, 'spatial', serializable)	2✔
71
72	contact_point = dataset.get('contactPoint')	2✔
73	if contact_point:	2✔
74	package['maintainer'] = contact_point.get('fn')	2✔
75	package['maintainer_email'] = contact_point.get('hasEmail')	2✔
76
77	keywords = dataset.get('keyword')	2✔
78	if keywords:	2✔
79	package['tags'] = [{'name': keyword} for keyword in keywords]	2✔
80
81	# Move themes to keywords
82	themes = dataset.get('theme', [])	2✔
83	if themes and demote_themes:	2✔
84	package['tags'] = package.get('tags', [])	2✔
85	for theme in themes:	2✔
86	# si falla continúa sin agregar ese theme a los tags del dataset
87	try:	2✔
88	label = _get_theme_label(catalog, theme)	2✔
89	package['tags'].append({'name': label})	2✔
90	except Exception:	×
91	logger.exception('Theme no presente en catálogo.')	×
92	continue	×
93	else:
94	package.setdefault('groups', [])	2✔
95	for theme in themes:	2✔
96	theme_dict = catalog.get_theme(identifier=theme) or\	2✔
97	catalog.get_theme(label=theme)
98	if theme_dict:	2✔
99	package['groups'].append(map_theme_to_group(theme_dict))	2✔
100	return package	2✔
101
102
103	def _get_theme_label(catalog, theme):	2✔
104	"""Intenta conseguir el theme por id o por label."""
105	try:	2✔
106	label = catalog.get_theme(identifier=theme)['label']	2✔
107	except BaseException:	×
108	try:	×
109	label = catalog.get_theme(label=theme)['label']	×
110	except BaseException:	×
111	raise ce.ThemeNonExistentError(theme)	×
112
113	label = re.sub(r'[^\wá-úÁ-ÚñÑ .-]+', '',	2✔
114	label, flags=re.UNICODE)
115	return label	2✔
116
117
118	def convert_iso_string_to_utc(date_string):	2✔
119	date_time = parser.parse(date_string)	2✔
120	if date_time.time() == time(0):	2✔
121	return date_string	2✔
122
123	if date_time.tzinfo is not None:	2✔
124	utc_date_time = date_time.astimezone(tz.tzutc())	2✔
125	else:
126	utc_date_time = date_time	2✔
127	utc_date_time = utc_date_time.replace(tzinfo=None)	2✔
128	return utc_date_time.isoformat()	2✔
129
130
131	def map_distributions_to_resources(distributions, catalog_id=None):	2✔
132	resources = []	2✔
133	for distribution in distributions:	2✔
134	resource = dict()	2✔
135	# Obligatorios
136	resource['id'] = catalog_id + '_' + \	2✔
137	distribution['identifier'] if catalog_id else distribution[
138	'identifier']
139	resource['name'] = distribution['title']	2✔
140	resource['url'] = distribution['downloadURL']	2✔
141	resource['created'] = convert_iso_string_to_utc(distribution['issued'])	2✔
142	# Recomendados y opcionales
143	resource['description'] = distribution.get('description')	2✔
144	resource['format'] = distribution.get('format')	2✔
145	last_modified = distribution.get('modified')	2✔
146	if last_modified:	2✔
147	resource['last_modified'] = convert_iso_string_to_utc(	2✔
148	last_modified)
149	resource['mimetype'] = distribution.get('mediaType')	2✔
150	resource['size'] = distribution.get('byteSize')	2✔
151	resource['accessURL'] = distribution.get('accessURL')	2✔
152	resource['resource_type'] = distribution.get('type')	2✔
153	fileName = distribution.get('fileName')	2✔
154	if fileName:	2✔
155	resource['fileName'] = fileName	2✔
156	dist_fields = distribution.get('field')	2✔
157	if dist_fields:	2✔
158	resource['attributesDescription'] = json.dumps(dist_fields)	2✔
159	resources.append(resource)	2✔
160
161	return resources	2✔
162
163
164	def map_theme_to_group(theme):	2✔
165
166	return {	2✔
167	"name": title_to_name(theme.get('id') or theme['label']),
168	"title": theme.get('label'),
169	"description": theme.get('description'),
170	}

datosgobar / pydatajson / 1138

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous