Build Type

github-actions

Committed by GitHub

Commit Message

<a href="https://github.com/open-contracting/kingfisher-collect/commit/<a class=hub.com/open-contracting/kingfisher-collect/commit/<a class="double-link" href="https://git"><a class=hub.com/open-contracting/kingfisher-collect/commit/<a class="double-link" href="https://git"><a class=hub.com/open-contracting/kingfisher-collect/commit/<a class="double-link" href="https://git"><a class=hub.com/open-contracting/kingfisher-collect/commit/17d121ddcdbbfb32a551c3e0c4a41eeaa6dc0555">17d121ddc">&lt;a href=&quot;https://github.com/open-contracting/kingfisher-collect/commit/</a><a class="double-link" href="https://github.com/open-contracting/kingfisher-collect/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/open-contracting/kingfisher-collect/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/open-contracting/kingfisher-collect/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/open-contracting/kingfisher-collect/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/open-contracting/kingfisher-collect/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/open-contracting/kingfisher-collect/commit/17d121ddcdbbfb32a551c3e0c4a41eeaa6dc0555">17d121ddc</a><a href="https://github.com/open-contracting/kingfisher-collect/commit/17d121ddcdbbfb32a551c3e0c4a41eeaa6dc0555">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/open-contracting/kingfisher-collect/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/open-contracting/kingfisher-collect/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/open-contracting/kingfisher-collect/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/open-contracting/kingfisher-collect/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/open-contracting/kingfisher-collect/commit/17d121ddcdbbfb32a551c3e0c4a41eeaa6dc0555&quot;&gt;17d121ddc&lt;/a&gt;&lt;a href=&quot;https://github.com/open-contracting/kingfisher-collect/commit/17d121ddcdbbfb32a551c3e0c4a41eeaa6dc0555&quot;&g... (continued)

Pull Request Pull Request #386: Add sentry logger as extension

Run Details

10 of 10 new or added lines in 2 files covered. (100.0%)

297 of 374 relevant lines covered (79.41%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.65

/kingfisher_scrapy/extensions.py

# https://docs.scrapy.org/en/latest/topics/extensions.html#writing-your-own-extension

import json
import os

import sentry_sdk
from scrapy import signals
from scrapy.exceptions import NotConfigured

from kingfisher_scrapy.items import File, FileError, FileItem
from kingfisher_scrapy.kingfisher_process import Client


class KingfisherFilesStore:
    def __init__(self, directory):
        self.directory = directory

    @classmethod
    def from_crawler(cls, crawler):
        directory = crawler.settings['FILES_STORE']
        extension = cls(directory)
        crawler.signals.connect(extension.item_scraped, signal=signals.item_scraped)
        return extension

    def item_scraped(self, item, spider):
        """
        If the item is a file, writes its data to the filename in the crawl's directory.

        Writes a ``<filename>.fileinfo`` metadata file in the crawl's directory, and returns a dict with the metadata.
        """
        if not isinstance(item, File):
            return

        # The crawl's relative directory, in the format `<spider_name>[_sample]/<YYMMDD_HHMMSS>`.
        name = spider.name
        if spider.sample:
            name += '_sample'
        path = os.path.join(name, spider.get_start_time('%Y%m%d_%H%M%S'), item['file_name'])

        self._write_file(path, item['data'], spider)
        metadata = {
            'url': item['url'],
            'data_type': item['data_type'],
            'encoding': item['encoding'],
        }
        self._write_file(path + '.fileinfo', metadata, spider)

        item['path'] = path
        item['files_store'] = self.directory

    def _write_file(self, path, data, spider):
        path = os.path.join(self.directory, path)
        os.makedirs(os.path.dirname(path), exist_ok=True)

        if isinstance(data, bytes):
            mode = 'wb'
        else:
            mode = 'w'

        with open(path, mode) as f:
            if isinstance(data, (bytes, str)):
                f.write(data)
            else:
                json.dump(data, f)


class KingfisherProcessAPI:
    """
    If the ``KINGFISHER_API_URI`` and ``KINGFISHER_API_KEY`` environment variables or configuration settings are set,
    then messages are sent to a Kingfisher Process API for the ``item_scraped`` and ``spider_closed`` signals.
    """
    def __init__(self, url, key, directory=None):
        """
        Initializes a Kingfisher Process API client.
        """
        self.client = Client(url, key)
        self.directory = directory

    @classmethod
    def from_crawler(cls, crawler):
        url = crawler.settings['KINGFISHER_API_URI']
        key = crawler.settings['KINGFISHER_API_KEY']
        directory = crawler.settings['KINGFISHER_API_LOCAL_DIRECTORY']

        if not url or not key:
            raise NotConfigured('KINGFISHER_API_URI and/or KINGFISHER_API_KEY is not set.')

        extension = cls(url, key, directory=directory)
        crawler.signals.connect(extension.item_scraped, signal=signals.item_scraped)
        crawler.signals.connect(extension.spider_closed, signal=signals.spider_closed)

        return extension

    def spider_closed(self, spider, reason):
        """
        Sends an API request to end the collection's store step.
        """
        # https://docs.scrapy.org/en/latest/topics/signals.html#spider-closed
        if reason != 'finished':
            return

        response = self.client.end_collection_store({
            'collection_source': spider.name,
            'collection_data_version': spider.get_start_time('%Y-%m-%d %H:%M:%S'),
            'collection_sample': spider.sample,
        })

        if not response.ok:
            spider.logger.warning(
                'Failed to post End Collection Store. API status code: {}'.format(response.status_code))

    def item_scraped(self, item, spider):
        """
        Sends an API request to store the file, file item or file error in Kingfisher Process.
        """
        if not item.get('post_to_api', True):
            return

        data = {
            'collection_source': spider.name,
            'collection_data_version': spider.get_start_time('%Y-%m-%d %H:%M:%S'),
            'collection_sample': spider.sample,
            'file_name': item['file_name'],
            'url': item['url'],
        }

        if isinstance(item, FileError):
            data['errors'] = json.dumps(item['errors'])

            self._request(item, spider, 'create_file_error', data, name='File Errors API')
        else:
            data['data_type'] = item['data_type']
            data['encoding'] = item.get('encoding', 'utf-8')
            if spider.note:
                data['collection_note'] = spider.note

            if isinstance(item, FileItem):
                data['number'] = item['number']
                data['data'] = item['data']

                self._request(item, spider, 'create_file_item', data)

            # File
            else:
                if self.directory:
                    path = item['path']
                    data['local_file_name'] = os.path.join(self.directory, path)
                    files = {}
                else:
                    path = os.path.join(item['files_store'], item['path'])
                    f = open(path, 'rb')
                    files = {'file': (item['file_name'], f, 'application/json')}

                self._request(item, spider, 'create_file', data, files)

    def _request(self, item, spider, method, *args, name='API'):
        response = getattr(self.client, method)(*args)
        if not response.ok:
            spider.logger.warning(
                'Failed to post [{}]. {} status code: {}'.format(item['url'], name, response.status_code))


# https://stackoverflow.com/questions/25262765/handle-all-exception-in-scrapy-with-sentry
class SentryLogging:
    """
    Sends exceptions and log records to Sentry. Log records with a level of ``ERROR`` or higher are captured as events.

    https://docs.sentry.io/platforms/python/logging/
    """

    @classmethod
    def from_crawler(cls, crawler):
        sentry_dsn = crawler.settings.get('SENTRY_DSN', None)
        if sentry_dsn is None:
            raise NotConfigured
        extension = cls()
        sentry_sdk.init(sentry_dsn)
        return extension

1	# https://docs.scrapy.org/en/latest/topics/extensions.html#writing-your-own-extension
2
3	import json	1✔
4	import os	1✔
5
6	import sentry_sdk	1✔
7	from scrapy import signals	1✔
8	from scrapy.exceptions import NotConfigured	1✔
9
10	from kingfisher_scrapy.items import File, FileError, FileItem	1✔
11	from kingfisher_scrapy.kingfisher_process import Client	1✔
12
13
14	class KingfisherFilesStore:	1✔
15	def __init__(self, directory):	1✔
16	self.directory = directory	1✔
17
18	@classmethod	1✔
19	def from_crawler(cls, crawler):
20	directory = crawler.settings['FILES_STORE']	1✔
21	extension = cls(directory)	1✔
22	crawler.signals.connect(extension.item_scraped, signal=signals.item_scraped)	1✔
23	return extension	1✔
24
25	def item_scraped(self, item, spider):	1✔
26	"""
27	If the item is a file, writes its data to the filename in the crawl's directory.
28
29	Writes a ``<filename>.fileinfo`` metadata file in the crawl's directory, and returns a dict with the metadata.
30	"""
31	if not isinstance(item, File):	1✔
32	return	×
33
34	# The crawl's relative directory, in the format `<spider_name>[_sample]/<YYMMDD_HHMMSS>`.
35	name = spider.name	1✔
36	if spider.sample:	1✔
37	name += '_sample'	1✔
38	path = os.path.join(name, spider.get_start_time('%Y%m%d_%H%M%S'), item['file_name'])	1✔
39
40	self._write_file(path, item['data'], spider)	1✔
41	metadata = {	1✔
42	'url': item['url'],
43	'data_type': item['data_type'],
44	'encoding': item['encoding'],
45	}
46	self._write_file(path + '.fileinfo', metadata, spider)	1✔
47
48	item['path'] = path	1✔
49	item['files_store'] = self.directory	1✔
50
51	def _write_file(self, path, data, spider):	1✔
52	path = os.path.join(self.directory, path)	1✔
53	os.makedirs(os.path.dirname(path), exist_ok=True)	1✔
54
55	if isinstance(data, bytes):	1✔
56	mode = 'wb'	1✔
57	else:
58	mode = 'w'	1✔
59
60	with open(path, mode) as f:	1✔
61	if isinstance(data, (bytes, str)):	1✔
62	f.write(data)	1✔
63	else:
64	json.dump(data, f)	1✔
65
66
67	class KingfisherProcessAPI:	1✔
68	"""
69	If the ``KINGFISHER_API_URI`` and ``KINGFISHER_API_KEY`` environment variables or configuration settings are set,
70	then messages are sent to a Kingfisher Process API for the ``item_scraped`` and ``spider_closed`` signals.
71	"""
72	def __init__(self, url, key, directory=None):	1✔
73	"""
74	Initializes a Kingfisher Process API client.
75	"""
76	self.client = Client(url, key)	1✔
77	self.directory = directory	1✔
78
79	@classmethod	1✔
80	def from_crawler(cls, crawler):
81	url = crawler.settings['KINGFISHER_API_URI']	1✔
82	key = crawler.settings['KINGFISHER_API_KEY']	1✔
83	directory = crawler.settings['KINGFISHER_API_LOCAL_DIRECTORY']	1✔
84
85	if not url or not key:	1✔
86	raise NotConfigured('KINGFISHER_API_URI and/or KINGFISHER_API_KEY is not set.')	1✔
87
88	extension = cls(url, key, directory=directory)	1✔
89	crawler.signals.connect(extension.item_scraped, signal=signals.item_scraped)	1✔
90	crawler.signals.connect(extension.spider_closed, signal=signals.spider_closed)	1✔
91
92	return extension	1✔
93
94	def spider_closed(self, spider, reason):	1✔
95	"""
96	Sends an API request to end the collection's store step.
97	"""
98	# https://docs.scrapy.org/en/latest/topics/signals.html#spider-closed
99	if reason != 'finished':	1✔
100	return	1✔
101
102	response = self.client.end_collection_store({	1✔
103	'collection_source': spider.name,
104	'collection_data_version': spider.get_start_time('%Y-%m-%d %H:%M:%S'),
105	'collection_sample': spider.sample,
106	})
107
108	if not response.ok:	1✔
109	spider.logger.warning(	1✔
110	'Failed to post End Collection Store. API status code: {}'.format(response.status_code))
111
112	def item_scraped(self, item, spider):	1✔
113	"""
114	Sends an API request to store the file, file item or file error in Kingfisher Process.
115	"""
116	if not item.get('post_to_api', True):	1✔
117	return	1✔
118
119	data = {	1✔
120	'collection_source': spider.name,
121	'collection_data_version': spider.get_start_time('%Y-%m-%d %H:%M:%S'),
122	'collection_sample': spider.sample,
123	'file_name': item['file_name'],
124	'url': item['url'],
125	}
126
127	if isinstance(item, FileError):	1✔
128	data['errors'] = json.dumps(item['errors'])	1✔
129
130	self._request(item, spider, 'create_file_error', data, name='File Errors API')	1✔
131	else:
132	data['data_type'] = item['data_type']	1✔
133	data['encoding'] = item.get('encoding', 'utf-8')	1✔
134	if spider.note:	1✔
135	data['collection_note'] = spider.note	1✔
136
137	if isinstance(item, FileItem):	1✔
138	data['number'] = item['number']	1✔
139	data['data'] = item['data']	1✔
140
141	self._request(item, spider, 'create_file_item', data)	1✔
142
143	# File
144	else:
145	if self.directory:	1✔
146	path = item['path']	1✔
147	data['local_file_name'] = os.path.join(self.directory, path)	1✔
148	files = {}	1✔
149	else:
150	path = os.path.join(item['files_store'], item['path'])	1✔
151	f = open(path, 'rb')	1✔
152	files = {'file': (item['file_name'], f, 'application/json')}	1✔
153
154	self._request(item, spider, 'create_file', data, files)	1✔
155
156	def _request(self, item, spider, method, *args, name='API'):	1✔
157	response = getattr(self.client, method)(*args)	1✔
158	if not response.ok:	1✔
159	spider.logger.warning(	1✔
160	'Failed to post [{}]. {} status code: {}'.format(item['url'], name, response.status_code))
161
162
163	# https://stackoverflow.com/questions/25262765/handle-all-exception-in-scrapy-with-sentry
164	class SentryLogging:	1✔
165	"""
166	Sends exceptions and log records to Sentry. Log records with a level of ``ERROR`` or higher are captured as events.
167
168	https://docs.sentry.io/platforms/python/logging/
169	"""
170
171	@classmethod	1✔
172	def from_crawler(cls, crawler):
173	sentry_dsn = crawler.settings.get('SENTRY_DSN', None)	1✔
174	if sentry_dsn is None:	1✔
175	raise NotConfigured	1✔
176	extension = cls()	×
177	sentry_sdk.init(sentry_dsn)	×
178	return extension	×

open-contracting / kingfisher-collect / 17d121ddcdbbfb32a551c3e0c4a41eeaa6dc0555-PR-386

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous