2618

pending completion

Build # 2618

Build Type

Pull #779

travis-ci-com

Committed by

web-flow

Commit Message

<a href="https://github.com/CenterForOpenScience/SHARE/commit/<a class=hub.com/CenterForOpenScience/SHARE/commit/<a class="double-link" href="https://git"><a class=hub.com/CenterForOpenScience/SHARE/commit/<a class="double-link" href="https://git"><a class=hub.com/CenterForOpenScience/SHARE/commit/7c3c697fc4b77ca0a52cd2fbcf1a1d4e840e698e">7c3c697fc<a href="https://github.com/CenterForOpenScience/SHARE/commit/7c3c697fc4b77ca0a52cd2fbcf1a1d4e840e698e">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/CenterForOpenScience/SHARE/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/CenterForOpenScience/SHARE/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/CenterForOpenScience/SHARE/commit/7c3c697fc4b77ca0a52cd2fbcf1a1d4e840e698e&quot;&gt;7c3c697fc&lt;/a&gt;&lt;a href=&quot;https://github.com/CenterForOpenScience/SHARE/commit/7c3c697fc4b77ca0a52cd2fbcf1a1d4e840e698e&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/CenterForOpenScience/SHARE/commit/7c3c697fc4b77ca0a52cd2fbcf1a1d4e&lt;/a&gt;840e698e&quot;>&quot;&gt;Merge &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/CenterForOpenScience/SHARE/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/CenterForOpenScience/SHARE/commit/<a class="double-link" href="https://github.com/CenterForOpenScience/SHARE/commit/ce3cca23da19c8cdc6bf4559ee518490e87e315a">ce3cca23d</a><a href="https://github.com/CenterForOpenScience/SHARE/commit/7c3c697fc4b77ca0a52cd2fbcf1a1d4e840e698e">&amp;quot;&amp;gt;ce3cca23d&amp;lt;/a&amp;gt;&amp;quot;&amp;gt;ce3cca23d&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/CenterForOpenScience/SHARE/commit/7c3c697fc4b77ca0a52cd2fbcf1a1d4e840e6&lt;/a&gt;98e&quot;&gt; into &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/CenterForOpenScience/SHARE/commit/<a class="double-link" href="https://github.com/CenterForOpenScience/SHARE/commit/9c776fb4d">9c776fb4d">9c776fb4d</a>

Pull Request Pull Request #779: [wip][ENG-1987] indexer shenanigans

Run Details

599 of 599 new or added lines in 30 files covered. (100.0%)

10199 of 12857 relevant lines covered (79.33%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

53.85

/share/search/index_setup/postrend_backcompat.py

import json

from share.models.core import FormattedMetadataRecord
from share.search.index_setup.base import IndexSetup
from share.search.messages import MessageType


class PostRendBackcompatIndexSetup(IndexSetup):
    @property
    def supported_message_types(self):
        return {MessageType.INDEX_SUID}

    @property
    def index_settings(self):
        from share.search.fetchers import CreativeWorkFetcher
        return {
            'analysis': {
                'filter': {
                    'autocomplete_filter': {
                        'type': 'edge_ngram',
                        'min_gram': 1,
                        'max_gram': 20
                    }
                },
                'analyzer': {
                    'default': {
                        # same as 'standard' analyzer, plus html_strip
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'filter': ['lowercase', 'stop'],
                        'char_filter': ['html_strip']
                    },
                    'autocomplete': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'filter': [
                            'lowercase',
                            'autocomplete_filter'
                        ]
                    },
                    'subject_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'subject_tokenizer',
                        'filter': [
                            'lowercase',
                        ]
                    },
                    'subject_search_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'keyword',
                        'filter': [
                            'lowercase',
                        ]
                    },
                },
                'tokenizer': {
                    'subject_tokenizer': {
                        'type': 'path_hierarchy',
                        'delimiter': CreativeWorkFetcher.SUBJECT_DELIMITER,
                    }
                }
            }
        }

    @property
    def index_mappings(self):
        autocomplete_field = {
            'autocomplete': {
                'type': 'string',
                'analyzer': 'autocomplete',
                'search_analyzer': 'standard',
                'include_in_all': False
            }
        }

        exact_field = {
            'exact': {
                'type': 'keyword',
                # From Elasticsearch documentation:
                # The value for ignore_above is the character count, but Lucene counts bytes.
                # If you use UTF-8 text with many non-ASCII characters, you may want to set the limit to 32766 / 3 = 10922 since UTF-8 characters may occupy at most 3 bytes
                'ignore_above': 10922
            }
        }

        return {
            'creativeworks': {
                'dynamic': 'strict',
                'properties': {
                    'affiliations': {'type': 'text', 'fields': exact_field},
                    'contributors': {'type': 'text', 'fields': exact_field},
                    'date': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
                    'date_created': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
                    'date_modified': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
                    'date_published': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
                    'date_updated': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
                    'description': {'type': 'text'},
                    'funders': {'type': 'text', 'fields': exact_field},
                    'hosts': {'type': 'text', 'fields': exact_field},
                    'id': {'type': 'keyword', 'include_in_all': False},
                    'identifiers': {'type': 'text', 'fields': exact_field},
                    'justification': {'type': 'text', 'include_in_all': False},
                    'language': {'type': 'keyword', 'include_in_all': False},
                    'publishers': {'type': 'text', 'fields': exact_field},
                    'registration_type': {'type': 'keyword', 'include_in_all': False},
                    'retracted': {'type': 'boolean', 'include_in_all': False},
                    'sources': {'type': 'keyword', 'include_in_all': False},
                    'subjects': {'type': 'text', 'include_in_all': False, 'analyzer': 'subject_analyzer', 'search_analyzer': 'subject_search_analyzer'},
                    'subject_synonyms': {'type': 'text', 'include_in_all': False, 'analyzer': 'subject_analyzer', 'search_analyzer': 'subject_search_analyzer', 'copy_to': 'subjects'},
                    'tags': {'type': 'text', 'fields': exact_field},
                    'title': {'type': 'text', 'fields': exact_field},
                    'type': {'type': 'keyword', 'include_in_all': False},
                    'types': {'type': 'keyword', 'include_in_all': False},
                    'withdrawn': {'type': 'boolean', 'include_in_all': False},
                    'lists': {'type': 'object', 'dynamic': True, 'include_in_all': False},
                },
                'dynamic_templates': [
                    {'exact_field_on_lists_strings': {'path_match': 'lists.*', 'match_mapping_type': 'string', 'mapping': {'type': 'text', 'fields': exact_field}}},
                ]
            },
            'agents': {
                'dynamic': False,
                'properties': {
                    'id': {'type': 'keyword', 'include_in_all': False},
                    'identifiers': {'type': 'text', 'fields': exact_field},
                    'name': {'type': 'text', 'fields': {**autocomplete_field, **exact_field}},
                    'family_name': {'type': 'text', 'include_in_all': False},
                    'given_name': {'type': 'text', 'include_in_all': False},
                    'additional_name': {'type': 'text', 'include_in_all': False},
                    'suffix': {'type': 'text', 'include_in_all': False},
                    'location': {'type': 'text', 'include_in_all': False},
                    'sources': {'type': 'keyword', 'include_in_all': False},
                    'type': {'type': 'keyword', 'include_in_all': False},
                    'types': {'type': 'keyword', 'include_in_all': False},
                }
            },
            'sources': {
                'dynamic': False,
                'properties': {
                    'id': {'type': 'keyword', 'include_in_all': False},
                    'name': {'type': 'text', 'fields': {**autocomplete_field, **exact_field}},
                    'short_name': {'type': 'keyword', 'include_in_all': False},
                    'type': {'type': 'keyword', 'include_in_all': False},
                }
            },
            'tags': {
                'dynamic': False,
                'properties': {
                    'id': {'type': 'keyword', 'include_in_all': False},
                    'name': {'type': 'text', 'fields': {**autocomplete_field, **exact_field}},
                    'type': {'type': 'keyword', 'include_in_all': False},
                }
            },
        }

    def build_action_generator(self, index_name, message_type):
        self.assert_message_type(message_type)

        action_template = {
            '_index': index_name,
            '_type': 'creativeworks',
        }

        def action_generator(target_id_iter):
            record_qs = FormattedMetadataRecord.objects.filter(
                suid_id__in=target_id_iter,
                record_format='sharev2_elastic',  # TODO specify in config? or don't
            )
            for record in record_qs:
                source_doc = json.loads(record.formatted_metadata)
                if source_doc.pop('is_deleted', False):
                    action = {
                        **action_template,
                        '_id': source_doc['id'],
                        '_op_type': 'delete',
                    }
                else:
                    action = {
                        **action_template,
                        '_id': source_doc['id'],
                        '_op_type': 'index',
                        '_source': source_doc,
                    }
                yield (record.suid_id, action)
        return action_generator

1	import json	1✔
2
3	from share.models.core import FormattedMetadataRecord	1✔
4	from share.search.index_setup.base import IndexSetup	1✔
5	from share.search.messages import MessageType	1✔
6
7
8	class PostRendBackcompatIndexSetup(IndexSetup):	1✔
9	@property	1✔
10	def supported_message_types(self):
11	return {MessageType.INDEX_SUID}	×
12
13	@property	1✔
14	def index_settings(self):
15	from share.search.fetchers import CreativeWorkFetcher	1✔
16	return {	1✔
17	'analysis': {
18	'filter': {
19	'autocomplete_filter': {
20	'type': 'edge_ngram',
21	'min_gram': 1,
22	'max_gram': 20
23	}
24	},
25	'analyzer': {
26	'default': {
27	# same as 'standard' analyzer, plus html_strip
28	'type': 'custom',
29	'tokenizer': 'standard',
30	'filter': ['lowercase', 'stop'],
31	'char_filter': ['html_strip']
32	},
33	'autocomplete': {
34	'type': 'custom',
35	'tokenizer': 'standard',
36	'filter': [
37	'lowercase',
38	'autocomplete_filter'
39	]
40	},
41	'subject_analyzer': {
42	'type': 'custom',
43	'tokenizer': 'subject_tokenizer',
44	'filter': [
45	'lowercase',
46	]
47	},
48	'subject_search_analyzer': {
49	'type': 'custom',
50	'tokenizer': 'keyword',
51	'filter': [
52	'lowercase',
53	]
54	},
55	},
56	'tokenizer': {
57	'subject_tokenizer': {
58	'type': 'path_hierarchy',
59	'delimiter': CreativeWorkFetcher.SUBJECT_DELIMITER,
60	}
61	}
62	}
63	}
64
65	@property	1✔
66	def index_mappings(self):
67	autocomplete_field = {	1✔
68	'autocomplete': {
69	'type': 'string',
70	'analyzer': 'autocomplete',
71	'search_analyzer': 'standard',
72	'include_in_all': False
73	}
74	}
75
76	exact_field = {	1✔
77	'exact': {
78	'type': 'keyword',
79	# From Elasticsearch documentation:
80	# The value for ignore_above is the character count, but Lucene counts bytes.
81	# If you use UTF-8 text with many non-ASCII characters, you may want to set the limit to 32766 / 3 = 10922 since UTF-8 characters may occupy at most 3 bytes
82	'ignore_above': 10922
83	}
84	}
85
86	return {	1✔
87	'creativeworks': {
88	'dynamic': 'strict',
89	'properties': {
90	'affiliations': {'type': 'text', 'fields': exact_field},
91	'contributors': {'type': 'text', 'fields': exact_field},
92	'date': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
93	'date_created': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
94	'date_modified': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
95	'date_published': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
96	'date_updated': {'type': 'date', 'format': 'strict_date_optional_time', 'include_in_all': False},
97	'description': {'type': 'text'},
98	'funders': {'type': 'text', 'fields': exact_field},
99	'hosts': {'type': 'text', 'fields': exact_field},
100	'id': {'type': 'keyword', 'include_in_all': False},
101	'identifiers': {'type': 'text', 'fields': exact_field},
102	'justification': {'type': 'text', 'include_in_all': False},
103	'language': {'type': 'keyword', 'include_in_all': False},
104	'publishers': {'type': 'text', 'fields': exact_field},
105	'registration_type': {'type': 'keyword', 'include_in_all': False},
106	'retracted': {'type': 'boolean', 'include_in_all': False},
107	'sources': {'type': 'keyword', 'include_in_all': False},
108	'subjects': {'type': 'text', 'include_in_all': False, 'analyzer': 'subject_analyzer', 'search_analyzer': 'subject_search_analyzer'},
109	'subject_synonyms': {'type': 'text', 'include_in_all': False, 'analyzer': 'subject_analyzer', 'search_analyzer': 'subject_search_analyzer', 'copy_to': 'subjects'},
110	'tags': {'type': 'text', 'fields': exact_field},
111	'title': {'type': 'text', 'fields': exact_field},
112	'type': {'type': 'keyword', 'include_in_all': False},
113	'types': {'type': 'keyword', 'include_in_all': False},
114	'withdrawn': {'type': 'boolean', 'include_in_all': False},
115	'lists': {'type': 'object', 'dynamic': True, 'include_in_all': False},
116	},
117	'dynamic_templates': [
118	{'exact_field_on_lists_strings': {'path_match': 'lists.*', 'match_mapping_type': 'string', 'mapping': {'type': 'text', 'fields': exact_field}}},
119	]
120	},
121	'agents': {
122	'dynamic': False,
123	'properties': {
124	'id': {'type': 'keyword', 'include_in_all': False},
125	'identifiers': {'type': 'text', 'fields': exact_field},
126	'name': {'type': 'text', 'fields': {autocomplete_field, exact_field}},
127	'family_name': {'type': 'text', 'include_in_all': False},
128	'given_name': {'type': 'text', 'include_in_all': False},
129	'additional_name': {'type': 'text', 'include_in_all': False},
130	'suffix': {'type': 'text', 'include_in_all': False},
131	'location': {'type': 'text', 'include_in_all': False},
132	'sources': {'type': 'keyword', 'include_in_all': False},
133	'type': {'type': 'keyword', 'include_in_all': False},
134	'types': {'type': 'keyword', 'include_in_all': False},
135	}
136	},
137	'sources': {
138	'dynamic': False,
139	'properties': {
140	'id': {'type': 'keyword', 'include_in_all': False},
141	'name': {'type': 'text', 'fields': {autocomplete_field, exact_field}},
142	'short_name': {'type': 'keyword', 'include_in_all': False},
143	'type': {'type': 'keyword', 'include_in_all': False},
144	}
145	},
146	'tags': {
147	'dynamic': False,
148	'properties': {
149	'id': {'type': 'keyword', 'include_in_all': False},
150	'name': {'type': 'text', 'fields': {autocomplete_field, exact_field}},
151	'type': {'type': 'keyword', 'include_in_all': False},
152	}
153	},
154	}
155
156	def build_action_generator(self, index_name, message_type):	1✔
157	self.assert_message_type(message_type)	×
158
159	action_template = {	×
160	'_index': index_name,
161	'_type': 'creativeworks',
162	}
163
164	def action_generator(target_id_iter):	×
165	record_qs = FormattedMetadataRecord.objects.filter(	×
166	suid_id__in=target_id_iter,
167	record_format='sharev2_elastic', # TODO specify in config? or don't
168	)
169	for record in record_qs:	×
170	source_doc = json.loads(record.formatted_metadata)	×
171	if source_doc.pop('is_deleted', False):	×
172	action = {	×
173	**action_template,
174	'_id': source_doc['id'],
175	'_op_type': 'delete',
176	}
177	else:
178	action = {	×
179	**action_template,
180	'_id': source_doc['id'],
181	'_op_type': 'index',
182	'_source': source_doc,
183	}
184	yield (record.suid_id, action)	×
185	return action_generator	×

CenterForOpenScience / SHARE / 2618

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous