1135613467

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/adsabs/ADSReferencePipeline/commit/<a class=hub.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSReferencePipeline/commit/<a class="double-link" href="https://git"><a class=hub.com/adsabs/ADSReferencePipeline/commit/2cc7a06f71a46e77d79ed70e36e171801d06d5cc">2cc7a06f7">&lt;a href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/</a><a class="double-link" href="https://github.com/adsabs/ADSReferencePipeline/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/adsabs/ADSReferencePipeline/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/adsabs/ADSReferencePipeline/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/adsabs/ADSReferencePipeline/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/adsabs/ADSReferencePipeline/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/adsabs/ADSReferencePipeline/commit/2cc7a06f71a46e77d79ed70e36e171801d06d5cc">2cc7a06f7</a><a href="https://github.com/adsabs/ADSReferencePipeline/commit/2cc7a06f71a46e77d79ed70e36e171801d06d5cc">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/adsabs/ADSReferencePipeline/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/adsabs/ADSReferencePipeline/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/adsabs/ADSReferencePipeline/commit/2cc7a06f71a46e77d79ed70e36e171801d06d5cc&quot;&gt;2cc7a06f7&lt;/a&gt;&lt;a href=&quot;https://github.com/adsabs/ADSReferencePipeline/commit/2cc7a06f71a46e77d79ed70e36e171801d06d5cc&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/adsabs/ADSReferencePipeline/commit/2cc7a06f71a46e77d79ed70e36e171801d06d5c... (continued)

Pull Request Pull Request #1: the first

Run Details

3007 of 3007 new or added lines in 24 files covered. (100.0%)

2438 of 3007 relevant lines covered (81.08%)

0.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.78

/adsrefpipe/tests/unittests/test_tasks.py

import sys, os
project_home = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../'))
if project_home not in sys.path:
    sys.path.insert(0, project_home)

import datetime
import unittest
from mock import patch

from adsrefpipe import app, tasks, utils
from adsrefpipe.models import Base, Action, Parser, Reference

class TestWorkers(unittest.TestCase):

    postgresql_url_dict = {
        'port': 5432,
        'host': '127.0.0.1',
        'user': 'postgres',
        'database': 'postgres'
    }
    postgresql_url = 'postgresql://{user}:{user}@{host}:{port}/{database}' \
        .format(user=postgresql_url_dict['user'],
                host=postgresql_url_dict['host'],
                port=postgresql_url_dict['port'],
                database=postgresql_url_dict['database']
                )

    def setUp(self):
        self.test_dir = os.path.join(project_home, 'adsrefpipe/tests')
        unittest.TestCase.setUp(self)
        self.app = app.ADSReferencePipelineCelery('test', local_config={
            'SQLALCHEMY_URL': self.postgresql_url,
            'SQLALCHEMY_ECHO': False,
            'PROJ_HOME': project_home,
            'TEST_DIR': self.test_dir,
            'COMPARE_CLASSIC': False
        })
        Base.metadata.bind = self.app._session.get_bind()
        Base.metadata.create_all()

    def tearDown(self):
        unittest.TestCase.tearDown(self)
        Base.metadata.drop_all()
        self.app.close_app()

    def test_app(self):
        assert self.app._config.get('SQLALCHEMY_URL') == self.postgresql_url
        assert self.app.conf.get('SQLALCHEMY_URL') == self.postgresql_url

    def add_stub_data(self):
        """ Add stub data """
        actions = ['initial', 'retry', 'delete']

        parsers = [
            ('AGU', '.agu.xml'), ('AIP', '.aip.xml'), ('APS', '.ref.xml'), ('CrossRef', '.xref.xml'),
            ('ELSEVIER', '.elsevier.xml'), ('IOP', '.iop.xml'), ('JATS', '.jats.xml'), ('NATURE', '.nature.xml'),
            ('NLM', '.nlm3.xml'), ('SPRINGER', '.springer.xml'), ('Text', '.raw'), ('WILEY', '.wiley2.xml')
        ]
        with self.app.session_scope() as session:
            for action in actions:
                session.add(Action(status=action))
            session.commit()
            for parser in parsers:
                session.add(Parser(name=parser[0], source_extension=parser[1]))
            session.commit()

    def test_process_references(self):
        """ test process_references task """
        self.add_stub_data()
        resolved = [
            {
                "score": "1.0",
                "bibcode": "2011LRR....14....2U",
                "refstring": "J.-P. Uzan, Varying constants, gravitation and cosmology, Living Rev. Rel. 14 (2011) 2, [1009.5514]. ",
                "id": "H1I1"
            },
            {
                "score": "1.0",
                "bibcode": "2017RPPh...80l6902M",
                "refstring": "C. J. A. P. Martins, The status of varying constants: A review of the physics, searches and implications, 1709.02923.",
                "id": "H1I2",
            }
        ]
        # with patch.object(utils, 'resolve_references') as resolve_references, \
        #     patch.object(tasks.task_process_reference_file, 'apply_async') as process_reference_file:
        #
        #     resolve_references.return_value = resolved
        #     tasks.task_process_reference_file(os.path.join(self.test_dir, 'unittests/stubdata','00013.raw'))
        #     self.assertFalse(process_reference_file.called)
        #
        #     current_num_records = '' \
        #                           'Currently there are 1 records in `Reference` table, which holds reference files information.\n' \
        #                           'Currently there are 1 records in `History` table, which holds file level information for resolved run.\n' \
        #                           'Currently there are 2 records in `Resolved` table, which holds reference level information for resolved run.\n' \
        #                           'Currently there are 0 records in `Compare` table, which holds comparison of new and classic resolved run.\n'
        #     self.assertTrue(self.app.get_count_records() == current_num_records)

    def test_reprocess_subset_references(self):
        """ test reprocess_subset_references task """
        self.add_stub_data()
        # need a reference record
        with self.app.session_scope() as session:
            success = self.app.insert_reference_record(session,
                                                       Reference(bibcode='2020arXiv200400014K',
                                                                 source_filename=os.path.join(project_home, 'adsrefpipe/tests', 'unittests/stubdata/00014.raw'),
                                                                 resolved_filename='',
                                                                 parser='Text'))
            self.assertTrue(success)

        record = {
             'source_bibcode': '2020arXiv200400014K',
             'source_filename': os.path.join(project_home, 'adsrefpipe/tests', 'unittests/stubdata/00014.raw'),
             'source_modified': datetime.datetime(2020, 4, 3, 18, 8, 42),
             'parser': 'Text',
             'references': [{'item_num': 2, 'refstr': 'Arcangeli, J., Desert, J.-M., Parmentier, V., et al. 2019, A&A, 625, A136   '}]
        }
        resolved = [
            {
                "score": "1.0",
                "bibcode": "2019A&A...625A.136A",
                "refstring": "Arcangeli, J., Desert, J.-M., Parmentier, V., et al. 2019, A&A, 625, A136   ",
                "id": "H1I1"
            }
        ]
        # with patch.object(utils, 'resolve_references') as resolve_references, \
        #     patch.object(tasks.task_reprocess_subset_references, 'apply_async') as reprocess_subset_references:
        #
        #     resolve_references.return_value = resolved
        #     tasks.task_reprocess_subset_references(record)
        #     self.assertFalse(reprocess_subset_references.called)
        #
        #     current_num_records = '' \
        #                           'Currently there are 1 records in `Reference` table, which holds reference files information.\n' \
        #                           'Currently there are 1 records in `History` table, which holds file level information for resolved run.\n' \
        #                           'Currently there are 1 records in `Resolved` table, which holds reference level information for resolved run.\n' \
        #                           'Currently there are 0 records in `Compare` table, which holds comparison of new and classic resolved run.\n'
        #     self.assertTrue(self.app.get_count_records() == current_num_records)

1	import sys, os	1✔
2	project_home = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../'))	1✔
3	if project_home not in sys.path:	1✔
4	sys.path.insert(0, project_home)	×
5
6	import datetime	1✔
7	import unittest	1✔
8	from mock import patch	1✔
9
10	from adsrefpipe import app, tasks, utils	1✔
11	from adsrefpipe.models import Base, Action, Parser, Reference	1✔
12
13	class TestWorkers(unittest.TestCase):	1✔
14
15	postgresql_url_dict = {	1✔
16	'port': 5432,
17	'host': '127.0.0.1',
18	'user': 'postgres',
19	'database': 'postgres'
20	}
21	postgresql_url = 'postgresql://{user}:{user}@{host}:{port}/{database}' \	1✔
22	.format(user=postgresql_url_dict['user'],
23	host=postgresql_url_dict['host'],
24	port=postgresql_url_dict['port'],
25	database=postgresql_url_dict['database']
26	)
27
28	def setUp(self):	1✔
29	self.test_dir = os.path.join(project_home, 'adsrefpipe/tests')	1✔
30	unittest.TestCase.setUp(self)	1✔
31	self.app = app.ADSReferencePipelineCelery('test', local_config={	1✔
32	'SQLALCHEMY_URL': self.postgresql_url,
33	'SQLALCHEMY_ECHO': False,
34	'PROJ_HOME': project_home,
35	'TEST_DIR': self.test_dir,
36	'COMPARE_CLASSIC': False
37	})
38	Base.metadata.bind = self.app._session.get_bind()	1✔
39	Base.metadata.create_all()	1✔
40
41	def tearDown(self):	1✔
42	unittest.TestCase.tearDown(self)	1✔
43	Base.metadata.drop_all()	1✔
44	self.app.close_app()	1✔
45
46	def test_app(self):	1✔
47	assert self.app._config.get('SQLALCHEMY_URL') == self.postgresql_url	1✔
48	assert self.app.conf.get('SQLALCHEMY_URL') == self.postgresql_url	1✔
49
50	def add_stub_data(self):	1✔
51	""" Add stub data """
52	actions = ['initial', 'retry', 'delete']	1✔
53
54	parsers = [	1✔
55	('AGU', '.agu.xml'), ('AIP', '.aip.xml'), ('APS', '.ref.xml'), ('CrossRef', '.xref.xml'),
56	('ELSEVIER', '.elsevier.xml'), ('IOP', '.iop.xml'), ('JATS', '.jats.xml'), ('NATURE', '.nature.xml'),
57	('NLM', '.nlm3.xml'), ('SPRINGER', '.springer.xml'), ('Text', '.raw'), ('WILEY', '.wiley2.xml')
58	]
59	with self.app.session_scope() as session:	1✔
60	for action in actions:	1✔
61	session.add(Action(status=action))	1✔
62	session.commit()	1✔
63	for parser in parsers:	1✔
64	session.add(Parser(name=parser[0], source_extension=parser[1]))	1✔
65	session.commit()	1✔
66
67	def test_process_references(self):	1✔
68	""" test process_references task """
69	self.add_stub_data()	1✔
70	resolved = [	1✔
71	{
72	"score": "1.0",
73	"bibcode": "2011LRR....14....2U",
74	"refstring": "J.-P. Uzan, Varying constants, gravitation and cosmology, Living Rev. Rel. 14 (2011) 2, [1009.5514]. ",
75	"id": "H1I1"
76	},
77	{
78	"score": "1.0",
79	"bibcode": "2017RPPh...80l6902M",
80	"refstring": "C. J. A. P. Martins, The status of varying constants: A review of the physics, searches and implications, 1709.02923.",
81	"id": "H1I2",
82	}
83	]
84	# with patch.object(utils, 'resolve_references') as resolve_references, \
85	# patch.object(tasks.task_process_reference_file, 'apply_async') as process_reference_file:
86	#
87	# resolve_references.return_value = resolved
88	# tasks.task_process_reference_file(os.path.join(self.test_dir, 'unittests/stubdata','00013.raw'))
89	# self.assertFalse(process_reference_file.called)
90	#
91	# current_num_records = '' \
92	# 'Currently there are 1 records in `Reference` table, which holds reference files information.\n' \
93	# 'Currently there are 1 records in `History` table, which holds file level information for resolved run.\n' \
94	# 'Currently there are 2 records in `Resolved` table, which holds reference level information for resolved run.\n' \
95	# 'Currently there are 0 records in `Compare` table, which holds comparison of new and classic resolved run.\n'
96	# self.assertTrue(self.app.get_count_records() == current_num_records)
97
98	def test_reprocess_subset_references(self):	1✔
99	""" test reprocess_subset_references task """
100	self.add_stub_data()	1✔
101	# need a reference record
102	with self.app.session_scope() as session:	1✔
103	success = self.app.insert_reference_record(session,	1✔
104	Reference(bibcode='2020arXiv200400014K',
105	source_filename=os.path.join(project_home, 'adsrefpipe/tests', 'unittests/stubdata/00014.raw'),
106	resolved_filename='',
107	parser='Text'))
108	self.assertTrue(success)	1✔
109
110	record = {	1✔
111	'source_bibcode': '2020arXiv200400014K',
112	'source_filename': os.path.join(project_home, 'adsrefpipe/tests', 'unittests/stubdata/00014.raw'),
113	'source_modified': datetime.datetime(2020, 4, 3, 18, 8, 42),
114	'parser': 'Text',
115	'references': [{'item_num': 2, 'refstr': 'Arcangeli, J., Desert, J.-M., Parmentier, V., et al. 2019, A&A, 625, A136 '}]
116	}
117	resolved = [	1✔
118	{
119	"score": "1.0",
120	"bibcode": "2019A&A...625A.136A",
121	"refstring": "Arcangeli, J., Desert, J.-M., Parmentier, V., et al. 2019, A&A, 625, A136 ",
122	"id": "H1I1"
123	}
124	]
125	# with patch.object(utils, 'resolve_references') as resolve_references, \
126	# patch.object(tasks.task_reprocess_subset_references, 'apply_async') as reprocess_subset_references:
127	#
128	# resolve_references.return_value = resolved
129	# tasks.task_reprocess_subset_references(record)
130	# self.assertFalse(reprocess_subset_references.called)
131	#
132	# current_num_records = '' \
133	# 'Currently there are 1 records in `Reference` table, which holds reference files information.\n' \
134	# 'Currently there are 1 records in `History` table, which holds file level information for resolved run.\n' \
135	# 'Currently there are 1 records in `Resolved` table, which holds reference level information for resolved run.\n' \
136	# 'Currently there are 0 records in `Compare` table, which holds comparison of new and classic resolved run.\n'
137	# self.assertTrue(self.app.get_count_records() == current_num_records)

adsabs / ADSReferencePipeline / 1135613467

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous