#7642

Committed 26 Mar 2025 10:50PM UTC coverage: 90.343%. Remained the same

Build # #7642

Build Type

push

coveralls-python

Committed by

ottojolanki

Commit Message

bump version to 72.0.0

Coverage Stats

0 of 1 new or added line in 1 file covered. (0.0%)

8148 of 9019 relevant lines covered (90.34%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.4

/src/igvfd/__init__.py

__version__ = '72.0.0'


import igvfd.schema_formats  # needed to import before snovault to add FormatCheckers
import base64
import codecs
import copy
import json
import os
import subprocess

from pathlib import Path
from pyramid.config import Configurator
from pyramid.path import (
    AssetResolver,
    caller_package,
)
from pyramid.session import SignedCookieSessionFactory
from pyramid.settings import (
    aslist,
    asbool,
)
from sqlalchemy import engine_from_config
from webob.cookies import JSONSerializer
from snovault.json_renderer import json_renderer


STATIC_MAX_AGE = 0


def json_asset(spec, **kw):
    utf8 = codecs.getreader('utf-8')
    asset = AssetResolver(caller_package()).resolve(spec)
    return json.load(utf8(asset.stream()), **kw)


def static_resources(config):
    from pkg_resources import resource_filename
    import mimetypes
    mimetypes.init()
    mimetypes.init([resource_filename('igvfd', 'static/mime.types')])
    config.add_static_view('static', 'static', cache_max_age=STATIC_MAX_AGE)
    config.add_static_view('profiles', 'schemas', cache_max_age=STATIC_MAX_AGE)

    favicon_path = '/static/img/favicon.ico'
    if config.route_prefix:
        favicon_path = '/%s%s' % (config.route_prefix, favicon_path)

    config.add_route('favicon.ico', 'favicon.ico')

    def favicon(request):
        subreq = request.copy()
        subreq.path_info = favicon_path
        response = request.invoke_subrequest(subreq)
        return response

    config.add_view(favicon, route_name='favicon.ico')


def changelogs(config):
    config.add_static_view(
        'profiles/changelogs',
        'schemas/changelogs',
        cache_max_age=STATIC_MAX_AGE
    )


def configure_engine(settings):
    settings = copy.deepcopy(settings)
    engine_url = os.environ.get('SQLALCHEMY_URL') or settings['sqlalchemy.url']
    settings['sqlalchemy.url'] = engine_url
    engine_opts = {}
    if engine_url.startswith('postgresql'):
        application_name = 'app'
        engine_opts = dict(
            isolation_level='REPEATABLE READ',
            json_serializer=json_renderer.dumps,
            connect_args={'application_name': application_name}
        )
    engine = engine_from_config(settings, 'sqlalchemy.', **engine_opts)
    if engine.url.drivername == 'postgresql':
        timeout = settings.get('postgresql.statement_timeout')
        if timeout:
            timeout = int(timeout) * 1000
            set_postgresql_statement_timeout(engine, timeout)
    return engine


def set_postgresql_statement_timeout(engine, timeout=20 * 1000):
    """ Prevent Postgres waiting indefinitely for a lock.
    """
    from sqlalchemy import event
    import psycopg2

    @event.listens_for(engine, 'connect')
    def connect(dbapi_connection, connection_record):
        cursor = dbapi_connection.cursor()
        try:
            cursor.execute('SET statement_timeout TO %d' % timeout)
        except psycopg2.Error:
            dbapi_connection.rollback()
        finally:
            cursor.close()
            dbapi_connection.commit()


def configure_dbsession(config):
    from snovault import DBSESSION
    settings = config.registry.settings
    DBSession = settings.pop(DBSESSION, None)
    if DBSession is None:
        engine = configure_engine(settings)

        if asbool(settings.get('create_tables', False)):
            from snovault.storage import Base
            Base.metadata.create_all(engine)

        import snovault.storage
        import zope.sqlalchemy
        from sqlalchemy import orm

        DBSession = orm.scoped_session(orm.sessionmaker(bind=engine))
        zope.sqlalchemy.register(DBSession)
        snovault.storage.register(DBSession)

    config.registry[DBSESSION] = DBSession


def load_workbook(app, workbook_filename, docsdir, test=False):
    from .loadxl import load_all
    from webtest import TestApp
    environ = {
        'HTTP_ACCEPT': 'application/json',
        'REMOTE_USER': 'IMPORT',
    }
    testapp = TestApp(app, environ)
    load_all(testapp, workbook_filename, docsdir, test=test)


def json_from_path(path, default=None):
    if path is None:
        return default
    return json.load(open(path))


def configure_sqs_client(config):
    from snovault.app import configure_sqs_client
    configure_sqs_client(config)


def configure_transaction_queue(config):
    from snovault.app import configure_transaction_queue
    configure_transaction_queue(config)


def configure_invalidation_queue(config):
    from snovault.app import configure_invalidation_queue
    configure_invalidation_queue(config)


def configure_deduplication_queue(config):
    from snovault.app import configure_deduplication_queue
    configure_deduplication_queue(config)


def configure_transaction_dead_letter_queue(config):
    from snovault.app import configure_transaction_dead_letter_queue
    configure_transaction_dead_letter_queue(config)


def configure_invalidation_dead_letter_queue(config):
    from snovault.app import configure_invalidation_dead_letter_queue
    configure_invalidation_dead_letter_queue(config)


def configure_deduplication_dead_letter_queue(config):
    from snovault.app import configure_deduplication_dead_letter_queue
    configure_deduplication_dead_letter_queue(config)


def session(config):
    """ To create a session secret on the server:
    $ cat /dev/urandom | head -c 256 | base64 > session-secret.b64
    """
    from igvfd.cookie import add_session_cookie_name_to_settings
    settings = config.registry.settings
    if 'session.secret' in settings:
        secret = settings['session.secret'].strip()
        if secret.startswith('/'):
            secret = open(secret).read()
            secret = base64.b64decode(secret)
    else:
        secret = os.environ.get('SESSION_SECRET') or os.urandom(256)
    # auth_tkt has no timeout set
    # cookie will still expire at browser close
    if 'session.timeout' in settings:
        timeout = int(settings['session.timeout'])
    else:
        timeout = 60 * 60 * 24
    add_session_cookie_name_to_settings(settings, secret)
    session_factory = SignedCookieSessionFactory(
        cookie_name=settings['session_cookie_name'],
        secret=secret,
        timeout=timeout,
        reissue_time=2**32,  # None does not work
        serializer=JSONSerializer(),
        domain=settings.get('session_cookie_domain', None)
    )
    config.set_session_factory(session_factory)


def app_version(config):
    config.registry.settings['snovault.app_version'] = __version__


def main(global_config, **local_config):
    """ This function returns a Pyramid WSGI application.
    """
    settings = global_config
    settings.update(local_config)

    settings['snovault.jsonld.namespaces'] = json_asset('igvfd:schemas/namespaces.json')
    settings['snovault.jsonld.terms_namespace'] = 'https://api.data.igvf.org/terms/'
    settings['snovault.jsonld.terms_prefix'] = 'igvf'

    # Before settings are passed to Configurator.
    OPENSEARCH_URL = os.environ.get('OPENSEARCH_URL')
    if OPENSEARCH_URL:
        settings['elasticsearch.server'] = OPENSEARCH_URL

    config = Configurator(settings=settings)
    config.include(app_version)

    config.include('pyramid_multiauth')  # must be before calling set_authorization_policy
    from pyramid_localroles import LocalRolesAuthorizationPolicy
    # Override default authz policy set by pyramid_multiauth
    config.set_authorization_policy(LocalRolesAuthorizationPolicy())
    config.include(session)
    # Must go before other route registration.
    config.include('.cors')
    config.include('.auth0')
    config.include('.cookie')

    config.include(configure_dbsession)
    config.include(configure_sqs_client)
    config.include(configure_transaction_queue)
    config.include(configure_invalidation_queue)
    config.include(configure_deduplication_queue)
    config.include(configure_transaction_dead_letter_queue)
    config.include(configure_invalidation_dead_letter_queue)
    config.include(configure_deduplication_dead_letter_queue)
    config.include('snovault')
    config.commit()  # commit so search can override listing

    # Render an HTML page to browsers and a JSON document for API clients
    config.include('.renderers')
    config.include('.authentication')
    config.include('.server_defaults')
    config.include('.types')
    config.include('.searches.configs')
    config.include('.root')

    config.include('.ontology')
    config.include('.report')
    config.include('.metadata.metadata')
    config.include('.metadata.batch_download')
    config.include('.verify_email')

    if 'elasticsearch.server' in config.registry.settings:
        config.include('snovault.elasticsearch')
        config.include('igvfd.search_views')

    config.include(static_resources)
    config.include(changelogs)

    config.include('.upgrade')
    config.include('.audit')

    if asbool(settings.get('testing', False)):
        config.include('.tests.testing_views')

    config.include('igvfd.mappings.register')
    config.include('igvfd.feature_flags')

    app = config.make_wsgi_app()

    return app

NEW 1	__version__ = '72.0.0'	×
2
3
4	import igvfd.schema_formats # needed to import before snovault to add FormatCheckers	×
5	import base64	×
6	import codecs	×
7	import copy	×
8	import json	×
9	import os	×
10	import subprocess	×
11
12	from pathlib import Path	×
13	from pyramid.config import Configurator	×
14	from pyramid.path import (	×
15	AssetResolver,
16	caller_package,
17	)
18	from pyramid.session import SignedCookieSessionFactory	×
19	from pyramid.settings import (	×
20	aslist,
21	asbool,
22	)
23	from sqlalchemy import engine_from_config	×
24	from webob.cookies import JSONSerializer	×
25	from snovault.json_renderer import json_renderer	×
26
27
28	STATIC_MAX_AGE = 0	×
29
30
31	def json_asset(spec, **kw):	×
32	utf8 = codecs.getreader('utf-8')	1✔
33	asset = AssetResolver(caller_package()).resolve(spec)	1✔
34	return json.load(utf8(asset.stream()), **kw)	1✔
35
36
37	def static_resources(config):	×
38	from pkg_resources import resource_filename	1✔
39	import mimetypes	1✔
40	mimetypes.init()	1✔
41	mimetypes.init([resource_filename('igvfd', 'static/mime.types')])	1✔
42	config.add_static_view('static', 'static', cache_max_age=STATIC_MAX_AGE)	1✔
43	config.add_static_view('profiles', 'schemas', cache_max_age=STATIC_MAX_AGE)	1✔
44
45	favicon_path = '/static/img/favicon.ico'	1✔
46	if config.route_prefix:	1✔
47	favicon_path = '/%s%s' % (config.route_prefix, favicon_path)	×
48
49	config.add_route('favicon.ico', 'favicon.ico')	1✔
50
51	def favicon(request):	1✔
52	subreq = request.copy()	×
53	subreq.path_info = favicon_path	×
54	response = request.invoke_subrequest(subreq)	×
55	return response	×
56
57	config.add_view(favicon, route_name='favicon.ico')	1✔
58
59
60	def changelogs(config):	×
61	config.add_static_view(	1✔
62	'profiles/changelogs',
63	'schemas/changelogs',
64	cache_max_age=STATIC_MAX_AGE
65	)
66
67
68	def configure_engine(settings):	×
69	settings = copy.deepcopy(settings)	1✔
70	engine_url = os.environ.get('SQLALCHEMY_URL') or settings['sqlalchemy.url']	1✔
71	settings['sqlalchemy.url'] = engine_url	1✔
72	engine_opts = {}	1✔
73	if engine_url.startswith('postgresql'):	1✔
74	application_name = 'app'	1✔
75	engine_opts = dict(	1✔
76	isolation_level='REPEATABLE READ',
77	json_serializer=json_renderer.dumps,
78	connect_args={'application_name': application_name}
79	)
80	engine = engine_from_config(settings, 'sqlalchemy.', **engine_opts)	1✔
81	if engine.url.drivername == 'postgresql':	1✔
82	timeout = settings.get('postgresql.statement_timeout')	1✔
83	if timeout:	1✔
84	timeout = int(timeout) * 1000	1✔
85	set_postgresql_statement_timeout(engine, timeout)	1✔
86	return engine	1✔
87
88
89	def set_postgresql_statement_timeout(engine, timeout=20 * 1000):	×
90	""" Prevent Postgres waiting indefinitely for a lock.
91	"""
92	from sqlalchemy import event	1✔
93	import psycopg2	1✔
94
95	@event.listens_for(engine, 'connect')	1✔
96	def connect(dbapi_connection, connection_record):	1✔
97	cursor = dbapi_connection.cursor()	1✔
98	try:	1✔
99	cursor.execute('SET statement_timeout TO %d' % timeout)	1✔
100	except psycopg2.Error:	×
101	dbapi_connection.rollback()	×
102	finally:
103	cursor.close()	1✔
104	dbapi_connection.commit()	1✔
105
106
107	def configure_dbsession(config):	×
108	from snovault import DBSESSION	1✔
109	settings = config.registry.settings	1✔
110	DBSession = settings.pop(DBSESSION, None)	1✔
111	if DBSession is None:	1✔
112	engine = configure_engine(settings)	1✔
113
114	if asbool(settings.get('create_tables', False)):	1✔
115	from snovault.storage import Base	1✔
116	Base.metadata.create_all(engine)	1✔
117
118	import snovault.storage	1✔
119	import zope.sqlalchemy	1✔
120	from sqlalchemy import orm	1✔
121
122	DBSession = orm.scoped_session(orm.sessionmaker(bind=engine))	1✔
123	zope.sqlalchemy.register(DBSession)	1✔
124	snovault.storage.register(DBSession)	1✔
125
126	config.registry[DBSESSION] = DBSession	1✔
127
128
129	def load_workbook(app, workbook_filename, docsdir, test=False):	×
130	from .loadxl import load_all	×
131	from webtest import TestApp	×
132	environ = {	×
133	'HTTP_ACCEPT': 'application/json',
134	'REMOTE_USER': 'IMPORT',
135	}
136	testapp = TestApp(app, environ)	×
137	load_all(testapp, workbook_filename, docsdir, test=test)	×
138
139
140	def json_from_path(path, default=None):	×
141	if path is None:	×
142	return default	×
143	return json.load(open(path))	×
144
145
146	def configure_sqs_client(config):	×
147	from snovault.app import configure_sqs_client	1✔
148	configure_sqs_client(config)	1✔
149
150
151	def configure_transaction_queue(config):	×
152	from snovault.app import configure_transaction_queue	1✔
153	configure_transaction_queue(config)	1✔
154
155
156	def configure_invalidation_queue(config):	×
157	from snovault.app import configure_invalidation_queue	1✔
158	configure_invalidation_queue(config)	1✔
159
160
161	def configure_deduplication_queue(config):	×
162	from snovault.app import configure_deduplication_queue	1✔
163	configure_deduplication_queue(config)	1✔
164
165
166	def configure_transaction_dead_letter_queue(config):	×
167	from snovault.app import configure_transaction_dead_letter_queue	1✔
168	configure_transaction_dead_letter_queue(config)	1✔
169
170
171	def configure_invalidation_dead_letter_queue(config):	×
172	from snovault.app import configure_invalidation_dead_letter_queue	1✔
173	configure_invalidation_dead_letter_queue(config)	1✔
174
175
176	def configure_deduplication_dead_letter_queue(config):	×
177	from snovault.app import configure_deduplication_dead_letter_queue	1✔
178	configure_deduplication_dead_letter_queue(config)	1✔
179
180
181	def session(config):	×
182	""" To create a session secret on the server:
183	$ cat /dev/urandom \| head -c 256 \| base64 > session-secret.b64
184	"""
185	from igvfd.cookie import add_session_cookie_name_to_settings	1✔
186	settings = config.registry.settings	1✔
187	if 'session.secret' in settings:	1✔
188	secret = settings['session.secret'].strip()	×
189	if secret.startswith('/'):	×
190	secret = open(secret).read()	×
191	secret = base64.b64decode(secret)	×
192	else:
193	secret = os.environ.get('SESSION_SECRET') or os.urandom(256)	1✔
194	# auth_tkt has no timeout set
195	# cookie will still expire at browser close
196	if 'session.timeout' in settings:	1✔
197	timeout = int(settings['session.timeout'])	×
198	else:
199	timeout = 60 * 60 * 24	1✔
200	add_session_cookie_name_to_settings(settings, secret)	1✔
201	session_factory = SignedCookieSessionFactory(	1✔
202	cookie_name=settings['session_cookie_name'],
203	secret=secret,
204	timeout=timeout,
205	reissue_time=2**32, # None does not work
206	serializer=JSONSerializer(),
207	domain=settings.get('session_cookie_domain', None)
208	)
209	config.set_session_factory(session_factory)	1✔
210
211
212	def app_version(config):	×
213	config.registry.settings['snovault.app_version'] = __version__	1✔
214
215
216	def main(global_config, **local_config):	×
217	""" This function returns a Pyramid WSGI application.
218	"""
219	settings = global_config	1✔
220	settings.update(local_config)	1✔
221
222	settings['snovault.jsonld.namespaces'] = json_asset('igvfd:schemas/namespaces.json')	1✔
223	settings['snovault.jsonld.terms_namespace'] = 'https://api.data.igvf.org/terms/'	1✔
224	settings['snovault.jsonld.terms_prefix'] = 'igvf'	1✔
225
226	# Before settings are passed to Configurator.
227	OPENSEARCH_URL = os.environ.get('OPENSEARCH_URL')	1✔
228	if OPENSEARCH_URL:	1✔
229	settings['elasticsearch.server'] = OPENSEARCH_URL	×
230
231	config = Configurator(settings=settings)	1✔
232	config.include(app_version)	1✔
233
234	config.include('pyramid_multiauth') # must be before calling set_authorization_policy	1✔
235	from pyramid_localroles import LocalRolesAuthorizationPolicy	1✔
236	# Override default authz policy set by pyramid_multiauth
237	config.set_authorization_policy(LocalRolesAuthorizationPolicy())	1✔
238	config.include(session)	1✔
239	# Must go before other route registration.
240	config.include('.cors')	1✔
241	config.include('.auth0')	1✔
242	config.include('.cookie')	1✔
243
244	config.include(configure_dbsession)	1✔
245	config.include(configure_sqs_client)	1✔
246	config.include(configure_transaction_queue)	1✔
247	config.include(configure_invalidation_queue)	1✔
248	config.include(configure_deduplication_queue)	1✔
249	config.include(configure_transaction_dead_letter_queue)	1✔
250	config.include(configure_invalidation_dead_letter_queue)	1✔
251	config.include(configure_deduplication_dead_letter_queue)	1✔
252	config.include('snovault')	1✔
253	config.commit() # commit so search can override listing	1✔
254
255	# Render an HTML page to browsers and a JSON document for API clients
256	config.include('.renderers')	1✔
257	config.include('.authentication')	1✔
258	config.include('.server_defaults')	1✔
259	config.include('.types')	1✔
260	config.include('.searches.configs')	1✔
261	config.include('.root')	1✔
262
263	config.include('.ontology')	1✔
264	config.include('.report')	1✔
265	config.include('.metadata.metadata')	1✔
266	config.include('.metadata.batch_download')	1✔
267	config.include('.verify_email')	1✔
268
269	if 'elasticsearch.server' in config.registry.settings:	1✔
270	config.include('snovault.elasticsearch')	1✔
271	config.include('igvfd.search_views')	1✔
272
273	config.include(static_resources)	1✔
274	config.include(changelogs)	1✔
275
276	config.include('.upgrade')	1✔
277	config.include('.audit')	1✔
278
279	if asbool(settings.get('testing', False)):	1✔
280	config.include('.tests.testing_views')	1✔
281
282	config.include('igvfd.mappings.register')	1✔
283	config.include('igvfd.feature_flags')	1✔
284
285	app = config.make_wsgi_app()	1✔
286
287	return app	1✔

IGVF-DACC / igvfd / #7642

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous