3618006776

Build Type

github-actions

Committed by GitHub

Commit Message

Merge 67bb84ee7 into c844a259b

Pull Request Pull Request #79: Use url_for to generate URLs

Run Details

8 of 8 new or added lines in 3 files covered. (100.0%)

118 of 1039 relevant lines covered (11.36%)

0.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/make_html.py

# Script to generate static HTML pages
# This uses Jinja templating to render the HTML templates in the 'templates' folder
# Data is based on the files in the 'stats-calculated' folder, and extra logic in other files in this repository

import argparse
import json
import re
import subprocess
from collections import defaultdict

from flask import Flask, render_template, abort, Response, send_from_directory
import pytz

import licenses
import timeliness
import forwardlooking
import comprehensiveness
# import coverage
import summary_stats
import humanitarian
from vars import expected_versions
import text
from datetime import datetime
from dateutil import parser
from data import (
    ckan,
    ckan_publishers,
    codelist_mapping,
    codelist_sets,
    codelist_lookup,
    current_stats,
    dataset_to_publisher_dict,
    github_issues,
    get_publisher_stats,
    MAJOR_VERSIONS,
    metadata,
    publisher_name,
    publishers_ordered_by_title,
    is_valid_element,
    slugs)

app = Flask(__name__, static_url_path='')


def dictinvert(d):
    inv = defaultdict(list)
    for k, v in d.items():
        inv[v].append(k)
    return inv


def nested_dictinvert(d):
    inv = defaultdict(lambda: defaultdict(int))
    for k, v in d.items():
        for k2, v2 in v.items():
            inv[k2][k] += v2
    return inv


def dataset_to_publisher(dataset_slug):
    """ Converts a dataset (package) slug e.g. dfid-bd to the corresponding publisher
    slug e.g. dfid """
    return dataset_to_publisher_dict.get(dataset_slug, '')


def firstint(s):
    if s[0].startswith('<'):
        return 0
    m = re.search(r'\d+', s[0])
    return int(m.group(0))


def round_nicely(val, ndigits=2):
    """ Round a float, but remove the trailing .0 from integers that python insists on
    """
    if int(val) == float(val):
        return int(val)
    return round(float(val), ndigits)


def xpath_to_url(path):
    path = path.strip('./')
    # remove conditions
    path = re.sub(r'\[[^]]+\]', '', path)
    if path.startswith('iati-activity'):
        url = 'https://reference.codeforiati.org/activity-standard/iati-activities/' + path.split('@')[0]
    elif path.startswith('iati-organisation'):
        url = 'https://reference.codeforiati.org/organisation-standard/iati-organisations/' + path.split('@')[0]
    else:
        url = 'https://reference.codeforiati.org/activity-standard/iati-activities/iati-activity/' + path.split('@')[0]
    if '@' in path:
        url += '#attributes'
    return url


def registration_agency(orgid):
    for code in codelist_sets['2']['OrganisationRegistrationAgency']:
        if orgid.startswith(code):
            return code


def get_codelist_values(codelist_values_for_element):
    """Return a list of unique values present within a one-level nested dictionary.
       Envisaged usage is to gather the codelist values used by each publisher, as in
       stats/current/inverted-publisher/codelist_values_by_major_version.json
       Input: Set of codelist values for a given element (listed by publisher), for example:
              current_stats['inverted_publisher']['codelist_values_by_major_version']['1']['.//@xml:lang']
    """
    return list(set([y for x in codelist_values_for_element.items() for y in list(x[1].keys())]))


# Store data processing times
date_time_data_obj = parser.parse(metadata['created_at'])

# Custom Jinja filters
app.jinja_env.filters['xpath_to_url'] = xpath_to_url
app.jinja_env.filters['url_to_filename'] = lambda x: x.rstrip('/').split('/')[-1]
app.jinja_env.filters['has_future_transactions'] = timeliness.has_future_transactions
app.jinja_env.filters['round_nicely'] = round_nicely

# Custom Jinja globals
app.jinja_env.globals['dataset_to_publisher'] = dataset_to_publisher
app.jinja_env.globals['url'] = lambda x: '/' if x == 'index.html' else x
app.jinja_env.globals['datetime_generated'] = lambda: datetime.utcnow().replace(tzinfo=pytz.utc).strftime('%-d %B %Y (at %H:%M %Z)')
app.jinja_env.globals['datetime_data'] = date_time_data_obj.strftime('%-d %B %Y (at %H:%M %Z)')
app.jinja_env.globals['commit_hash'] = subprocess.run(
    'git show --format=%H --no-patch'.split(),
    capture_output=True).stdout.decode().strip()
app.jinja_env.globals['stats_commit_hash'] = subprocess.run(
    'git -C stats-calculated show --format=%H --no-patch'.split(),
    capture_output=True).stdout.decode().strip()
app.jinja_env.globals['stats_url'] = 'https://stats.codeforiati.org'
app.jinja_env.globals['stats_gh_url'] = 'https://github.com/codeforIATI/IATI-Stats-public/tree/' + app.jinja_env.globals['stats_commit_hash']
app.jinja_env.globals['sorted'] = sorted
app.jinja_env.globals['enumerate'] = enumerate
app.jinja_env.globals['top_titles'] = text.top_titles
app.jinja_env.globals['page_titles'] = text.page_titles
app.jinja_env.globals['short_page_titles'] = text.short_page_titles
app.jinja_env.globals['page_leads'] = text.page_leads
app.jinja_env.globals['page_sub_leads'] = text.page_sub_leads
app.jinja_env.globals['top_navigation'] = text.top_navigation
app.jinja_env.globals['navigation'] = text.navigation
app.jinja_env.globals['navigation_reverse'] = {page: k for k, pages in text.navigation.items() for page in pages}
app.jinja_env.globals['navigation_reverse'].update({k: k for k in text.navigation})
app.jinja_env.globals['current_stats'] = current_stats
app.jinja_env.globals['ckan'] = ckan
app.jinja_env.globals['ckan_publishers'] = ckan_publishers
app.jinja_env.globals['github_issues'] = github_issues
app.jinja_env.globals['publisher_name'] = publisher_name
app.jinja_env.globals['publishers_ordered_by_title'] = publishers_ordered_by_title
app.jinja_env.globals['get_publisher_stats'] = get_publisher_stats
app.jinja_env.globals['set'] = set
app.jinja_env.globals['firstint'] = firstint
app.jinja_env.globals['expected_versions'] = expected_versions
app.jinja_env.globals['current_year'] = datetime.utcnow().year
# Following variables set in coverage branch but not in master
# app.jinja_env.globals['float'] = float
# app.jinja_env.globals['dac2012'] = dac2012
app.jinja_env.globals['MAJOR_VERSIONS'] = MAJOR_VERSIONS

app.jinja_env.globals['slugs'] = slugs
app.jinja_env.globals['codelist_mapping'] = codelist_mapping
app.jinja_env.globals['codelist_sets'] = codelist_sets
app.jinja_env.globals['codelist_lookup'] = codelist_lookup
app.jinja_env.globals['get_codelist_values'] = get_codelist_values
app.jinja_env.globals['is_valid_element'] = is_valid_element

basic_page_names = [
    'headlines',
    'data_quality',
    'exploring_data',
    'publishers',
    'publishing_stats',
    'timeliness',
    'timeliness_timelag',
    'forwardlooking',
    'comprehensiveness',
    'comprehensiveness_core',
    'comprehensiveness_financials',
    'comprehensiveness_valueadded',
    # 'coverage',
    'summary_stats',
    'humanitarian',
    'files',
    'activities',
    'download',
    'xml',
    'validation',
    'versions',
    'organisation',
    'identifiers',
    'reporting_orgs',
    'elements',
    'codelists',
    'booleans',
    'dates',
    'traceability',
    'faq',
]


@app.route('/<page_name>.html')
def basic_page(page_name):
    if page_name in basic_page_names:
        kwargs = {}
        if page_name.startswith('timeliness'):
            kwargs['timeliness'] = timeliness
            parent_page_name = 'timeliness'
        elif page_name.startswith('forwardlooking'):
            kwargs['forwardlooking'] = forwardlooking
            parent_page_name = 'forwardlooking'
        elif page_name.startswith('comprehensiveness'):
            kwargs['comprehensiveness'] = comprehensiveness
            parent_page_name = 'comprehensiveness'
        elif page_name.startswith('coverage'):
            # kwargs['coverage'] = coverage
            parent_page_name = 'coverage'
        elif page_name.startswith('summary_stats'):
            kwargs['summary_stats'] = summary_stats
            parent_page_name = 'summary_stats'
        elif page_name.startswith('humanitarian'):
            kwargs['humanitarian'] = humanitarian
            parent_page_name = 'humanitarian'
        else:
            parent_page_name = page_name
        return render_template(page_name + '.html', page=parent_page_name, **kwargs)
    else:
        abort(404)


@app.route('/data/download_errors.json')
def download_errors_json():
    return Response(json.dumps(current_stats['download_errors'], indent=2), mimetype='application/json'),


@app.route('/')
def homepage():
    return render_template('index.html', page='index')


app.add_url_rule('/licenses.html', 'licenses', licenses.main)
app.add_url_rule('/license/<license>.html', 'licenses_individual_license', licenses.individual_license)


@app.route('/publisher/<publisher>.html')
def publisher(publisher):
    publisher_stats = get_publisher_stats(publisher)
    budget_table = [{
                    'year': 'Total',
                    'count_total': sum(sum(x.values()) for x in publisher_stats['count_budgets_by_type_by_year'].values()),
                    'sum_total': {currency: sum(sums.values()) for by_currency in publisher_stats['sum_budgets_by_type_by_year'].values() for currency, sums in by_currency.items()},
                    'count_original': sum(publisher_stats['count_budgets_by_type_by_year']['1'].values()) if '1' in publisher_stats['count_budgets_by_type_by_year'] else None,
                    'sum_original': {k: sum(v.values()) for k, v in publisher_stats['sum_budgets_by_type_by_year']['1'].items()} if '1' in publisher_stats['sum_budgets_by_type_by_year'] else None,
                    'count_revised': sum(publisher_stats['count_budgets_by_type_by_year']['2'].values()) if '2' in publisher_stats['count_budgets_by_type_by_year'] else None,
                    'sum_revised': {k: sum(v.values()) for k, v in publisher_stats['sum_budgets_by_type_by_year']['2'].items()} if '2' in publisher_stats['sum_budgets_by_type_by_year'] else None
                    }] + [{'year': year,
                           'count_total': sum(x[year] for x in publisher_stats['count_budgets_by_type_by_year'].values() if year in x),
                           'sum_total': {currency: sums.get(year) for by_currency in publisher_stats['sum_budgets_by_type_by_year'].values() for currency, sums in by_currency.items()},
                           'count_original': publisher_stats['count_budgets_by_type_by_year']['1'].get(year) if '1' in publisher_stats['count_budgets_by_type_by_year'] else None,
                           'sum_original': {k: v.get(year) for k, v in publisher_stats['sum_budgets_by_type_by_year']['1'].items()} if '1' in publisher_stats['sum_budgets_by_type_by_year'] else None,
                           'count_revised': publisher_stats['count_budgets_by_type_by_year']['2'].get(year) if '2' in publisher_stats['count_budgets_by_type_by_year'] else None,
                           'sum_revised': {k: v.get(year) for k, v in publisher_stats['sum_budgets_by_type_by_year']['2'].items()} if '2' in publisher_stats['sum_budgets_by_type_by_year'] else None
                           } for year in sorted(set(sum((list(x.keys()) for x in publisher_stats['count_budgets_by_type_by_year'].values()), [])))
                          ]
    failure_count = len(current_stats['inverted_file_publisher'][publisher]['validation'].get('fail', {}))
    return render_template('publisher.html',
                           url=lambda x: '../' + x,
                           publisher=publisher,
                           publisher_stats=publisher_stats,
                           failure_count=failure_count,
                           publisher_inverted=get_publisher_stats(publisher, 'inverted-file'),
                           publisher_licenses=licenses.licenses_for_publisher(publisher),
                           budget_table=budget_table,)


@app.route('/codelist/<major_version>/<slug>.html')
def codelist(major_version, slug):
    i = slugs['codelist'][major_version]['by_slug'][slug]
    element = list(current_stats['inverted_publisher']['codelist_values_by_major_version'][major_version])[i]
    values = nested_dictinvert(list(current_stats['inverted_publisher']['codelist_values_by_major_version'][major_version].values())[i])
    return render_template('codelist.html',
                           element=element,
                           values=values,
                           reverse_codelist_mapping={major_version: dictinvert(mapping) for major_version, mapping in codelist_mapping.items()},
                           url=lambda x: '../../' + x,
                           major_version=major_version,
                           page='codelists')


@app.route('/element/<slug>.html')
def element(slug):
    i = slugs['element']['by_slug'][slug]
    element = list(current_stats['inverted_publisher']['elements'])[i]
    publishers = list(current_stats['inverted_publisher']['elements'].values())[i]
    return render_template('element.html',
                           element=element,
                           publishers=publishers,
                           url=lambda x: '../' + x,
                           element_or_attribute='attribute' if '@' in element else 'element',
                           page='elements')


@app.route('/registration_agencies.html')
def registration_agencies():
    registration_agencies = defaultdict(int)
    registration_agencies_publishers = defaultdict(list)
    nonmatching = []
    for orgid, publishers in current_stats['inverted_publisher']['reporting_orgs'].items():
        reg_ag = registration_agency(orgid)
        if reg_ag:
            registration_agencies[reg_ag] += 1
            registration_agencies_publishers[reg_ag] += list(publishers)
        else:
            nonmatching.append((orgid, publishers))
    return render_template('registration_agencies.html',
                           page='registration_agencies',
                           registration_agencies=registration_agencies,
                           registration_agencies_publishers=registration_agencies_publishers,
                           nonmatching=nonmatching)


# Serve static files through the development server (--live)
@app.route('/<any("favicon.ico", "style.css", "img/tablesorter-icons.gif"):filename>')
def favicon_development(filename):
    return send_from_directory('static', filename)


@app.route('/<name>.csv')
def csv_development(name):
    return send_from_directory('out', name + '.csv')


@app.route('/publisher_imgs/<image>.png')
def image_development_publisher(image):
    return send_from_directory('out/publisher_imgs', image + '.png')


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--live", action="store_true",
                        help="Run a development server")
    args = parser.parse_args()
    if args.live:
        app.debug = True
        app.run()
    else:
        from flask_frozen import Freezer
        app.config['FREEZER_DESTINATION'] = 'out'
        app.config['FREEZER_REMOVE_EXTRA_FILES'] = False
        app.debug = False    # Comment to turn off debugging
        app.testing = True   # Comment to turn off debugging
        freezer = Freezer(app)

        @freezer.register_generator
        def url_generator():
            for page_name in basic_page_names:
                yield 'basic_page', {'page_name': page_name}
            for publisher in current_stats['inverted_publisher']['activities'].keys():
                yield 'publisher', {'publisher': publisher}
            for slug in slugs['element']['by_slug']:
                yield 'element', {'slug': slug}
            for major_version, codelist_slugs in slugs['codelist'].items():
                for slug in codelist_slugs['by_slug']:
                    yield 'codelist', {
                        'slug': slug,
                        'major_version': major_version
                    }
            for license in set(licenses.licenses):
                yield 'licenses_individual_license', {'license': license}

        freezer.freeze()

1	# Script to generate static HTML pages
2	# This uses Jinja templating to render the HTML templates in the 'templates' folder
3	# Data is based on the files in the 'stats-calculated' folder, and extra logic in other files in this repository
4
5	import argparse	×
6	import json	×
7	import re	×
8	import subprocess	×
9	from collections import defaultdict	×
10
11	from flask import Flask, render_template, abort, Response, send_from_directory	×
12	import pytz	×
13
14	import licenses	×
15	import timeliness	×
16	import forwardlooking	×
17	import comprehensiveness	×
18	# import coverage
19	import summary_stats	×
20	import humanitarian	×
21	from vars import expected_versions	×
22	import text	×
23	from datetime import datetime	×
24	from dateutil import parser	×
25	from data import (	×
26	ckan,
27	ckan_publishers,
28	codelist_mapping,
29	codelist_sets,
30	codelist_lookup,
31	current_stats,
32	dataset_to_publisher_dict,
33	github_issues,
34	get_publisher_stats,
35	MAJOR_VERSIONS,
36	metadata,
37	publisher_name,
38	publishers_ordered_by_title,
39	is_valid_element,
40	slugs)
41
42	app = Flask(__name__, static_url_path='')	×
43
44
45	def dictinvert(d):	×
46	inv = defaultdict(list)	×
47	for k, v in d.items():	×
48	inv[v].append(k)	×
49	return inv	×
50
51
52	def nested_dictinvert(d):	×
53	inv = defaultdict(lambda: defaultdict(int))	×
54	for k, v in d.items():	×
55	for k2, v2 in v.items():	×
56	inv[k2][k] += v2	×
57	return inv	×
58
59
60	def dataset_to_publisher(dataset_slug):	×
61	""" Converts a dataset (package) slug e.g. dfid-bd to the corresponding publisher
62	slug e.g. dfid """
63	return dataset_to_publisher_dict.get(dataset_slug, '')	×
64
65
66	def firstint(s):	×
67	if s[0].startswith('<'):	×
68	return 0	×
69	m = re.search(r'\d+', s[0])	×
70	return int(m.group(0))	×
71
72
73	def round_nicely(val, ndigits=2):	×
74	""" Round a float, but remove the trailing .0 from integers that python insists on
75	"""
76	if int(val) == float(val):	×
77	return int(val)	×
78	return round(float(val), ndigits)	×
79
80
81	def xpath_to_url(path):	×
82	path = path.strip('./')	×
83	# remove conditions
84	path = re.sub(r'\[[^]]+\]', '', path)	×
85	if path.startswith('iati-activity'):	×
86	url = 'https://reference.codeforiati.org/activity-standard/iati-activities/' + path.split('@')[0]	×
87	elif path.startswith('iati-organisation'):	×
88	url = 'https://reference.codeforiati.org/organisation-standard/iati-organisations/' + path.split('@')[0]	×
89	else:
90	url = 'https://reference.codeforiati.org/activity-standard/iati-activities/iati-activity/' + path.split('@')[0]	×
91	if '@' in path:	×
92	url += '#attributes'	×
93	return url	×
94
95
96	def registration_agency(orgid):	×
97	for code in codelist_sets['2']['OrganisationRegistrationAgency']:	×
98	if orgid.startswith(code):	×
99	return code	×
100
101
102	def get_codelist_values(codelist_values_for_element):	×
103	"""Return a list of unique values present within a one-level nested dictionary.
104	Envisaged usage is to gather the codelist values used by each publisher, as in
105	stats/current/inverted-publisher/codelist_values_by_major_version.json
106	Input: Set of codelist values for a given element (listed by publisher), for example:
107	current_stats['inverted_publisher']['codelist_values_by_major_version']['1']['.//@xml:lang']
108	"""
109	return list(set([y for x in codelist_values_for_element.items() for y in list(x[1].keys())]))	×
110
111
112	# Store data processing times
113	date_time_data_obj = parser.parse(metadata['created_at'])	×
114
115	# Custom Jinja filters
116	app.jinja_env.filters['xpath_to_url'] = xpath_to_url	×
117	app.jinja_env.filters['url_to_filename'] = lambda x: x.rstrip('/').split('/')[-1]	×
118	app.jinja_env.filters['has_future_transactions'] = timeliness.has_future_transactions	×
119	app.jinja_env.filters['round_nicely'] = round_nicely	×
120
121	# Custom Jinja globals
122	app.jinja_env.globals['dataset_to_publisher'] = dataset_to_publisher	×
123	app.jinja_env.globals['url'] = lambda x: '/' if x == 'index.html' else x	×
124	app.jinja_env.globals['datetime_generated'] = lambda: datetime.utcnow().replace(tzinfo=pytz.utc).strftime('%-d %B %Y (at %H:%M %Z)')	×
125	app.jinja_env.globals['datetime_data'] = date_time_data_obj.strftime('%-d %B %Y (at %H:%M %Z)')	×
126	app.jinja_env.globals['commit_hash'] = subprocess.run(	×
127	'git show --format=%H --no-patch'.split(),
128	capture_output=True).stdout.decode().strip()
129	app.jinja_env.globals['stats_commit_hash'] = subprocess.run(	×
130	'git -C stats-calculated show --format=%H --no-patch'.split(),
131	capture_output=True).stdout.decode().strip()
132	app.jinja_env.globals['stats_url'] = 'https://stats.codeforiati.org'	×
133	app.jinja_env.globals['stats_gh_url'] = 'https://github.com/codeforIATI/IATI-Stats-public/tree/' + app.jinja_env.globals['stats_commit_hash']	×
134	app.jinja_env.globals['sorted'] = sorted	×
135	app.jinja_env.globals['enumerate'] = enumerate	×
136	app.jinja_env.globals['top_titles'] = text.top_titles	×
137	app.jinja_env.globals['page_titles'] = text.page_titles	×
138	app.jinja_env.globals['short_page_titles'] = text.short_page_titles	×
139	app.jinja_env.globals['page_leads'] = text.page_leads	×
140	app.jinja_env.globals['page_sub_leads'] = text.page_sub_leads	×
141	app.jinja_env.globals['top_navigation'] = text.top_navigation	×
142	app.jinja_env.globals['navigation'] = text.navigation	×
143	app.jinja_env.globals['navigation_reverse'] = {page: k for k, pages in text.navigation.items() for page in pages}	×
144	app.jinja_env.globals['navigation_reverse'].update({k: k for k in text.navigation})	×
145	app.jinja_env.globals['current_stats'] = current_stats	×
146	app.jinja_env.globals['ckan'] = ckan	×
147	app.jinja_env.globals['ckan_publishers'] = ckan_publishers	×
148	app.jinja_env.globals['github_issues'] = github_issues	×
149	app.jinja_env.globals['publisher_name'] = publisher_name	×
150	app.jinja_env.globals['publishers_ordered_by_title'] = publishers_ordered_by_title	×
151	app.jinja_env.globals['get_publisher_stats'] = get_publisher_stats	×
152	app.jinja_env.globals['set'] = set	×
153	app.jinja_env.globals['firstint'] = firstint	×
154	app.jinja_env.globals['expected_versions'] = expected_versions	×
155	app.jinja_env.globals['current_year'] = datetime.utcnow().year	×
156	# Following variables set in coverage branch but not in master
157	# app.jinja_env.globals['float'] = float
158	# app.jinja_env.globals['dac2012'] = dac2012
159	app.jinja_env.globals['MAJOR_VERSIONS'] = MAJOR_VERSIONS	×
160
161	app.jinja_env.globals['slugs'] = slugs	×
162	app.jinja_env.globals['codelist_mapping'] = codelist_mapping	×
163	app.jinja_env.globals['codelist_sets'] = codelist_sets	×
164	app.jinja_env.globals['codelist_lookup'] = codelist_lookup	×
165	app.jinja_env.globals['get_codelist_values'] = get_codelist_values	×
166	app.jinja_env.globals['is_valid_element'] = is_valid_element	×
167
168	basic_page_names = [	×
169	'headlines',
170	'data_quality',
171	'exploring_data',
172	'publishers',
173	'publishing_stats',
174	'timeliness',
175	'timeliness_timelag',
176	'forwardlooking',
177	'comprehensiveness',
178	'comprehensiveness_core',
179	'comprehensiveness_financials',
180	'comprehensiveness_valueadded',
181	# 'coverage',
182	'summary_stats',
183	'humanitarian',
184	'files',
185	'activities',
186	'download',
187	'xml',
188	'validation',
189	'versions',
190	'organisation',
191	'identifiers',
192	'reporting_orgs',
193	'elements',
194	'codelists',
195	'booleans',
196	'dates',
197	'traceability',
198	'faq',
199	]
200
201
202	@app.route('/<page_name>.html')	×
203	def basic_page(page_name):
204	if page_name in basic_page_names:	×
205	kwargs = {}	×
206	if page_name.startswith('timeliness'):	×
207	kwargs['timeliness'] = timeliness	×
208	parent_page_name = 'timeliness'	×
209	elif page_name.startswith('forwardlooking'):	×
210	kwargs['forwardlooking'] = forwardlooking	×
211	parent_page_name = 'forwardlooking'	×
212	elif page_name.startswith('comprehensiveness'):	×
213	kwargs['comprehensiveness'] = comprehensiveness	×
214	parent_page_name = 'comprehensiveness'	×
215	elif page_name.startswith('coverage'):	×
216	# kwargs['coverage'] = coverage
217	parent_page_name = 'coverage'	×
218	elif page_name.startswith('summary_stats'):	×
219	kwargs['summary_stats'] = summary_stats	×
220	parent_page_name = 'summary_stats'	×
221	elif page_name.startswith('humanitarian'):	×
222	kwargs['humanitarian'] = humanitarian	×
223	parent_page_name = 'humanitarian'	×
224	else:
225	parent_page_name = page_name	×
226	return render_template(page_name + '.html', page=parent_page_name, **kwargs)	×
227	else:
228	abort(404)	×
229
230
231	@app.route('/data/download_errors.json')	×
232	def download_errors_json():
233	return Response(json.dumps(current_stats['download_errors'], indent=2), mimetype='application/json'),	×
234
235
236	@app.route('/')	×
237	def homepage():
238	return render_template('index.html', page='index')	×
239
240
241	app.add_url_rule('/licenses.html', 'licenses', licenses.main)	×
242	app.add_url_rule('/license/<license>.html', 'licenses_individual_license', licenses.individual_license)	×
243
244
245	@app.route('/publisher/<publisher>.html')	×
246	def publisher(publisher):
247	publisher_stats = get_publisher_stats(publisher)	×
248	budget_table = [{	×
249	'year': 'Total',
250	'count_total': sum(sum(x.values()) for x in publisher_stats['count_budgets_by_type_by_year'].values()),
251	'sum_total': {currency: sum(sums.values()) for by_currency in publisher_stats['sum_budgets_by_type_by_year'].values() for currency, sums in by_currency.items()},
252	'count_original': sum(publisher_stats['count_budgets_by_type_by_year']['1'].values()) if '1' in publisher_stats['count_budgets_by_type_by_year'] else None,
253	'sum_original': {k: sum(v.values()) for k, v in publisher_stats['sum_budgets_by_type_by_year']['1'].items()} if '1' in publisher_stats['sum_budgets_by_type_by_year'] else None,
254	'count_revised': sum(publisher_stats['count_budgets_by_type_by_year']['2'].values()) if '2' in publisher_stats['count_budgets_by_type_by_year'] else None,
255	'sum_revised': {k: sum(v.values()) for k, v in publisher_stats['sum_budgets_by_type_by_year']['2'].items()} if '2' in publisher_stats['sum_budgets_by_type_by_year'] else None
256	}] + [{'year': year,
257	'count_total': sum(x[year] for x in publisher_stats['count_budgets_by_type_by_year'].values() if year in x),
258	'sum_total': {currency: sums.get(year) for by_currency in publisher_stats['sum_budgets_by_type_by_year'].values() for currency, sums in by_currency.items()},
259	'count_original': publisher_stats['count_budgets_by_type_by_year']['1'].get(year) if '1' in publisher_stats['count_budgets_by_type_by_year'] else None,
260	'sum_original': {k: v.get(year) for k, v in publisher_stats['sum_budgets_by_type_by_year']['1'].items()} if '1' in publisher_stats['sum_budgets_by_type_by_year'] else None,
261	'count_revised': publisher_stats['count_budgets_by_type_by_year']['2'].get(year) if '2' in publisher_stats['count_budgets_by_type_by_year'] else None,
262	'sum_revised': {k: v.get(year) for k, v in publisher_stats['sum_budgets_by_type_by_year']['2'].items()} if '2' in publisher_stats['sum_budgets_by_type_by_year'] else None
263	} for year in sorted(set(sum((list(x.keys()) for x in publisher_stats['count_budgets_by_type_by_year'].values()), [])))
264	]
265	failure_count = len(current_stats['inverted_file_publisher'][publisher]['validation'].get('fail', {}))	×
266	return render_template('publisher.html',	×
267	url=lambda x: '../' + x,
268	publisher=publisher,
269	publisher_stats=publisher_stats,
270	failure_count=failure_count,
271	publisher_inverted=get_publisher_stats(publisher, 'inverted-file'),
272	publisher_licenses=licenses.licenses_for_publisher(publisher),
273	budget_table=budget_table,)
274
275
276	@app.route('/codelist/<major_version>/<slug>.html')	×
277	def codelist(major_version, slug):
278	i = slugs['codelist'][major_version]['by_slug'][slug]	×
279	element = list(current_stats['inverted_publisher']['codelist_values_by_major_version'][major_version])[i]	×
280	values = nested_dictinvert(list(current_stats['inverted_publisher']['codelist_values_by_major_version'][major_version].values())[i])	×
281	return render_template('codelist.html',	×
282	element=element,
283	values=values,
284	reverse_codelist_mapping={major_version: dictinvert(mapping) for major_version, mapping in codelist_mapping.items()},
285	url=lambda x: '../../' + x,
286	major_version=major_version,
287	page='codelists')
288
289
290	@app.route('/element/<slug>.html')	×
291	def element(slug):
292	i = slugs['element']['by_slug'][slug]	×
293	element = list(current_stats['inverted_publisher']['elements'])[i]	×
294	publishers = list(current_stats['inverted_publisher']['elements'].values())[i]	×
295	return render_template('element.html',	×
296	element=element,
297	publishers=publishers,
298	url=lambda x: '../' + x,
299	element_or_attribute='attribute' if '@' in element else 'element',
300	page='elements')
301
302
303	@app.route('/registration_agencies.html')	×
304	def registration_agencies():
305	registration_agencies = defaultdict(int)	×
306	registration_agencies_publishers = defaultdict(list)	×
307	nonmatching = []	×
308	for orgid, publishers in current_stats['inverted_publisher']['reporting_orgs'].items():	×
309	reg_ag = registration_agency(orgid)	×
310	if reg_ag:	×
311	registration_agencies[reg_ag] += 1	×
312	registration_agencies_publishers[reg_ag] += list(publishers)	×
313	else:
314	nonmatching.append((orgid, publishers))	×
315	return render_template('registration_agencies.html',	×
316	page='registration_agencies',
317	registration_agencies=registration_agencies,
318	registration_agencies_publishers=registration_agencies_publishers,
319	nonmatching=nonmatching)
320
321
322	# Serve static files through the development server (--live)
323	@app.route('/<any("favicon.ico", "style.css", "img/tablesorter-icons.gif"):filename>')	×
324	def favicon_development(filename):
325	return send_from_directory('static', filename)	×
326
327
328	@app.route('/<name>.csv')	×
329	def csv_development(name):
330	return send_from_directory('out', name + '.csv')	×
331
332
333	@app.route('/publisher_imgs/<image>.png')	×
334	def image_development_publisher(image):
335	return send_from_directory('out/publisher_imgs', image + '.png')	×
336
337
338	if __name__ == '__main__':	×
339	parser = argparse.ArgumentParser()	×
340	parser.add_argument("--live", action="store_true",	×
341	help="Run a development server")
342	args = parser.parse_args()	×
343	if args.live:	×
344	app.debug = True	×
345	app.run()	×
346	else:
347	from flask_frozen import Freezer	×
348	app.config['FREEZER_DESTINATION'] = 'out'	×
349	app.config['FREEZER_REMOVE_EXTRA_FILES'] = False	×
350	app.debug = False # Comment to turn off debugging	×
351	app.testing = True # Comment to turn off debugging	×
352	freezer = Freezer(app)	×
353
354	@freezer.register_generator	×
355	def url_generator():
356	for page_name in basic_page_names:	×
357	yield 'basic_page', {'page_name': page_name}	×
358	for publisher in current_stats['inverted_publisher']['activities'].keys():	×
359	yield 'publisher', {'publisher': publisher}	×
360	for slug in slugs['element']['by_slug']:	×
361	yield 'element', {'slug': slug}	×
362	for major_version, codelist_slugs in slugs['codelist'].items():	×
363	for slug in codelist_slugs['by_slug']:	×
364	yield 'codelist', {	×
365	'slug': slug,
366	'major_version': major_version
367	}
368	for license in set(licenses.licenses):	×
369	yield 'licenses_individual_license', {'license': license}	×
370
371	freezer.freeze()	×

codeforIATI / analytics / 3618006776

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous