9189630163

Committed 22 May 2024 10:26AM UTC coverage: 94.311%. Remained the same

Build # 9189630163

Build Type

Pull #121

github

Committed by

web-flow

Commit Message

Merge 389cf0ae7 into fcdb3ce6a

Pull Request Pull Request #121: ⬆️ Bump jinja2 from 3.1.2 to 3.1.4

Run Details

2984 of 3164 relevant lines covered (94.31%)

0.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.78

/src/data/import_breeds.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Apr 19 16:25:15 2021

@author: Paolo Cozzi <paolo.cozzi@ibba.cnr.it>

Import breeds from an excel template in a smarter dataset
"""

import click
import logging
from pathlib import Path

from mongoengine.errors import NotUniqueError

from src.features.smarterdb import (
    global_connection, get_or_create_breed, SmarterDBException,
    BreedAlias)
from src.data.common import deal_with_datasets, pandas_open

logger = logging.getLogger(__name__)


@click.command()
@click.option(
    '--species_class',
    type=click.Choice(
        ['Sheep', 'Goat'],
        case_sensitive=False),
    required=True,
    help="The generic species of this breed (Sheep or Goat)"
    )
@click.option(
    '--src_dataset', type=str, required=True,
    help="The raw dataset file name (zip archive) in which search datafile"
)
@click.option(
    '--dst_dataset', type=str, required=False,
    help=("The raw dataset file name (zip archive) in which define breeds "
          "(def. the 'src_dataset')")
)
@click.option(
    '--datafile',
    type=str,
    required=True,
    help="The metadata file in which search for information")
@click.option(
    '--code_column',
    type=str,
    default="code",
    help="The name of the breed code column in metadata table")
@click.option(
    '--breed_column',
    type=str,
    default="breed",
    help="The name of the breed column in metadata table")
@click.option(
    '--fid_column',
    type=str,
    help="The name of the FID column used in genotype file")
@click.option(
    '--country_column',
    type=str,
    help="The name of the country column in metadata table")
def main(species_class, src_dataset, dst_dataset, datafile, code_column,
         breed_column, fid_column, country_column):
    """Import breeds from metadata file into SMARTER-database"""

    logger.info(f"{Path(__file__).name} started")

    src_dataset, dst_dataset, datapath = deal_with_datasets(
        src_dataset, dst_dataset, datafile)

    # read breed into data
    data = pandas_open(datapath)

    for index, row in data.iterrows():
        logger.debug(row)

        code = row.get(code_column)
        name = row.get(breed_column)

        # by default, fid is equal to code
        if not fid_column:
            fid_column = code_column

        fid = str(row.get(fid_column))

        logger.debug(
            f"Got code: '{code}', breed_name: '{name}', "
            f"fid: '{fid}'"
        )

        # deal with multi countries dataset
        country = None

        if country_column:
            country = row.get(country_column)

        # need to define also an alias in order to retrieve such breed when
        # dealing with original file
        alias = BreedAlias(fid=fid, dataset=dst_dataset, country=country)

        try:
            breed, modified = get_or_create_breed(
                species_class=species_class.capitalize(),
                name=name,
                code=code,
                aliases=[alias])

            if modified:
                logger.info(f"{breed} added to database")

        except NotUniqueError as e:
            logger.error(e)
            raise SmarterDBException(
                f"Got an error while inserting '{name}'. '{code}'")

    logger.info(f"{Path(__file__).name} ended")


if __name__ == '__main__':
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    # connect to database
    global_connection()

    main()

1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	"""	1✔
4	Created on Mon Apr 19 16:25:15 2021
5
6	@author: Paolo Cozzi <paolo.cozzi@ibba.cnr.it>
7
8	Import breeds from an excel template in a smarter dataset
9	"""
10
11	import click	1✔
12	import logging	1✔
13	from pathlib import Path	1✔
14
15	from mongoengine.errors import NotUniqueError	1✔
16
17	from src.features.smarterdb import (	1✔
18	global_connection, get_or_create_breed, SmarterDBException,
19	BreedAlias)
20	from src.data.common import deal_with_datasets, pandas_open	1✔
21
22	logger = logging.getLogger(__name__)	1✔
23
24
25	@click.command()	1✔
26	@click.option(	1✔
27	'--species_class',
28	type=click.Choice(
29	['Sheep', 'Goat'],
30	case_sensitive=False),
31	required=True,
32	help="The generic species of this breed (Sheep or Goat)"
33	)
34	@click.option(	1✔
35	'--src_dataset', type=str, required=True,
36	help="The raw dataset file name (zip archive) in which search datafile"
37	)
38	@click.option(	1✔
39	'--dst_dataset', type=str, required=False,
40	help=("The raw dataset file name (zip archive) in which define breeds "
41	"(def. the 'src_dataset')")
42	)
43	@click.option(	1✔
44	'--datafile',
45	type=str,
46	required=True,
47	help="The metadata file in which search for information")
48	@click.option(	1✔
49	'--code_column',
50	type=str,
51	default="code",
52	help="The name of the breed code column in metadata table")
53	@click.option(	1✔
54	'--breed_column',
55	type=str,
56	default="breed",
57	help="The name of the breed column in metadata table")
58	@click.option(	1✔
59	'--fid_column',
60	type=str,
61	help="The name of the FID column used in genotype file")
62	@click.option(	1✔
63	'--country_column',
64	type=str,
65	help="The name of the country column in metadata table")
66	def main(species_class, src_dataset, dst_dataset, datafile, code_column,	1✔
67	breed_column, fid_column, country_column):
68	"""Import breeds from metadata file into SMARTER-database"""
69
70	logger.info(f"{Path(__file__).name} started")	1✔
71
72	src_dataset, dst_dataset, datapath = deal_with_datasets(	1✔
73	src_dataset, dst_dataset, datafile)
74
75	# read breed into data
76	data = pandas_open(datapath)	1✔
77
78	for index, row in data.iterrows():	1✔
79	logger.debug(row)	1✔
80
81	code = row.get(code_column)	1✔
82	name = row.get(breed_column)	1✔
83
84	# by default, fid is equal to code
85	if not fid_column:	1✔
86	fid_column = code_column	1✔
87
88	fid = str(row.get(fid_column))	1✔
89
90	logger.debug(	1✔
91	f"Got code: '{code}', breed_name: '{name}', "
92	f"fid: '{fid}'"
93	)
94
95	# deal with multi countries dataset
96	country = None	1✔
97
98	if country_column:	1✔
99	country = row.get(country_column)	1✔
100
101	# need to define also an alias in order to retrieve such breed when
102	# dealing with original file
103	alias = BreedAlias(fid=fid, dataset=dst_dataset, country=country)	1✔
104
105	try:	1✔
106	breed, modified = get_or_create_breed(	1✔
107	species_class=species_class.capitalize(),
108	name=name,
109	code=code,
110	aliases=[alias])
111
112	if modified:	1✔
113	logger.info(f"{breed} added to database")	1✔
114
115	except NotUniqueError as e:	×
116	logger.error(e)	×
117	raise SmarterDBException(	×
118	f"Got an error while inserting '{name}'. '{code}'")
119
120	logger.info(f"{Path(__file__).name} ended")	1✔
121
122
123	if __name__ == '__main__':	1✔
124	log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'	×
125	logging.basicConfig(level=logging.INFO, format=log_fmt)	×
126
127	# connect to database
128	global_connection()	×
129
130	main()	×

cnr-ibba / SMARTER-database / 9189630163

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous