5374609890

pending completion

Build # 5374609890

Build Type

Pull #25

github

Committed by

web-flow

Commit Message

Merge 3b565d94a into f4b06c86d

Pull Request Pull Request #25: New: React migration

Run Details

968 of 1777 branches covered (54.47%)

Branch coverage included in aggregate %.

15 of 26 new or added lines in 3 files covered. (57.69%)

40 existing lines in 1 file now uncovered.

1702 of 3516 relevant lines covered (48.41%)

0.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

12.3

/tomatic/retriever.py

#!/usr/bin/env python

import requests
import datetime
from pathlib import Path
from consolemsg import step, out, warn, fail, u
from yamlns import namespace as ns
from somutils.sheetfetcher import SheetFetcher
from .persons import persons

# Dirty Hack: Behave like python3 open regarding unicode
def open(*args, **kwd):
    import io
    return io.open(encoding='utf8', *args, **kwd)

def transliterate(word):
    word=u(word).lower().strip()
    for old, new in zip(
        u'àèìòùáéíóúçñ',
        u'aeiouaeioucn',
    ) :
        word = word.replace(old,new)
    return word

def addDays(date, ndays):
    return date + datetime.timedelta(days=ndays)


class Notoi(object):
    'Abstracts Notoi API'
    login_ep = '/login/'
    token_head='JWT '

    def __init__(self, service_url, user, password):
        self.service_url = service_url
        self.token = self.login(user, password)

    def _url(self, ep):
        return self.service_url + ep

    def login(self, username, password):
        login = requests.post(
            self._url(Notoi.login_ep),
            data=dict(
                username = username,
                password = password,
            ),
            verify=False,
        )
        return login.json()['token']

    def absences(self, firstDate, lastDate):
        return self._pagedGet('/absencies/absences',
            start_period = firstDate,
            end_period = lastDate,
        )

    def persons(self):
        return self._pagedGet('/absencies/workers')

    def _pagedGet(self, endpoint, **params):
        url = self._url(endpoint)
        result = []
        while url:
            response = requests.get(
                url,
                params=params,
                headers={
                    'Authorization': Notoi.token_head + self.token,
                },
                verify=False
            )
            params = None
            url = response.json()['next']
            result.extend(response.json()['results'])
        return result


def downloadVacations_notoi(config):
    step("Baixant vacances del gestor d'absencies...")

    import dbconfig
    notoiApi = Notoi(**dbconfig.tomatic.notoi_data)

    firstDay = addDays(config.monday, -1)
    lastDay = addDays(config.monday, +5)
    absences = notoiApi.absences(firstDay, lastDay)
    notoipersons = [ns(p) for p in notoiApi.persons()]
    email2tomatic = {
        email: id
        for id, email in persons().emails.items()
    }
    notoi2tomatic = {
        p.id: email2tomatic[p.email]
        for p in notoipersons
    }

    step("  Guardant indisponibilitats per vacances a 'indisponibilitats-vacances.conf'...")
    translate_days = ['dl', 'dm', 'dx', 'dj', 'dv']

    def dateFromIso(isoString):
        return datetime.datetime.strptime(
                isoString,
                '%Y-%m-%dT%H:%M:%S'
            ).date()

    with open('indisponibilitats-vacances.conf', 'w') as vacationsfile:
        for absence in absences:
            name = notoi2tomatic.get(absence['worker'])
            start = dateFromIso(absence['start_time'])
            end = dateFromIso(absence['end_time'])
            days = [
                translate_days[weekday]
                for weekday in range(5)
                if start <= addDays(config.monday, weekday) <= end
            ]
            for day in days:
                out("+{} {} # vacances", name, day)
                vacationsfile.write("+{} {} # vacances\n".format(name, day))

def downloadVacations_odoo(config):
    step("Baixant vacances de l'odoo...")

    import dbconfig
    import erppeek
    erp = erppeek.Client(**dbconfig.tomatic.holidaysodoo)
    firstDay = addDays(config.monday, 0)
    lastDay = addDays(config.monday, 4)
    absences = erp.model('hr.leave').get_leaves(
        firstDay.strftime("%Y-%m-%d"), lastDay.strftime("%Y-%m-%d")
    )

    def dateFromIso(isoString):
        return datetime.datetime.strptime(
                isoString,
                '%Y-%m-%d %H:%M:%S'
            ).date()
    email2tomatic = {
        email: id
        for id, email in persons().emails.items()
    }

    step("  Guardant indisponibilitats per vacances a 'indisponibilitats-vacances.conf'...")
    weekdays = ['dl', 'dm', 'dx', 'dj', 'dv']
    days = [addDays(config.monday, i) for i in range(5)]
    with open('indisponibilitats-vacances.conf', 'w') as holidaysfile:
        for absence in absences:
            name = email2tomatic.get(absence['worker'])
            if name is None:
                warn("Ignorant les vacances de {} que no està al Tomàtic",
                    absence['worker'])
                continue

            start = dateFromIso(absence['start_time'])
            end = dateFromIso(absence['end_time'])
            for weekday, day in zip(weekdays, days):
                if start <= day <= end:
                    out("+{} {} # vacances", name, weekday)
                    holidaysfile.write("+{} {} # vacances\n".format(name, weekday))



def downloadVacations_drive(config):
    step('Autentificant al Google Drive')
    fetcher = SheetFetcher(
        documentName=config.documentDrive,
        credentialFilename=config.driveCertificate,
    )

    step('Baixant vacances del drive...')

    nextFriday = addDays(config.monday, 4)
    mondayYear = config.monday.year
    startingSemester = 1 if config.monday < datetime.date(mondayYear,7,1) else 2

    # HACK: Activate this flag until we have holidays spreadsheet for the new year
    if config.get("newYearHack",False):
        mondayYear-=1
        startingSemester = 2

    semesterFirsMonth = 1 if startingSemester==1 else 7
    semesterFirstDay = datetime.date(mondayYear, semesterFirsMonth, 1)
    startingOffset = (config.monday - semesterFirstDay).days

    holidays2SRange = 'Vacances{}Semestre{}'.format(
        mondayYear,
        startingSemester,
        )
    step("  Baixant vacances de l'interval {}".format(holidays2SRange))
    holidays2S = fetcher.get_range(u(mondayYear), holidays2SRange)

    # TODO: Compose from two semesters (won't happen till 2018 Jan)
#    endingSemester = 1 if nextFriday < datetime.date(mondayYear,7,1) else 2
#    if startingSemester == endingSemester :
    who = [row[0] for row in holidays2S ]
    holidays = [
        (transliterate(name), [
            day for day, value in zip(
                ['dl','dm','dx','dj','dv'],
                row[startingOffset+1:startingOffset+6]
                )
            if value.strip()
            ])
        for name, row in zip(who, holidays2S)
        ]
    step("  Guardant indisponibilitats per vacances a 'indisponibilitats-vacances.conf'...")
    with open('indisponibilitats-vacances.conf','w') as holidaysfile:
        for name, days in holidays:
            for day in days:
                holidaysfile.write("+{} {} # vacances\n".format(name, day))


def downloadVacations(config, source=None):
    if source=='notoi':
        return downloadVacations_notoi(config)
    if source=='odoo':
        return downloadVacations_odoo(config)
    if source=='drive':
        return downloadVacations_drive(config)

    fail("Bad source for vacations: {}".format(source))

def downloadFestivities(config):
    step("Baixant festivitats de l'odoo...")
    intervals=dict(
        workweek=4, # This is what is needed just for schedulings
        year=366, # This is useful for many other uses, not yet slower
    )

    import dbconfig
    import erppeek
    from yamlns.dateutils import Date
    erp = erppeek.Client(**dbconfig.tomatic.holidaysodoo)
    firstDay = addDays(config.monday, 0)
    lastDay = addDays(config.monday, intervals['year'])
    Festivities = erp.model('hr.holidays.public.line')
    festivity_ids = Festivities.search([
        ('date', '>=', str(firstDay)),
        ('date', '<=', str(lastDay)),
    ])
    holidaysfile = Path('holidays.conf')
    with holidaysfile.open('w', encoding='utf8') as output:
        for festivity in Festivities.read(festivity_ids, ['date','meeting_id']) or []:
            output.write('{date}\t{meeting_id[1]}\n'.format(**festivity))

def downloadIdealLoad(config):
    step('Autentificant al Google Drive')
    fetcher = SheetFetcher(
        documentName=config.documentDrive,
        credentialFilename=config.driveCertificate,
        )

    step('Baixant carrega setmanal...')

    step("  Descarregant el rang '{}'...", config.idealLoadNamesRange)
    names = fetcher.get_range(
        config.fullCarregaIdeal, config.idealLoadNamesRange)
    step("  Descarregant el rang '{}'...", config.idealLoadValuesRange)
    values = fetcher.get_range(
        config.fullCarregaIdeal, config.idealLoadValuesRange)
    step("  Guardant-ho com '{}'...".format(config.idealshifts))
    carregaIdeal = ns(
        (transliterate(name[0]), int(value[0]))
        for name, value in zip(names,values))
    carregaIdeal.dump(config.idealshifts)

def downloadLeaves(config):
    step('Autentificant al Google Drive')
    fetcher = SheetFetcher(
        documentName=config.documentDrive,
        credentialFilename=config.driveCertificate,
        )
    leavesSheet = config.get('leavesSheet',"Baixes")
    leavesFile = "leaves.conf"

    step('Baixant baixes...')

    step("  Descarregant fulla '{}'...", leavesSheet)
    leaves = fetcher.get_fullsheet(leavesSheet)
    leaves = u'\n'.join([person for line in leaves for person in line])
    step("  Guardant-ho com '{}'...".format(leavesFile))
    Path(leavesFile).write_text(leaves, encoding='utf8')

def downloadPersons(config):
    step("Baixant informació de les persones del tomatic...")
    url = config.baseUrl + '/api/persons'
    r = requests.get(url)
    r.raise_for_status()
    from yamlns import namespace as ns
    persons = ns.loads(r.content)
    persons.persons.dump(config.personsfile)


def downloadBusy(config):
    step("Baixant indisponibilitats del tomatic...")

    baseUrl = config.baseUrl + '/api/busy/download/'
    for name, filename in [
            ('weekly', 'indisponibilitats.conf'),
            ('oneshot', 'oneshot.conf'),
        ]:
        url = baseUrl + name
        step("  Baixant {} from {}", filename, url)
        r = requests.get(url)
        r.raise_for_status()
        Path(filename).write_bytes(r.content)

def downloadShiftCredit(config):
    step("Baixant crèdit de torns del tomatic...")
    pastMonday = addDays(config.monday,-7)
    url = config.baseUrl + '/api/shifts/download/credit/{}'.format(pastMonday)
    filename='shiftcredit.yaml'
    step("  Baixant {} from {}", filename, url)
    r = requests.get(url)
    r.raise_for_status()
    Path(filename).write_bytes(r.content)


# vim: et ts=4 sw=4

1	#!/usr/bin/env python
2
3	import requests	1✔
4	import datetime	1✔
5	from pathlib import Path	1✔
6	from consolemsg import step, out, warn, fail, u	1✔
7	from yamlns import namespace as ns	1✔
8	from somutils.sheetfetcher import SheetFetcher	1✔
9	from .persons import persons	1✔
10
11	# Dirty Hack: Behave like python3 open regarding unicode
12	def open(args, *kwd):	1✔
13	import io	×
14	return io.open(encoding='utf8', args, *kwd)	×
15
16	def transliterate(word):	1✔
17	word=u(word).lower().strip()	×
18	for old, new in zip(	×
19	u'àèìòùáéíóúçñ',
20	u'aeiouaeioucn',
21	) :
22	word = word.replace(old,new)	×
23	return word	×
24
25	def addDays(date, ndays):	1✔
26	return date + datetime.timedelta(days=ndays)	1✔
27
28
29	class Notoi(object):	1✔
30	'Abstracts Notoi API'
31	login_ep = '/login/'	1✔
32	token_head='JWT '	1✔
33
34	def __init__(self, service_url, user, password):	1✔
35	self.service_url = service_url	×
36	self.token = self.login(user, password)	×
37
38	def _url(self, ep):	1✔
39	return self.service_url + ep	×
40
41	def login(self, username, password):	1✔
42	login = requests.post(	×
43	self._url(Notoi.login_ep),
44	data=dict(
45	username = username,
46	password = password,
47	),
48	verify=False,
49	)
50	return login.json()['token']	×
51
52	def absences(self, firstDate, lastDate):	1✔
53	return self._pagedGet('/absencies/absences',	×
54	start_period = firstDate,
55	end_period = lastDate,
56	)
57
58	def persons(self):	1✔
59	return self._pagedGet('/absencies/workers')	×
60
61	def _pagedGet(self, endpoint, **params):	1✔
62	url = self._url(endpoint)	×
63	result = []	×
64	while url:	×
65	response = requests.get(	×
66	url,
67	params=params,
68	headers={
69	'Authorization': Notoi.token_head + self.token,
70	},
71	verify=False
72	)
73	params = None	×
74	url = response.json()['next']	×
75	result.extend(response.json()['results'])	×
76	return result	×
77
78
79	def downloadVacations_notoi(config):	1✔
80	step("Baixant vacances del gestor d'absencies...")	×
81
82	import dbconfig	×
83	notoiApi = Notoi(**dbconfig.tomatic.notoi_data)	×
84
85	firstDay = addDays(config.monday, -1)	×
86	lastDay = addDays(config.monday, +5)	×
87	absences = notoiApi.absences(firstDay, lastDay)	×
88	notoipersons = [ns(p) for p in notoiApi.persons()]	×
89	email2tomatic = {	×
90	email: id
91	for id, email in persons().emails.items()
92	}
93	notoi2tomatic = {	×
94	p.id: email2tomatic[p.email]
95	for p in notoipersons
96	}
97
98	step(" Guardant indisponibilitats per vacances a 'indisponibilitats-vacances.conf'...")	×
99	translate_days = ['dl', 'dm', 'dx', 'dj', 'dv']	×
100
101	def dateFromIso(isoString):	×
102	return datetime.datetime.strptime(	×
103	isoString,
104	'%Y-%m-%dT%H:%M:%S'
105	).date()
106
107	with open('indisponibilitats-vacances.conf', 'w') as vacationsfile:	×
108	for absence in absences:	×
109	name = notoi2tomatic.get(absence['worker'])	×
110	start = dateFromIso(absence['start_time'])	×
111	end = dateFromIso(absence['end_time'])	×
112	days = [	×
113	translate_days[weekday]
114	for weekday in range(5)
115	if start <= addDays(config.monday, weekday) <= end
116	]
117	for day in days:	×
118	out("+{} {} # vacances", name, day)	×
119	vacationsfile.write("+{} {} # vacances\n".format(name, day))	×
120
121	def downloadVacations_odoo(config):	1✔
122	step("Baixant vacances de l'odoo...")	×
123
124	import dbconfig	×
125	import erppeek	×
126	erp = erppeek.Client(**dbconfig.tomatic.holidaysodoo)	×
127	firstDay = addDays(config.monday, 0)	×
128	lastDay = addDays(config.monday, 4)	×
129	absences = erp.model('hr.leave').get_leaves(	×
130	firstDay.strftime("%Y-%m-%d"), lastDay.strftime("%Y-%m-%d")
131	)
132
133	def dateFromIso(isoString):	×
134	return datetime.datetime.strptime(	×
135	isoString,
136	'%Y-%m-%d %H:%M:%S'
137	).date()
138	email2tomatic = {	×
139	email: id
140	for id, email in persons().emails.items()
141	}
142
143	step(" Guardant indisponibilitats per vacances a 'indisponibilitats-vacances.conf'...")	×
144	weekdays = ['dl', 'dm', 'dx', 'dj', 'dv']	×
145	days = [addDays(config.monday, i) for i in range(5)]	×
NEW 146	with open('indisponibilitats-vacances.conf', 'w') as holidaysfile:	×
147	for absence in absences:	×
148	name = email2tomatic.get(absence['worker'])	×
NEW 149	if name is None:	×
NEW 150	warn("Ignorant les vacances de {} que no està al Tomàtic",	×
151	absence['worker'])
NEW 152	continue	×
153
NEW 154	start = dateFromIso(absence['start_time'])	×
NEW 155	end = dateFromIso(absence['end_time'])	×
NEW 156	for weekday, day in zip(weekdays, days):	×
NEW 157	if start <= day <= end:	×
NEW 158	out("+{} {} # vacances", name, weekday)	×
UNCOV 159	holidaysfile.write("+{} {} # vacances\n".format(name, weekday))	×
160
161
162
163	def downloadVacations_drive(config):	1✔
164	step('Autentificant al Google Drive')	×
NEW 165	fetcher = SheetFetcher(	×
166	documentName=config.documentDrive,
167	credentialFilename=config.driveCertificate,
168	)
169
UNCOV 170	step('Baixant vacances del drive...')	×
171
172	nextFriday = addDays(config.monday, 4)	×
173	mondayYear = config.monday.year	×
UNCOV 174	startingSemester = 1 if config.monday < datetime.date(mondayYear,7,1) else 2	×
175
176	# HACK: Activate this flag until we have holidays spreadsheet for the new year
UNCOV 177	if config.get("newYearHack",False):	×
178	mondayYear-=1	×
UNCOV 179	startingSemester = 2	×
180
181	semesterFirsMonth = 1 if startingSemester==1 else 7	×
182	semesterFirstDay = datetime.date(mondayYear, semesterFirsMonth, 1)	×
UNCOV 183	startingOffset = (config.monday - semesterFirstDay).days	×
184
185	holidays2SRange = 'Vacances{}Semestre{}'.format(	×
186	mondayYear,
187	startingSemester,
188	)
189	step(" Baixant vacances de l'interval {}".format(holidays2SRange))	×
190	holidays2S = fetcher.get_range(u(mondayYear), holidays2SRange)	×
191
192	# TODO: Compose from two semesters (won't happen till 2018 Jan)
193	# endingSemester = 1 if nextFriday < datetime.date(mondayYear,7,1) else 2
194	# if startingSemester == endingSemester :
UNCOV 195	who = [row[0] for row in holidays2S ]	×
UNCOV 196	holidays = [	×
197	(transliterate(name), [
198	day for day, value in zip(
199	['dl','dm','dx','dj','dv'],
200	row[startingOffset+1:startingOffset+6]
201	)
202	if value.strip()
203	])
204	for name, row in zip(who, holidays2S)
205	]
UNCOV 206	step(" Guardant indisponibilitats per vacances a 'indisponibilitats-vacances.conf'...")	×
UNCOV 207	with open('indisponibilitats-vacances.conf','w') as holidaysfile:	×
UNCOV 208	for name, days in holidays:	×
UNCOV 209	for day in days:	×
UNCOV 210	holidaysfile.write("+{} {} # vacances\n".format(name, day))	×
211
212
213	def downloadVacations(config, source=None):	1✔
214	if source=='notoi':	×
215	return downloadVacations_notoi(config)	×
216	if source=='odoo':	×
217	return downloadVacations_odoo(config)	×
218	if source=='drive':	×
UNCOV 219	return downloadVacations_drive(config)	×
220
UNCOV 221	fail("Bad source for vacations: {}".format(source))	×
222
223	def downloadFestivities(config):	1✔
224	step("Baixant festivitats de l'odoo...")	×
225	intervals=dict(	×
226	workweek=4, # This is what is needed just for schedulings
227	year=366, # This is useful for many other uses, not yet slower
228	)
229
UNCOV 230	import dbconfig	×
UNCOV 231	import erppeek	×
232	from yamlns.dateutils import Date	×
233	erp = erppeek.Client(**dbconfig.tomatic.holidaysodoo)	×
UNCOV 234	firstDay = addDays(config.monday, 0)	×
UNCOV 235	lastDay = addDays(config.monday, intervals['year'])	×
UNCOV 236	Festivities = erp.model('hr.holidays.public.line')	×
UNCOV 237	festivity_ids = Festivities.search([	×
238	('date', '>=', str(firstDay)),
239	('date', '<=', str(lastDay)),
240	])
241	holidaysfile = Path('holidays.conf')	×
242	with holidaysfile.open('w', encoding='utf8') as output:	×
243	for festivity in Festivities.read(festivity_ids, ['date','meeting_id']) or []:	×
244	output.write('{date}\t{meeting_id[1]}\n'.format(**festivity))	×
245
246	def downloadIdealLoad(config):	1✔
UNCOV 247	step('Autentificant al Google Drive')	×
UNCOV 248	fetcher = SheetFetcher(	×
249	documentName=config.documentDrive,
250	credentialFilename=config.driveCertificate,
251	)
252
UNCOV 253	step('Baixant carrega setmanal...')	×
254
255	step(" Descarregant el rang '{}'...", config.idealLoadNamesRange)	×
256	names = fetcher.get_range(	×
257	config.fullCarregaIdeal, config.idealLoadNamesRange)
UNCOV 258	step(" Descarregant el rang '{}'...", config.idealLoadValuesRange)	×
UNCOV 259	values = fetcher.get_range(	×
260	config.fullCarregaIdeal, config.idealLoadValuesRange)
261	step(" Guardant-ho com '{}'...".format(config.idealshifts))	×
UNCOV 262	carregaIdeal = ns(	×
263	(transliterate(name[0]), int(value[0]))
264	for name, value in zip(names,values))
UNCOV 265	carregaIdeal.dump(config.idealshifts)	×
266
267	def downloadLeaves(config):	1✔
UNCOV 268	step('Autentificant al Google Drive')	×
269	fetcher = SheetFetcher(	×
270	documentName=config.documentDrive,
271	credentialFilename=config.driveCertificate,
272	)
273	leavesSheet = config.get('leavesSheet',"Baixes")	×
UNCOV 274	leavesFile = "leaves.conf"	×
275
276	step('Baixant baixes...')	×
277
UNCOV 278	step(" Descarregant fulla '{}'...", leavesSheet)	×
UNCOV 279	leaves = fetcher.get_fullsheet(leavesSheet)	×
UNCOV 280	leaves = u'\n'.join([person for line in leaves for person in line])	×
281	step(" Guardant-ho com '{}'...".format(leavesFile))	×
282	Path(leavesFile).write_text(leaves, encoding='utf8')	×
283
284	def downloadPersons(config):	1✔
UNCOV 285	step("Baixant informació de les persones del tomatic...")	×
286	url = config.baseUrl + '/api/persons'	×
287	r = requests.get(url)	×
288	r.raise_for_status()	×
289	from yamlns import namespace as ns	×
290	persons = ns.loads(r.content)	×
UNCOV 291	persons.persons.dump(config.personsfile)	×
292
293
294	def downloadBusy(config):	1✔
295	step("Baixant indisponibilitats del tomatic...")	×
296
297	baseUrl = config.baseUrl + '/api/busy/download/'	×
298	for name, filename in [	×
299	('weekly', 'indisponibilitats.conf'),
300	('oneshot', 'oneshot.conf'),
301	]:
UNCOV 302	url = baseUrl + name	×
303	step(" Baixant {} from {}", filename, url)	×
UNCOV 304	r = requests.get(url)	×
305	r.raise_for_status()	×
306	Path(filename).write_bytes(r.content)	×
307
308	def downloadShiftCredit(config):	1✔
UNCOV 309	step("Baixant crèdit de torns del tomatic...")	×
310	pastMonday = addDays(config.monday,-7)	×
311	url = config.baseUrl + '/api/shifts/download/credit/{}'.format(pastMonday)	×
312	filename='shiftcredit.yaml'	×
313	step(" Baixant {} from {}", filename, url)	×
314	r = requests.get(url)	×
UNCOV 315	r.raise_for_status()	×
UNCOV 316	Path(filename).write_bytes(r.content)	×
317
318
319	# vim: et ts=4 sw=4

Som-Energia / somenergia-tomatic / 5374609890

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous