2622

pending completion

Build # 2622

Build Type

Pull #779

travis-ci-com

Committed by

web-flow

Commit Message

<a href="https://github.com/CenterForOpenScience/SHARE/commit/<a class=hub.com/CenterForOpenScience/SHARE/commit/<a class="double-link" href="https://git"><a class=hub.com/CenterForOpenScience/SHARE/commit/<a class="double-link" href="https://git"><a class=hub.com/CenterForOpenScience/SHARE/commit/<a class="double-link" href="https://git"><a class=hub.com/CenterForOpenScience/SHARE/commit/<a class="double-link" href="https://git"><a class=hub.com/CenterForOpenScience/SHARE/commit/921eec153b1fb20bbf2cd8c0bf87aabf5f2e2aa9">921eec153">&lt;a href=&quot;https://github.com/CenterForOpenScience/SHARE/commit/</a><a class="double-link" href="https://github.com/CenterForOpenScience/SHARE/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/CenterForOpenScience/SHARE/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/CenterForOpenScience/SHARE/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/CenterForOpenScience/SHARE/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/CenterForOpenScience/SHARE/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/CenterForOpenScience/SHARE/commit/921eec153b1fb20bbf2cd8c0bf87aabf5f2e2aa9">921eec153</a><a href="https://github.com/CenterForOpenScience/SHARE/commit/921eec153b1fb20bbf2cd8c0bf87aabf5f2e2aa9">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/CenterForOpenScience/SHARE/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/CenterForOpenScience/SHARE/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/CenterForOpenScience/SHARE/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/CenterForOpenScience/SHARE/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/CenterForOpenScience/SHARE/commit/921eec153b1fb20bbf2cd8c0bf87aabf5f2e2aa9&quot;&gt;921eec153&lt;/a&gt;&lt;a href=&quot;https://github.com/CenterForOpenScience/SHARE/commit/921eec153b1fb20bbf2cd8c0bf87aabf5f2e2aa9&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/CenterForOpenScience/SHARE/commit/921eec153b1fb20bbf2cd8c0bf87aabf5f2e2aa9&amp;quot;&amp;... (continued)

Pull Request Pull Request #779: [ENG-1987] indexer shenanigans

Run Details

598 of 598 new or added lines in 30 files covered. (100.0%)

10201 of 12855 relevant lines covered (79.35%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/share/bin/ingest.py

from share.bin.util import command
from share.ingest.scheduler import IngestScheduler
from share.models import SourceUniqueIdentifier
from share.models.jobs import IngestJob
from share.tasks import ingest as ingest_task
from share.util.osf import osf_sources


@command('Create IngestJobs for the specified RawDatum(s)')
def ingest(args, argv):
    """
    Usage: {0} ingest <source_configs>... [--superfluous] [--now]
           {0} ingest --suids <suid_ids>... [--superfluous] [--now]
           {0} ingest --osf [--superfluous] [--now]

    Options:
        -i, --suids         Provide Suid IDs to ingest specifically
        --osf               Shorthand for all source configs belonging to OSF sources
        -s, --superfluous   Don't skip RawDatums that already have an IngestJob
        -n, --now           Run ingest tasks synchronously for each IngestJob
    """
    suid_ids = args['<suid_ids>']
    source_configs = args['<source_configs>']
    superfluous = args.get('--superfluous')
    run_now = args['--now']
    osf = args['--osf']

    qs = SourceUniqueIdentifier.objects.all()
    if suid_ids:
        qs = qs.filter(id__in=suid_ids)
    elif source_configs:
        qs = qs.filter(source_config__label__in=source_configs)
    elif osf:
        qs = qs.filter(source_config__source__in=osf_sources())
    else:
        raise ValueError('Need raw ids, suid ids, or source configs')

    if not superfluous:
        qs = qs.filter(ingest_jobs=None)

    scheduler = IngestScheduler()
    if run_now:
        for suid in qs:
            print('Ingesting {!r}...'.format(suid))
            scheduler.reingest(suid)
    else:
        jobs = scheduler.bulk_reingest(qs)
        print('Scheduled {} IngestJobs'.format(len(jobs)))


@command('Put IngestJobs back in the worker queue')
def enqueue_ingest(args, argv):
    """
    Usage: {0} enqueue_ingest <job_ids>...
           {0} enqueue_ingest --all-of-status <job_statuses>...

    Options:
        -s, --all-of-status   Enqueue and start tasks for all jobs of the given statuses (e.g. created, failed)
    """
    job_ids = args.get('<job_ids>')

    if not job_ids:
        job_statuses = args.get('<job_statuses>')
        if not job_statuses:
            return
        status_values = [
            getattr(IngestJob.STATUS, status_key)
            for status_key in job_statuses
        ]
        job_ids = list(
            IngestJob.objects.filter(status__in=status_values).values_list('id', flat=True)
        )

    print(f're-enqueuing {len(job_ids)} jobs...')
    job_qs = IngestJob.objects.filter(id__in=job_ids)
    job_qs.update(status=IngestJob.STATUS.created)
    for job_id in job_ids:
        ingest_task.delay(job_id=job_id)

1	from share.bin.util import command	×
2	from share.ingest.scheduler import IngestScheduler	×
3	from share.models import SourceUniqueIdentifier	×
4	from share.models.jobs import IngestJob	×
5	from share.tasks import ingest as ingest_task	×
6	from share.util.osf import osf_sources	×
7
8
9	@command('Create IngestJobs for the specified RawDatum(s)')	×
10	def ingest(args, argv):
11	"""
12	Usage: {0} ingest <source_configs>... [--superfluous] [--now]
13	{0} ingest --suids <suid_ids>... [--superfluous] [--now]
14	{0} ingest --osf [--superfluous] [--now]
15
16	Options:
17	-i, --suids Provide Suid IDs to ingest specifically
18	--osf Shorthand for all source configs belonging to OSF sources
19	-s, --superfluous Don't skip RawDatums that already have an IngestJob
20	-n, --now Run ingest tasks synchronously for each IngestJob
21	"""
22	suid_ids = args['<suid_ids>']	×
23	source_configs = args['<source_configs>']	×
24	superfluous = args.get('--superfluous')	×
25	run_now = args['--now']	×
26	osf = args['--osf']	×
27
28	qs = SourceUniqueIdentifier.objects.all()	×
29	if suid_ids:	×
30	qs = qs.filter(id__in=suid_ids)	×
31	elif source_configs:	×
32	qs = qs.filter(source_config__label__in=source_configs)	×
33	elif osf:	×
34	qs = qs.filter(source_config__source__in=osf_sources())	×
35	else:
36	raise ValueError('Need raw ids, suid ids, or source configs')	×
37
38	if not superfluous:	×
39	qs = qs.filter(ingest_jobs=None)	×
40
41	scheduler = IngestScheduler()	×
42	if run_now:	×
43	for suid in qs:	×
44	print('Ingesting {!r}...'.format(suid))	×
45	scheduler.reingest(suid)	×
46	else:
47	jobs = scheduler.bulk_reingest(qs)	×
48	print('Scheduled {} IngestJobs'.format(len(jobs)))	×
49
50
51	@command('Put IngestJobs back in the worker queue')	×
52	def enqueue_ingest(args, argv):
53	"""
54	Usage: {0} enqueue_ingest <job_ids>...
55	{0} enqueue_ingest --all-of-status <job_statuses>...
56
57	Options:
58	-s, --all-of-status Enqueue and start tasks for all jobs of the given statuses (e.g. created, failed)
59	"""
60	job_ids = args.get('<job_ids>')	×
61
62	if not job_ids:	×
63	job_statuses = args.get('<job_statuses>')	×
64	if not job_statuses:	×
65	return	×
66	status_values = [	×
67	getattr(IngestJob.STATUS, status_key)
68	for status_key in job_statuses
69	]
70	job_ids = list(	×
71	IngestJob.objects.filter(status__in=status_values).values_list('id', flat=True)
72	)
73
74	print(f're-enqueuing {len(job_ids)} jobs...')	×
75	job_qs = IngestJob.objects.filter(id__in=job_ids)	×
76	job_qs.update(status=IngestJob.STATUS.created)	×
77	for job_id in job_ids:	×
78	ingest_task.delay(job_id=job_id)	×

CenterForOpenScience / SHARE / 2622

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous