2658

pending completion

Build # 2658

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/mozilla/mozregression/commit/<a class=hub.com/mozilla/mozregression/commit/319ff9b3f71d8843c15b0e8a3d232f6d23a4b037">319ff9b3f<a href="https://github.com/mozilla/mozregression/commit/319ff9b3f71d8843c15b0e8a3d232f6d23a4b037">&quot;&gt;Bug 1655682 - Stop trying old taskcluster root URL (#716)

This is effectively a manual/partial backout of
</a><a class="double-link" href="https://github.com/mozilla/mozregression/commit/<a class="double-link" href="https://github.com/mozilla/mozregression/commit/f5f7466dd14e4a4e863cbc061a38025d152f7d99">f5f7466dd</a>">f5f7466dd</a><a href="https://github.com/mozilla/mozregression/commit/319ff9b3f71d8843c15b0e8a3d232f6d23a4b037">

Not only has the old taskcluster been decommissioned and thus no longer returns
anything, but changes to taskcluster-urls in version 13.0.1 were leading to
mozregression terminating when it passed the old TC root URLs.

Run Details

20 of 20 new or added lines in 2 files covered. (100.0%)

2333 of 2546 relevant lines covered (91.63%)

2.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.25

/mozregression/fetch_build_info.py

"""
This module offers an API to get information for one build.

The public API is composed of two classes, :class:`NightlyInfoFetcher` and
:class:`IntegrationInfoFetcher`, able to return
:class:`mozregression.build_info.BuildInfo` instances.
"""

from __future__ import absolute_import

import os
import re
from datetime import datetime
from threading import Lock, Thread

# Fix intermittent bug due to strptime first call not being thread safe
# see https://bugzilla.mozilla.org/show_bug.cgi?id=1200270
# and http://bugs.python.org/issue7980
import _strptime  # noqa
import taskcluster
from mozlog import get_proxy_logger
from requests import HTTPError
from taskcluster.exceptions import TaskclusterFailure

from mozregression.build_info import IntegrationBuildInfo, NightlyBuildInfo
from mozregression.errors import BuildInfoNotFound, MozRegressionError
from mozregression.json_pushes import JsonPushes, Push
from mozregression.network import retry_get, url_links

LOG = get_proxy_logger(__name__)


class InfoFetcher(object):
    def __init__(self, fetch_config):
        self.fetch_config = fetch_config
        self.build_regex = re.compile(fetch_config.build_regex())
        self.build_info_regex = re.compile(fetch_config.build_info_regex())

    def _update_build_info_from_txt(self, build_info):
        LOG.debug("Update build info from {}".format(build_info))
        if "build_txt_url" in build_info:
            build_info.update(self._fetch_txt_info(build_info["build_txt_url"]))

    def _fetch_txt_info(self, url):
        """
        Retrieve information from a build information txt file.

        Returns a dict with keys repository and changeset if information
        is found.
        """
        LOG.debug("Fetching txt info from {}".format(url))
        data = {}
        response = retry_get(url)
        for line in response.text.splitlines():
            if "/rev/" in line:
                repository, changeset = line.split("/rev/")
                data["repository"] = repository
                data["changeset"] = changeset
                break
        if not data:
            # the txt file could be in an old format:
            # DATE CHANGESET
            # we can try to extract that to get the changeset at least.
            matched = re.match(r"^\d+ (\w+)$", response.text.strip())
            if matched:
                data["changeset"] = matched.group(1)
        return data

    def find_build_info(self, changeset_or_date, fetch_txt_info=True):
        """
        Abstract method to retrieve build information over the internet for
        one build.

        This returns a :class:`BuildInfo` instance that contain build
        information.

        Note that this method may raise :class:`BuildInfoNotFound` on error.
        """
        raise NotImplementedError


class IntegrationInfoFetcher(InfoFetcher):
    def __init__(self, fetch_config):
        InfoFetcher.__init__(self, fetch_config)
        self.jpushes = JsonPushes(branch=fetch_config.integration_branch)
        options = fetch_config.tk_options()
        self.index = taskcluster.Index(options)
        self.queue = taskcluster.Queue(options)

    def find_build_info(self, push):
        """
        Find build info for an integration build, given a Push, a changeset or a
        date/datetime.

        if `push` is not an instance of Push (e.g. it is a date, datetime, or
        string representing the changeset), a query to json pushes will be
        done.

        Return a :class:`IntegrationBuildInfo` instance.
        """
        if not isinstance(push, Push):
            try:
                push = self.jpushes.push(push)
            except MozRegressionError as exc:
                raise BuildInfoNotFound(str(exc))

        changeset = push.changeset

        tk_routes = self.fetch_config.tk_routes(push)
        try:
            task_id = None
            stored_failure = None
            for tk_route in tk_routes:
                LOG.debug("using taskcluster route %r" % tk_route)
                try:
                    task_id = self.index.findTask(tk_route)["taskId"]
                except TaskclusterFailure as ex:
                    LOG.debug("nothing found via route %r" % tk_route)
                    stored_failure = ex
                    continue
                if task_id:
                    status = self.queue.status(task_id)["status"]
                    break
            if not task_id:
                raise stored_failure
        except TaskclusterFailure:
            raise BuildInfoNotFound(
                "Unable to find build info using the"
                " taskcluster route %r" % self.fetch_config.tk_route(push)
            )

        # find a completed run for that task
        run_id, build_date = None, None
        for run in reversed(status["runs"]):
            if run["state"] == "completed":
                run_id = run["runId"]
                build_date = datetime.strptime(run["resolved"], "%Y-%m-%dT%H:%M:%S.%fZ")
                break

        if run_id is None:
            raise BuildInfoNotFound("Unable to find completed runs for task %s" % task_id)
        artifacts = self.queue.listArtifacts(task_id, run_id)["artifacts"]

        # look over the artifacts of that run
        build_url = None
        for a in artifacts:
            name = os.path.basename(a["name"])
            if self.build_regex.search(name):
                meth = self.queue.buildUrl
                if self.fetch_config.tk_needs_auth():
                    meth = self.queue.buildSignedUrl
                build_url = meth("getArtifact", task_id, run_id, a["name"])
                break
        if build_url is None:
            raise BuildInfoNotFound(
                "unable to find a build url for the" " changeset %r" % changeset
            )
        return IntegrationBuildInfo(
            self.fetch_config,
            build_url=build_url,
            build_date=build_date,
            changeset=changeset,
            repo_url=self.jpushes.repo_url,
            task_id=task_id,
        )


class NightlyInfoFetcher(InfoFetcher):
    def __init__(self, fetch_config):
        InfoFetcher.__init__(self, fetch_config)
        self._cache_months = {}
        self._lock = Lock()
        self._fetch_lock = Lock()

    def _fetch_build_info_from_url(self, url, index, lst):
        """
        Retrieve information from a build folder url.

        Stores in a list the url index and a dict instance with keys
        build_url and build_txt_url if respectively a build file and a
        build info file are found for the url.
        """
        LOG.debug("Fetching build info from {}".format(url))
        data = {}
        if not url.endswith("/"):
            url += "/"
        links = url_links(url)
        if not self.fetch_config.has_build_info:
            links += url_links(self.fetch_config.get_nightly_info_url(url))
        for link in links:
            name = os.path.basename(link)
            if "build_url" not in data and self.build_regex.match(name):
                data["build_url"] = link
            elif "build_txt_url" not in data and self.build_info_regex.match(name):
                data["build_txt_url"] = link
        if data:
            # Check that we found all required data. The URL in build_url is
            # required. build_txt_url is optional.
            if "build_url" not in data:
                raise BuildInfoNotFound(
                    "Failed to find a build file in directory {} that "
                    "matches regex '{}'".format(url, self.build_regex.pattern)
                )

            with self._fetch_lock:
                lst.append((index, data))

    def _get_month_links(self, url):
        with self._lock:
            if url not in self._cache_months:
                self._cache_months[url] = url_links(url)
            return self._cache_months[url]

    def _get_urls(self, date):
        """
        Get the url list of the build folder for a given date.

        This methods needs to be thread-safe as it is used in
        :meth:`NightlyBuildData.get_build_url`.
        """
        LOG.debug("Get URLs for {}".format(date))
        url = self.fetch_config.get_nightly_base_url(date)
        link_regex = re.compile(self.fetch_config.get_nightly_repo_regex(date))

        month_links = self._get_month_links(url)

        # first parse monthly list to get correct directory
        matches = []
        for dirlink in month_links:
            if link_regex.search(dirlink):
                matches.append(dirlink)
        # the most recent build urls first
        matches.reverse()
        return matches

    def find_build_info(self, date, fetch_txt_info=True, max_workers=2):
        """
        Find build info for a nightly build, given a date.

        Returns a :class:`NightlyBuildInfo` instance.
        """
        # getting a valid build for a given date on nightly is tricky.
        # there is multiple possible builds folders for one date,
        # and some of them may be invalid (without binary for example)

        # to save time, we will try multiple build folders at the same
        # time in some threads. The first good one found is returned.
        try:
            build_urls = self._get_urls(date)
            LOG.debug("got build_urls %s" % build_urls)
        except HTTPError as exc:
            raise BuildInfoNotFound(str(exc))
        build_info = None

        valid_builds = []
        while build_urls:
            some = build_urls[:max_workers]
            threads = [
                Thread(target=self._fetch_build_info_from_url, args=(url, i, valid_builds))
                for i, url in enumerate(some)
            ]
            for thread in threads:
                thread.daemon = True
                thread.start()
            for thread in threads:
                while thread.is_alive():
                    thread.join(0.1)
            LOG.debug("got valid_builds %s" % valid_builds)
            if valid_builds:
                infos = sorted(valid_builds, key=lambda b: b[0])[0][1]
                if fetch_txt_info:
                    self._update_build_info_from_txt(infos)

                build_info = NightlyBuildInfo(
                    self.fetch_config,
                    build_url=infos["build_url"],
                    build_date=date,
                    changeset=infos.get("changeset"),
                    repo_url=infos.get("repository"),
                )
                break
            build_urls = build_urls[max_workers:]

        if build_info is None:
            raise BuildInfoNotFound("Unable to find build info for %s" % date)

        return build_info

1	"""
2	This module offers an API to get information for one build.
3
4	The public API is composed of two classes, :class:`NightlyInfoFetcher` and
5	:class:`IntegrationInfoFetcher`, able to return
6	:class:`mozregression.build_info.BuildInfo` instances.
7	"""
8
9	from __future__ import absolute_import	3✔
10
11	import os	3✔
12	import re	3✔
13	from datetime import datetime	3✔
14	from threading import Lock, Thread	3✔
15
16	# Fix intermittent bug due to strptime first call not being thread safe
17	# see https://bugzilla.mozilla.org/show_bug.cgi?id=1200270
18	# and http://bugs.python.org/issue7980
19	import _strptime # noqa	3✔
20	import taskcluster	3✔
21	from mozlog import get_proxy_logger	3✔
22	from requests import HTTPError	3✔
23	from taskcluster.exceptions import TaskclusterFailure	3✔
24
25	from mozregression.build_info import IntegrationBuildInfo, NightlyBuildInfo	3✔
26	from mozregression.errors import BuildInfoNotFound, MozRegressionError	3✔
27	from mozregression.json_pushes import JsonPushes, Push	3✔
28	from mozregression.network import retry_get, url_links	3✔
29
30	LOG = get_proxy_logger(__name__)	3✔
31
32
33	class InfoFetcher(object):	3✔
34	def __init__(self, fetch_config):	3✔
35	self.fetch_config = fetch_config	3✔
36	self.build_regex = re.compile(fetch_config.build_regex())	3✔
37	self.build_info_regex = re.compile(fetch_config.build_info_regex())	3✔
38
39	def _update_build_info_from_txt(self, build_info):	3✔
40	LOG.debug("Update build info from {}".format(build_info))	3✔
41	if "build_txt_url" in build_info:	3✔
42	build_info.update(self._fetch_txt_info(build_info["build_txt_url"]))	3✔
43
44	def _fetch_txt_info(self, url):	3✔
45	"""
46	Retrieve information from a build information txt file.
47
48	Returns a dict with keys repository and changeset if information
49	is found.
50	"""
51	LOG.debug("Fetching txt info from {}".format(url))	3✔
52	data = {}	3✔
53	response = retry_get(url)	3✔
54	for line in response.text.splitlines():	3✔
55	if "/rev/" in line:	3✔
56	repository, changeset = line.split("/rev/")	3✔
57	data["repository"] = repository	3✔
58	data["changeset"] = changeset	3✔
59	break	3✔
60	if not data:	3✔
61	# the txt file could be in an old format:
62	# DATE CHANGESET
63	# we can try to extract that to get the changeset at least.
64	matched = re.match(r"^\d+ (\w+)$", response.text.strip())	3✔
65	if matched:	3✔
66	data["changeset"] = matched.group(1)	3✔
67	return data	3✔
68
69	def find_build_info(self, changeset_or_date, fetch_txt_info=True):	3✔
70	"""
71	Abstract method to retrieve build information over the internet for
72	one build.
73
74	This returns a :class:`BuildInfo` instance that contain build
75	information.
76
77	Note that this method may raise :class:`BuildInfoNotFound` on error.
78	"""
79	raise NotImplementedError
80
81
82	class IntegrationInfoFetcher(InfoFetcher):	3✔
83	def __init__(self, fetch_config):	3✔
84	InfoFetcher.__init__(self, fetch_config)	3✔
85	self.jpushes = JsonPushes(branch=fetch_config.integration_branch)	3✔
86	options = fetch_config.tk_options()	3✔
87	self.index = taskcluster.Index(options)	3✔
88	self.queue = taskcluster.Queue(options)	3✔
89
90	def find_build_info(self, push):	3✔
91	"""
92	Find build info for an integration build, given a Push, a changeset or a
93	date/datetime.
94
95	if `push` is not an instance of Push (e.g. it is a date, datetime, or
96	string representing the changeset), a query to json pushes will be
97	done.
98
99	Return a :class:`IntegrationBuildInfo` instance.
100	"""
101	if not isinstance(push, Push):	3✔
102	try:	3✔
103	push = self.jpushes.push(push)	3✔
104	except MozRegressionError as exc:	3✔
105	raise BuildInfoNotFound(str(exc))	3✔
106
107	changeset = push.changeset	3✔
108
109	tk_routes = self.fetch_config.tk_routes(push)	3✔
110	try:	3✔
111	task_id = None	3✔
112	stored_failure = None	3✔
113	for tk_route in tk_routes:	3✔
114	LOG.debug("using taskcluster route %r" % tk_route)	3✔
115	try:	3✔
116	task_id = self.index.findTask(tk_route)["taskId"]	3✔
117	except TaskclusterFailure as ex:	×
118	LOG.debug("nothing found via route %r" % tk_route)	×
119	stored_failure = ex	×
120	continue	×
121	if task_id:	3✔
122	status = self.queue.status(task_id)["status"]	3✔
123	break	3✔
124	if not task_id:	3✔
125	raise stored_failure	×
126	except TaskclusterFailure:	3✔
127	raise BuildInfoNotFound(	3✔
128	"Unable to find build info using the"
129	" taskcluster route %r" % self.fetch_config.tk_route(push)
130	)
131
132	# find a completed run for that task
133	run_id, build_date = None, None	3✔
134	for run in reversed(status["runs"]):	3✔
135	if run["state"] == "completed":	3✔
136	run_id = run["runId"]	3✔
137	build_date = datetime.strptime(run["resolved"], "%Y-%m-%dT%H:%M:%S.%fZ")	3✔
138	break	3✔
139
140	if run_id is None:	3✔
141	raise BuildInfoNotFound("Unable to find completed runs for task %s" % task_id)	3✔
142	artifacts = self.queue.listArtifacts(task_id, run_id)["artifacts"]	3✔
143
144	# look over the artifacts of that run
145	build_url = None	3✔
146	for a in artifacts:	3✔
147	name = os.path.basename(a["name"])	3✔
148	if self.build_regex.search(name):	3✔
149	meth = self.queue.buildUrl	3✔
150	if self.fetch_config.tk_needs_auth():	3✔
151	meth = self.queue.buildSignedUrl	×
152	build_url = meth("getArtifact", task_id, run_id, a["name"])	3✔
153	break	3✔
154	if build_url is None:	3✔
155	raise BuildInfoNotFound(	×
156	"unable to find a build url for the" " changeset %r" % changeset
157	)
158	return IntegrationBuildInfo(	3✔
159	self.fetch_config,
160	build_url=build_url,
161	build_date=build_date,
162	changeset=changeset,
163	repo_url=self.jpushes.repo_url,
164	task_id=task_id,
165	)
166
167
168	class NightlyInfoFetcher(InfoFetcher):	3✔
169	def __init__(self, fetch_config):	3✔
170	InfoFetcher.__init__(self, fetch_config)	3✔
171	self._cache_months = {}	3✔
172	self._lock = Lock()	3✔
173	self._fetch_lock = Lock()	3✔
174
175	def _fetch_build_info_from_url(self, url, index, lst):	3✔
176	"""
177	Retrieve information from a build folder url.
178
179	Stores in a list the url index and a dict instance with keys
180	build_url and build_txt_url if respectively a build file and a
181	build info file are found for the url.
182	"""
183	LOG.debug("Fetching build info from {}".format(url))	3✔
184	data = {}	3✔
185	if not url.endswith("/"):	3✔
186	url += "/"	3✔
187	links = url_links(url)	3✔
188	if not self.fetch_config.has_build_info:	3✔
189	links += url_links(self.fetch_config.get_nightly_info_url(url))	×
190	for link in links:	3✔
191	name = os.path.basename(link)	3✔
192	if "build_url" not in data and self.build_regex.match(name):	3✔
193	data["build_url"] = link	3✔
194	elif "build_txt_url" not in data and self.build_info_regex.match(name):	3✔
195	data["build_txt_url"] = link	3✔
196	if data:	3✔
197	# Check that we found all required data. The URL in build_url is
198	# required. build_txt_url is optional.
199	if "build_url" not in data:	3✔
200	raise BuildInfoNotFound(	3✔
201	"Failed to find a build file in directory {} that "
202	"matches regex '{}'".format(url, self.build_regex.pattern)
203	)
204
205	with self._fetch_lock:	3✔
206	lst.append((index, data))	3✔
207
208	def _get_month_links(self, url):	3✔
209	with self._lock:	3✔
210	if url not in self._cache_months:	3✔
211	self._cache_months[url] = url_links(url)	3✔
212	return self._cache_months[url]	3✔
213
214	def _get_urls(self, date):	3✔
215	"""
216	Get the url list of the build folder for a given date.
217
218	This methods needs to be thread-safe as it is used in
219	:meth:`NightlyBuildData.get_build_url`.
220	"""
221	LOG.debug("Get URLs for {}".format(date))	3✔
222	url = self.fetch_config.get_nightly_base_url(date)	3✔
223	link_regex = re.compile(self.fetch_config.get_nightly_repo_regex(date))	3✔
224
225	month_links = self._get_month_links(url)	3✔
226
227	# first parse monthly list to get correct directory
228	matches = []	3✔
229	for dirlink in month_links:	3✔
230	if link_regex.search(dirlink):	3✔
231	matches.append(dirlink)	3✔
232	# the most recent build urls first
233	matches.reverse()	3✔
234	return matches	3✔
235
236	def find_build_info(self, date, fetch_txt_info=True, max_workers=2):	3✔
237	"""
238	Find build info for a nightly build, given a date.
239
240	Returns a :class:`NightlyBuildInfo` instance.
241	"""
242	# getting a valid build for a given date on nightly is tricky.
243	# there is multiple possible builds folders for one date,
244	# and some of them may be invalid (without binary for example)
245
246	# to save time, we will try multiple build folders at the same
247	# time in some threads. The first good one found is returned.
248	try:	3✔
249	build_urls = self._get_urls(date)	3✔
250	LOG.debug("got build_urls %s" % build_urls)	3✔
251	except HTTPError as exc:	×
252	raise BuildInfoNotFound(str(exc))	×
253	build_info = None	3✔
254
255	valid_builds = []	3✔
256	while build_urls:	3✔
257	some = build_urls[:max_workers]	3✔
258	threads = [	3✔
259	Thread(target=self._fetch_build_info_from_url, args=(url, i, valid_builds))
260	for i, url in enumerate(some)
261	]
262	for thread in threads:	3✔
263	thread.daemon = True	3✔
264	thread.start()	3✔
265	for thread in threads:	3✔
266	while thread.is_alive():	3✔
267	thread.join(0.1)	×
268	LOG.debug("got valid_builds %s" % valid_builds)	3✔
269	if valid_builds:	3✔
270	infos = sorted(valid_builds, key=lambda b: b[0])[0][1]	3✔
271	if fetch_txt_info:	3✔
272	self._update_build_info_from_txt(infos)	3✔
273
274	build_info = NightlyBuildInfo(	3✔
275	self.fetch_config,
276	build_url=infos["build_url"],
277	build_date=date,
278	changeset=infos.get("changeset"),
279	repo_url=infos.get("repository"),
280	)
281	break	3✔
282	build_urls = build_urls[max_workers:]	3✔
283
284	if build_info is None:	3✔
285	raise BuildInfoNotFound("Unable to find build info for %s" % date)	3✔
286
287	return build_info	3✔

mozilla / mozregression / 2658

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous