chromium / web-page-replay / 78

pending completion

Build # 78

Build Type

push

travis-ci

Committed by

nedn

Commit Message

ISSUE-71: implement saving of record time in ArchivedHTTPResponse.

This change allows to save time of all requests into response data and retrieve
it at replay time to match record date.  This is useful in deterministic.js
to properly adjust fixed Date objects and not to confuse pages which assume
that the current date is not too old.  Also we maintain backward compatibility
with existing archives by using the same pre-seeded date back from 2008 if
we do not find record time field in archived response.

BUG=#71
R=nednguyen@google.com

Review URL: https://codereview.chromium.org/2507003004 .

Patch from Malets <malets@yandex-team.ru>.

Committed: 5285a73c6

Run Details

64 of 64 new or added lines in 6 files covered. (100.0%)

2606 of 3852 relevant lines covered (67.65%)

0.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.4

/httpclient.py

#!/usr/bin/env python
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Retrieve web resources over http."""

import copy
import datetime
import httplib
import logging
import random
import ssl
import StringIO

import httparchive
import platformsettings
import script_injector


# PIL isn't always available, but we still want to be able to run without
# the image scrambling functionality in this case.
try:
  import Image
except ImportError:
  Image = None

TIMER = platformsettings.timer


class HttpClientException(Exception):
  """Base class for all exceptions in httpclient."""
  pass


def _InjectScripts(response, injector):
  """Injects script generated by |injector| immediately after <head> or <html>.

  Copies |response| if it is modified.

  Args:
    response: an ArchivedHttpResponse
    injector: function which generates JavaScript string
      based on recording time (e.g. "Math.random = function(){...}")
  Returns:
    an ArchivedHttpResponse
  """
  if type(response) == tuple:
    logging.warn('tuple response: %s', response)
  content_type = response.get_header('content-type')
  if content_type and content_type.startswith('text/html'):
    text_chunks = response.get_data_as_chunks()
    text_chunks, just_injected = script_injector.InjectScript(
        text_chunks, 'text/html', injector(response.request_time))
    if just_injected:
      response = copy.deepcopy(response)
      response.set_data_from_chunks(text_chunks)
  return response


def _ScrambleImages(response):
  """If the |response| is an image, attempt to scramble it.

  Copies |response| if it is modified.

  Args:
    response: an ArchivedHttpResponse
  Returns:
    an ArchivedHttpResponse
  """

  assert Image, '--scramble_images requires the PIL module to be installed.'

  content_type = response.get_header('content-type')
  if content_type and content_type.startswith('image/'):
    try:
      image_data = response.response_data[0]
      image_data.decode(encoding='base64')
      im = Image.open(StringIO.StringIO(image_data))

      pixel_data = list(im.getdata())
      random.shuffle(pixel_data)

      scrambled_image = im.copy()
      scrambled_image.putdata(pixel_data)

      output_image_io = StringIO.StringIO()
      scrambled_image.save(output_image_io, im.format)
      output_image_data = output_image_io.getvalue()
      output_image_data.encode(encoding='base64')

      response = copy.deepcopy(response)
      response.set_data(output_image_data)
    except Exception:
      pass

  return response


class DetailedHTTPResponse(httplib.HTTPResponse):
  """Preserve details relevant to replaying responses.

  WARNING: This code uses attributes and methods of HTTPResponse
  that are not part of the public interface.
  """

  def read_chunks(self):
    """Return the response body content and timing data.

    The returned chunks have the chunk size and CRLFs stripped off.
    If the response was compressed, the returned data is still compressed.

    Returns:
      (chunks, delays)
        chunks:
          [response_body]                  # non-chunked responses
          [chunk_1, chunk_2, ...]          # chunked responses
        delays:
          [0]                              # non-chunked responses
          [chunk_1_first_byte_delay, ...]  # chunked responses

      The delay for the first body item should be recorded by the caller.
    """
    buf = []
    chunks = []
    delays = []
    if not self.chunked:
      chunks.append(self.read())
      delays.append(0)
    else:
      start = TIMER()
      try:
        while True:
          line = self.fp.readline()
          chunk_size = self._read_chunk_size(line)
          if chunk_size is None:
            raise httplib.IncompleteRead(''.join(chunks))
          if chunk_size == 0:
            break
          delays.append(TIMER() - start)
          chunks.append(self._safe_read(chunk_size))
          self._safe_read(2)  # skip the CRLF at the end of the chunk
          start = TIMER()

        # Ignore any trailers.
        while True:
          line = self.fp.readline()
          if not line or line == '\r\n':
            break
      finally:
        self.close()
    return chunks, delays

  @classmethod
  def _read_chunk_size(cls, line):
    chunk_extensions_pos = line.find(';')
    if chunk_extensions_pos != -1:
      line = line[:chunk_extensions_pos]  # strip chunk-extensions
    try:
      chunk_size = int(line, 16)
    except ValueError:
      return None
    return chunk_size


class DetailedHTTPConnection(httplib.HTTPConnection):
  """Preserve details relevant to replaying connections."""
  response_class = DetailedHTTPResponse


class DetailedHTTPSResponse(DetailedHTTPResponse):
  """Preserve details relevant to replaying SSL responses."""
  pass


class DetailedHTTPSConnection(httplib.HTTPSConnection):
  """Preserve details relevant to replaying SSL connections."""
  response_class = DetailedHTTPSResponse

  def __init__(self, host, port):
    # https://www.python.org/dev/peps/pep-0476/#opting-out
    if hasattr(ssl, '_create_unverified_context'):
      httplib.HTTPSConnection.__init__(
          self, host=host, port=port, context=ssl._create_unverified_context())
    else:
      httplib.HTTPSConnection.__init__(self, host=host, port=port)


class RealHttpFetch(object):

  def __init__(self, real_dns_lookup):
    """Initialize RealHttpFetch.

    Args:
      real_dns_lookup: a function that resolves a host to an IP. RealHttpFetch
        will resolve host name to the IP before making fetching request if this
        is not None.
    """
    self._real_dns_lookup = real_dns_lookup

  @staticmethod
  def _GetHeaderNameValue(header):
    """Parse the header line and return a name/value tuple.

    Args:
      header: a string for a header such as "Content-Length: 314".
    Returns:
      A tuple (header_name, header_value) on success or None if the header
      is not in expected format. header_name is in lowercase.
    """
    i = header.find(':')
    if i > 0:
      return (header[:i].lower(), header[i+1:].strip())
    return None

  @staticmethod
  def _ToTuples(headers):
    """Parse headers and save them to a list of tuples.

    This method takes HttpResponse.msg.headers as input and convert it
    to a list of (header_name, header_value) tuples.
    HttpResponse.msg.headers is a list of strings where each string
    represents either a header or a continuation line of a header.
    1. a normal header consists of two parts which are separated by colon :
       "header_name:header_value..."
    2. a continuation line is a string starting with whitespace
       "[whitespace]continued_header_value..."
    If a header is not in good shape or an unexpected continuation line is
    seen, it will be ignored.

    Should avoid using response.getheaders() directly
    because response.getheaders() can't handle multiple headers
    with the same name properly. Instead, parse the
    response.msg.headers using this method to get all headers.

    Args:
      headers: an instance of HttpResponse.msg.headers.
    Returns:
      A list of tuples which looks like:
      [(header_name, header_value), (header_name2, header_value2)...]
    """
    all_headers = []
    for line in headers:
      if line[0] in '\t ':
        if not all_headers:
          logging.warning(
              'Unexpected response header continuation line [%s]', line)
          continue
        name, value = all_headers.pop()
        value += '\n ' + line.strip()
      else:
        name_value = RealHttpFetch._GetHeaderNameValue(line)
        if not name_value:
          logging.warning(
              'Response header in wrong format [%s]', line)
          continue
        name, value = name_value  # pylint: disable=unpacking-non-sequence
      all_headers.append((name, value))
    return all_headers

  @staticmethod
  def _get_request_host_port(request):
    host_parts = request.host.split(':')
    host = host_parts[0]
    port = int(host_parts[1]) if len(host_parts) == 2 else None
    return host, port

  @staticmethod
  def _get_system_proxy(is_ssl):
    return platformsettings.get_system_proxy(is_ssl)

  def _get_connection(self, request_host, request_port, is_ssl):
    """Return a detailed connection object for host/port pair.

    If a system proxy is defined (see platformsettings.py), it will be used.

    Args:
      request_host: a host string (e.g. "www.example.com").
      request_port: a port integer (e.g. 8080) or None (for the default port).
      is_ssl: True if HTTPS connection is needed.
    Returns:
      A DetailedHTTPSConnection or DetailedHTTPConnection instance.
    """
    connection_host = request_host
    connection_port = request_port
    system_proxy = self._get_system_proxy(is_ssl)
    if system_proxy:
      connection_host = system_proxy.host
      connection_port = system_proxy.port

    # Use an IP address because WPR may override DNS settings.
    if self._real_dns_lookup:
      connection_ip = self._real_dns_lookup(connection_host)
      if not connection_ip:
        logging.critical(
            'Unable to find IP for host name: %s', connection_host)
        return None
      connection_host = connection_ip

    if is_ssl:
      connection = DetailedHTTPSConnection(connection_host, connection_port)
      if system_proxy:
        connection.set_tunnel(request_host, request_port)
    else:
      connection = DetailedHTTPConnection(connection_host, connection_port)
    return connection

  def __call__(self, request):
    """Fetch an HTTP request.

    Args:
      request: an ArchivedHttpRequest
    Returns:
      an ArchivedHttpResponse
    """
    logging.debug('RealHttpFetch: %s %s', request.host, request.full_path)
    request_host, request_port = self._get_request_host_port(request)
    retries = 3
    while True:
      try:
        request_time = datetime.datetime.utcnow()
        connection = self._get_connection(
            request_host, request_port, request.is_ssl)
        connect_start = TIMER()
        connection.connect()
        connect_delay = int((TIMER() - connect_start) * 1000)
        start = TIMER()
        connection.request(
            request.command,
            request.full_path,
            request.request_body,
            request.headers)
        response = connection.getresponse()
        headers_delay = int((TIMER() - start) * 1000)

        chunks, chunk_delays = response.read_chunks()
        delays = {
            'connect': connect_delay,
            'headers': headers_delay,
            'data': chunk_delays
            }
        archived_http_response = httparchive.ArchivedHttpResponse(
            response.version,
            response.status,
            response.reason,
            RealHttpFetch._ToTuples(response.msg.headers),
            chunks,
            delays,
            request_time)
        return archived_http_response
      except Exception, e:
        if retries:
          retries -= 1
          logging.warning('Retrying fetch %s: %s', request, repr(e))
          continue
        logging.critical('Could not fetch %s: %s', request, repr(e))
        return None


class RecordHttpArchiveFetch(object):
  """Make real HTTP fetches and save responses in the given HttpArchive."""

  def __init__(self, http_archive, injector):
    """Initialize RecordHttpArchiveFetch.

    Args:
      http_archive: an instance of a HttpArchive
      injector: script injector to inject scripts in all pages
    """
    self.http_archive = http_archive
    # Do not resolve host name to IP when recording to avoid SSL3 handshake
    # failure.
    # See https://github.com/chromium/web-page-replay/issues/73 for details.
    self.real_http_fetch = RealHttpFetch(real_dns_lookup=None)
    self.injector = injector

  def __call__(self, request):
    """Fetch the request and return the response.

    Args:
      request: an ArchivedHttpRequest.
    Returns:
      an ArchivedHttpResponse
    """
    # If request is already in the archive, return the archived response.
    if request in self.http_archive:
      logging.debug('Repeated request found: %s', request)
      response = self.http_archive[request]
    else:
      response = self.real_http_fetch(request)
      if response is None:
        return None
      self.http_archive[request] = response
    if self.injector:
      response = _InjectScripts(response, self.injector)
    logging.debug('Recorded: %s', request)
    return response


class ReplayHttpArchiveFetch(object):
  """Serve responses from the given HttpArchive."""

  def __init__(self, http_archive, real_dns_lookup, injector,
               use_diff_on_unknown_requests=False,
               use_closest_match=False, scramble_images=False):
    """Initialize ReplayHttpArchiveFetch.

    Args:
      http_archive: an instance of a HttpArchive
      real_dns_lookup: a function that resolves a host to an IP.
      injector: script injector to inject scripts in all pages
      use_diff_on_unknown_requests: If True, log unknown requests
        with a diff to requests that look similar.
      use_closest_match: If True, on replay mode, serve the closest match
        in the archive instead of giving a 404.
    """
    self.http_archive = http_archive
    self.injector = injector
    self.use_diff_on_unknown_requests = use_diff_on_unknown_requests
    self.use_closest_match = use_closest_match
    self.scramble_images = scramble_images
    self.real_http_fetch = RealHttpFetch(real_dns_lookup)

  def __call__(self, request):
    """Fetch the request and return the response.

    Args:
      request: an instance of an ArchivedHttpRequest.
    Returns:
      Instance of ArchivedHttpResponse (if found) or None
    """
    if request.host.startswith('127.0.0.1:'):
      return self.real_http_fetch(request)

    response = self.http_archive.get(request)

    if self.use_closest_match and not response:
      closest_request = self.http_archive.find_closest_request(
          request, use_path=True)
      if closest_request:
        response = self.http_archive.get(closest_request)
        if response:
          logging.info('Request not found: %s\nUsing closest match: %s',
                       request, closest_request)

    if not response:
      reason = str(request)
      if self.use_diff_on_unknown_requests:
        diff = self.http_archive.diff(request)
        if diff:
          reason += (
              "\nNearest request diff "
              "('-' for archived request, '+' for current request):\n%s" % diff)
      logging.warning('Could not replay: %s', reason)
    else:
      if self.injector:
        response = _InjectScripts(response, self.injector)
      if self.scramble_images:
        response = _ScrambleImages(response)
    return response


class ControllableHttpArchiveFetch(object):
  """Controllable fetch function that can swap between record and replay."""

  def __init__(self, http_archive, real_dns_lookup,
               injector, use_diff_on_unknown_requests,
               use_record_mode, use_closest_match, scramble_images):
    """Initialize HttpArchiveFetch.

    Args:
      http_archive: an instance of a HttpArchive
      real_dns_lookup: a function that resolves a host to an IP.
      injector: function to inject scripts in all pages.
        takes recording time as datetime.datetime object.
      use_diff_on_unknown_requests: If True, log unknown requests
        with a diff to requests that look similar.
      use_record_mode: If True, start in server in record mode.
      use_closest_match: If True, on replay mode, serve the closest match
        in the archive instead of giving a 404.
    """
    self.http_archive = http_archive
    self.record_fetch = RecordHttpArchiveFetch(http_archive, injector)
    self.replay_fetch = ReplayHttpArchiveFetch(
        http_archive, real_dns_lookup, injector,
        use_diff_on_unknown_requests, use_closest_match, scramble_images)
    if use_record_mode:
      self.SetRecordMode()
    else:
      self.SetReplayMode()

  def SetRecordMode(self):
    self.fetch = self.record_fetch
    self.is_record_mode = True

  def SetReplayMode(self):
    self.fetch = self.replay_fetch
    self.is_record_mode = False

  def __call__(self, *args, **kwargs):
    """Forward calls to Replay/Record fetch functions depending on mode."""
    return self.fetch(*args, **kwargs)

1	#!/usr/bin/env python
2	# Copyright 2012 Google Inc. All Rights Reserved.
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15
16	"""Retrieve web resources over http."""	1✔
17
18	import copy	1✔
19	import datetime	1✔
20	import httplib	1✔
21	import logging	1✔
22	import random	1✔
23	import ssl	1✔
24	import StringIO	1✔
25
26	import httparchive	1✔
27	import platformsettings	1✔
28	import script_injector	1✔
29
30
31	# PIL isn't always available, but we still want to be able to run without
32	# the image scrambling functionality in this case.
33	try:	1✔
34	import Image	1✔
35	except ImportError:	1✔
36	Image = None	1✔
37
38	TIMER = platformsettings.timer	1✔
39
40
41	class HttpClientException(Exception):	1✔
42	"""Base class for all exceptions in httpclient."""
43	pass	1✔
44
45
46	def _InjectScripts(response, injector):	1✔
47	"""Injects script generated by \|injector\| immediately after <head> or <html>.
48
49	Copies \|response\| if it is modified.
50
51	Args:
52	response: an ArchivedHttpResponse
53	injector: function which generates JavaScript string
54	based on recording time (e.g. "Math.random = function(){...}")
55	Returns:
56	an ArchivedHttpResponse
57	"""
58	if type(response) == tuple:	1✔
59	logging.warn('tuple response: %s', response)	×
60	content_type = response.get_header('content-type')	1✔
61	if content_type and content_type.startswith('text/html'):	1✔
62	text_chunks = response.get_data_as_chunks()	1✔
63	text_chunks, just_injected = script_injector.InjectScript(	1✔
64	text_chunks, 'text/html', injector(response.request_time))
65	if just_injected:	1✔
66	response = copy.deepcopy(response)	×
67	response.set_data_from_chunks(text_chunks)	×
68	return response	1✔
69
70
71	def _ScrambleImages(response):	1✔
72	"""If the \|response\| is an image, attempt to scramble it.
73
74	Copies \|response\| if it is modified.
75
76	Args:
77	response: an ArchivedHttpResponse
78	Returns:
79	an ArchivedHttpResponse
80	"""
81
82	assert Image, '--scramble_images requires the PIL module to be installed.'	×
83
84	content_type = response.get_header('content-type')	×
85	if content_type and content_type.startswith('image/'):	×
86	try:	×
87	image_data = response.response_data[0]	×
88	image_data.decode(encoding='base64')	×
89	im = Image.open(StringIO.StringIO(image_data))	×
90
91	pixel_data = list(im.getdata())	×
92	random.shuffle(pixel_data)	×
93
94	scrambled_image = im.copy()	×
95	scrambled_image.putdata(pixel_data)	×
96
97	output_image_io = StringIO.StringIO()	×
98	scrambled_image.save(output_image_io, im.format)	×
99	output_image_data = output_image_io.getvalue()	×
100	output_image_data.encode(encoding='base64')	×
101
102	response = copy.deepcopy(response)	×
103	response.set_data(output_image_data)	×
104	except Exception:	×
105	pass	×
106
107	return response	×
108
109
110	class DetailedHTTPResponse(httplib.HTTPResponse):	1✔
111	"""Preserve details relevant to replaying responses.
112
113	WARNING: This code uses attributes and methods of HTTPResponse
114	that are not part of the public interface.
115	"""
116
117	def read_chunks(self):	1✔
118	"""Return the response body content and timing data.
119
120	The returned chunks have the chunk size and CRLFs stripped off.
121	If the response was compressed, the returned data is still compressed.
122
123	Returns:
124	(chunks, delays)
125	chunks:
126	[response_body] # non-chunked responses
127	[chunk_1, chunk_2, ...] # chunked responses
128	delays:
129	[0] # non-chunked responses
130	[chunk_1_first_byte_delay, ...] # chunked responses
131
132	The delay for the first body item should be recorded by the caller.
133	"""
134	buf = []	1✔
135	chunks = []	1✔
136	delays = []	1✔
137	if not self.chunked:	1✔
138	chunks.append(self.read())	1✔
139	delays.append(0)	1✔
140	else:
141	start = TIMER()	×
142	try:	×
143	while True:	×
144	line = self.fp.readline()	×
145	chunk_size = self._read_chunk_size(line)	×
146	if chunk_size is None:	×
147	raise httplib.IncompleteRead(''.join(chunks))	×
148	if chunk_size == 0:	×
149	break	×
150	delays.append(TIMER() - start)	×
151	chunks.append(self._safe_read(chunk_size))	×
152	self._safe_read(2) # skip the CRLF at the end of the chunk	×
153	start = TIMER()	×
154
155	# Ignore any trailers.
156	while True:	×
157	line = self.fp.readline()	×
158	if not line or line == '\r\n':	×
159	break	×
160	finally:
161	self.close()	×
162	return chunks, delays	1✔
163
164	@classmethod	1✔
165	def _read_chunk_size(cls, line):
166	chunk_extensions_pos = line.find(';')	×
167	if chunk_extensions_pos != -1:	×
168	line = line[:chunk_extensions_pos] # strip chunk-extensions	×
169	try:	×
170	chunk_size = int(line, 16)	×
171	except ValueError:	×
172	return None	×
173	return chunk_size	×
174
175
176	class DetailedHTTPConnection(httplib.HTTPConnection):	1✔
177	"""Preserve details relevant to replaying connections."""
178	response_class = DetailedHTTPResponse	1✔
179
180
181	class DetailedHTTPSResponse(DetailedHTTPResponse):	1✔
182	"""Preserve details relevant to replaying SSL responses."""
183	pass	1✔
184
185
186	class DetailedHTTPSConnection(httplib.HTTPSConnection):	1✔
187	"""Preserve details relevant to replaying SSL connections."""
188	response_class = DetailedHTTPSResponse	1✔
189
190	def __init__(self, host, port):	1✔
191	# https://www.python.org/dev/peps/pep-0476/#opting-out
192	if hasattr(ssl, '_create_unverified_context'):	1✔
193	httplib.HTTPSConnection.__init__(	1✔
194	self, host=host, port=port, context=ssl._create_unverified_context())
195	else:
196	httplib.HTTPSConnection.__init__(self, host=host, port=port)	×
197
198
199	class RealHttpFetch(object):	1✔
200
201	def __init__(self, real_dns_lookup):	1✔
202	"""Initialize RealHttpFetch.
203
204	Args:
205	real_dns_lookup: a function that resolves a host to an IP. RealHttpFetch
206	will resolve host name to the IP before making fetching request if this
207	is not None.
208	"""
209	self._real_dns_lookup = real_dns_lookup	1✔
210
211	@staticmethod	1✔
212	def _GetHeaderNameValue(header):
213	"""Parse the header line and return a name/value tuple.
214
215	Args:
216	header: a string for a header such as "Content-Length: 314".
217	Returns:
218	A tuple (header_name, header_value) on success or None if the header
219	is not in expected format. header_name is in lowercase.
220	"""
221	i = header.find(':')	1✔
222	if i > 0:	1✔
223	return (header[:i].lower(), header[i+1:].strip())	1✔
224	return None	1✔
225
226	@staticmethod	1✔
227	def _ToTuples(headers):
228	"""Parse headers and save them to a list of tuples.
229
230	This method takes HttpResponse.msg.headers as input and convert it
231	to a list of (header_name, header_value) tuples.
232	HttpResponse.msg.headers is a list of strings where each string
233	represents either a header or a continuation line of a header.
234	1. a normal header consists of two parts which are separated by colon :
235	"header_name:header_value..."
236	2. a continuation line is a string starting with whitespace
237	"[whitespace]continued_header_value..."
238	If a header is not in good shape or an unexpected continuation line is
239	seen, it will be ignored.
240
241	Should avoid using response.getheaders() directly
242	because response.getheaders() can't handle multiple headers
243	with the same name properly. Instead, parse the
244	response.msg.headers using this method to get all headers.
245
246	Args:
247	headers: an instance of HttpResponse.msg.headers.
248	Returns:
249	A list of tuples which looks like:
250	[(header_name, header_value), (header_name2, header_value2)...]
251	"""
252	all_headers = []	1✔
253	for line in headers:	1✔
254	if line[0] in '\t ':	1✔
255	if not all_headers:	1✔
256	logging.warning(	1✔
257	'Unexpected response header continuation line [%s]', line)
258	continue	1✔
259	name, value = all_headers.pop()	1✔
260	value += '\n ' + line.strip()	1✔
261	else:
262	name_value = RealHttpFetch._GetHeaderNameValue(line)	1✔
263	if not name_value:	1✔
264	logging.warning(	1✔
265	'Response header in wrong format [%s]', line)
266	continue	1✔
267	name, value = name_value # pylint: disable=unpacking-non-sequence	1✔
268	all_headers.append((name, value))	1✔
269	return all_headers	1✔
270
271	@staticmethod	1✔
272	def _get_request_host_port(request):
273	host_parts = request.host.split(':')	1✔
274	host = host_parts[0]	1✔
275	port = int(host_parts[1]) if len(host_parts) == 2 else None	1✔
276	return host, port	1✔
277
278	@staticmethod	1✔
279	def _get_system_proxy(is_ssl):
280	return platformsettings.get_system_proxy(is_ssl)	1✔
281
282	def _get_connection(self, request_host, request_port, is_ssl):	1✔
283	"""Return a detailed connection object for host/port pair.
284
285	If a system proxy is defined (see platformsettings.py), it will be used.
286
287	Args:
288	request_host: a host string (e.g. "www.example.com").
289	request_port: a port integer (e.g. 8080) or None (for the default port).
290	is_ssl: True if HTTPS connection is needed.
291	Returns:
292	A DetailedHTTPSConnection or DetailedHTTPConnection instance.
293	"""
294	connection_host = request_host	1✔
295	connection_port = request_port	1✔
296	system_proxy = self._get_system_proxy(is_ssl)	1✔
297	if system_proxy:	1✔
298	connection_host = system_proxy.host	1✔
299	connection_port = system_proxy.port	1✔
300
301	# Use an IP address because WPR may override DNS settings.
302	if self._real_dns_lookup:	1✔
303	connection_ip = self._real_dns_lookup(connection_host)	1✔
304	if not connection_ip:	1✔
305	logging.critical(	×
306	'Unable to find IP for host name: %s', connection_host)
307	return None	×
308	connection_host = connection_ip	1✔
309
310	if is_ssl:	1✔
311	connection = DetailedHTTPSConnection(connection_host, connection_port)	1✔
312	if system_proxy:	1✔
313	connection.set_tunnel(request_host, request_port)	1✔
314	else:
315	connection = DetailedHTTPConnection(connection_host, connection_port)	1✔
316	return connection	1✔
317
318	def __call__(self, request):	1✔
319	"""Fetch an HTTP request.
320
321	Args:
322	request: an ArchivedHttpRequest
323	Returns:
324	an ArchivedHttpResponse
325	"""
326	logging.debug('RealHttpFetch: %s %s', request.host, request.full_path)	1✔
327	request_host, request_port = self._get_request_host_port(request)	1✔
328	retries = 3	1✔
329	while True:	1✔
330	try:	1✔
331	request_time = datetime.datetime.utcnow()	1✔
332	connection = self._get_connection(	1✔
333	request_host, request_port, request.is_ssl)
334	connect_start = TIMER()	1✔
335	connection.connect()	1✔
336	connect_delay = int((TIMER() - connect_start) * 1000)	1✔
337	start = TIMER()	1✔
338	connection.request(	1✔
339	request.command,
340	request.full_path,
341	request.request_body,
342	request.headers)
343	response = connection.getresponse()	1✔
344	headers_delay = int((TIMER() - start) * 1000)	1✔
345
346	chunks, chunk_delays = response.read_chunks()	1✔
347	delays = {	1✔
348	'connect': connect_delay,
349	'headers': headers_delay,
350	'data': chunk_delays
351	}
352	archived_http_response = httparchive.ArchivedHttpResponse(	1✔
353	response.version,
354	response.status,
355	response.reason,
356	RealHttpFetch._ToTuples(response.msg.headers),
357	chunks,
358	delays,
359	request_time)
360	return archived_http_response	1✔
361	except Exception, e:	×
362	if retries:	×
363	retries -= 1	×
364	logging.warning('Retrying fetch %s: %s', request, repr(e))	×
365	continue	×
366	logging.critical('Could not fetch %s: %s', request, repr(e))	×
367	return None	×
368
369
370	class RecordHttpArchiveFetch(object):	1✔
371	"""Make real HTTP fetches and save responses in the given HttpArchive."""
372
373	def __init__(self, http_archive, injector):	1✔
374	"""Initialize RecordHttpArchiveFetch.
375
376	Args:
377	http_archive: an instance of a HttpArchive
378	injector: script injector to inject scripts in all pages
379	"""
380	self.http_archive = http_archive	1✔
381	# Do not resolve host name to IP when recording to avoid SSL3 handshake
382	# failure.
383	# See https://github.com/chromium/web-page-replay/issues/73 for details.
384	self.real_http_fetch = RealHttpFetch(real_dns_lookup=None)	1✔
385	self.injector = injector	1✔
386
387	def __call__(self, request):	1✔
388	"""Fetch the request and return the response.
389
390	Args:
391	request: an ArchivedHttpRequest.
392	Returns:
393	an ArchivedHttpResponse
394	"""
395	# If request is already in the archive, return the archived response.
396	if request in self.http_archive:	1✔
397	logging.debug('Repeated request found: %s', request)	×
398	response = self.http_archive[request]	×
399	else:
400	response = self.real_http_fetch(request)	1✔
401	if response is None:	1✔
402	return None	×
403	self.http_archive[request] = response	1✔
404	if self.injector:	1✔
405	response = _InjectScripts(response, self.injector)	1✔
406	logging.debug('Recorded: %s', request)	1✔
407	return response	1✔
408
409
410	class ReplayHttpArchiveFetch(object):	1✔
411	"""Serve responses from the given HttpArchive."""
412
413	def __init__(self, http_archive, real_dns_lookup, injector,	1✔
414	use_diff_on_unknown_requests=False,
415	use_closest_match=False, scramble_images=False):
416	"""Initialize ReplayHttpArchiveFetch.
417
418	Args:
419	http_archive: an instance of a HttpArchive
420	real_dns_lookup: a function that resolves a host to an IP.
421	injector: script injector to inject scripts in all pages
422	use_diff_on_unknown_requests: If True, log unknown requests
423	with a diff to requests that look similar.
424	use_closest_match: If True, on replay mode, serve the closest match
425	in the archive instead of giving a 404.
426	"""
427	self.http_archive = http_archive	1✔
428	self.injector = injector	1✔
429	self.use_diff_on_unknown_requests = use_diff_on_unknown_requests	1✔
430	self.use_closest_match = use_closest_match	1✔
431	self.scramble_images = scramble_images	1✔
432	self.real_http_fetch = RealHttpFetch(real_dns_lookup)	1✔
433
434	def __call__(self, request):	1✔
435	"""Fetch the request and return the response.
436
437	Args:
438	request: an instance of an ArchivedHttpRequest.
439	Returns:
440	Instance of ArchivedHttpResponse (if found) or None
441	"""
442	if request.host.startswith('127.0.0.1:'):	1✔
443	return self.real_http_fetch(request)	×
444
445	response = self.http_archive.get(request)	1✔
446
447	if self.use_closest_match and not response:	1✔
448	closest_request = self.http_archive.find_closest_request(	×
449	request, use_path=True)
450	if closest_request:	×
451	response = self.http_archive.get(closest_request)	×
452	if response:	×
453	logging.info('Request not found: %s\nUsing closest match: %s',	×
454	request, closest_request)
455
456	if not response:	1✔
457	reason = str(request)	×
458	if self.use_diff_on_unknown_requests:	×
459	diff = self.http_archive.diff(request)	×
460	if diff:	×
461	reason += (	×
462	"\nNearest request diff "
463	"('-' for archived request, '+' for current request):\n%s" % diff)
464	logging.warning('Could not replay: %s', reason)	×
465	else:
466	if self.injector:	1✔
467	response = _InjectScripts(response, self.injector)	1✔
468	if self.scramble_images:	1✔
469	response = _ScrambleImages(response)	×
470	return response	1✔
471
472
473	class ControllableHttpArchiveFetch(object):	1✔
474	"""Controllable fetch function that can swap between record and replay."""
475
476	def __init__(self, http_archive, real_dns_lookup,	1✔
477	injector, use_diff_on_unknown_requests,
478	use_record_mode, use_closest_match, scramble_images):
479	"""Initialize HttpArchiveFetch.
480
481	Args:
482	http_archive: an instance of a HttpArchive
483	real_dns_lookup: a function that resolves a host to an IP.
484	injector: function to inject scripts in all pages.
485	takes recording time as datetime.datetime object.
486	use_diff_on_unknown_requests: If True, log unknown requests
487	with a diff to requests that look similar.
488	use_record_mode: If True, start in server in record mode.
489	use_closest_match: If True, on replay mode, serve the closest match
490	in the archive instead of giving a 404.
491	"""
492	self.http_archive = http_archive	×
493	self.record_fetch = RecordHttpArchiveFetch(http_archive, injector)	×
494	self.replay_fetch = ReplayHttpArchiveFetch(	×
495	http_archive, real_dns_lookup, injector,
496	use_diff_on_unknown_requests, use_closest_match, scramble_images)
497	if use_record_mode:	×
498	self.SetRecordMode()	×
499	else:
500	self.SetReplayMode()	×
501
502	def SetRecordMode(self):	1✔
503	self.fetch = self.record_fetch	×
504	self.is_record_mode = True	×
505
506	def SetReplayMode(self):	1✔
507	self.fetch = self.replay_fetch	×
508	self.is_record_mode = False	×
509
510	def __call__(self, args, *kwargs):	1✔
511	"""Forward calls to Replay/Record fetch functions depending on mode."""
512	return self.fetch(args, *kwargs)	×

chromium / web-page-replay / 78

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous