• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

my8100 / logparser / 1014

01 Jan 2025 01:45PM UTC coverage: 80.811% (-6.6%) from 87.405%
1014

push

circleci

web-flow
Release v0.8.3 and support Python 3.13 (#30)

1 of 1 new or added line in 1 file covered. (100.0%)

57 existing lines in 4 files now uncovered.

737 of 912 relevant lines covered (80.81%)

0.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

48.73
/logparser/telnet.py
1
# coding: utf-8
2
import io
1✔
3
import logging
1✔
4
import os
1✔
5
import platform
1✔
6
import re
1✔
7
import sys
1✔
8

9
# DeprecationWarning: 'telnetlib' is deprecated and slated for removal in Python 3.13
10
try:
1✔
11
    import telnetlib
1✔
12
except ImportError:
×
13
    telnetlib = None
×
14

15
import traceback
1✔
16

17
import pexpect
1✔
18

19
from .common import Common
1✔
20
from .utils import get_logger
1✔
21

22

23
logger = get_logger(__name__)
1✔
24

25
SUPPORTED_SCRAPY_VERSION = '1.5.1'
1✔
26
TELNET_TIMEOUT = 10
1✔
27
TELNET_LOG_FILE = 'telnet_log'
1✔
28
TELNETCONSOLE_DEFAULT_USERNAME = 'scrapy'
1✔
29
TELNETCONSOLE_COMMAND_MAP = dict(
1✔
30
    log_file=b'settings.attributes["LOG_FILE"].value',
31
    crawler_stats=b'stats.get_stats()',
32
    crawler_engine=b'est()'
33
)
34

35

36
# noinspection PyBroadException
37
class MyTelnet(Common):
1✔
38
    logger = logger
1✔
39
    # Linux-5.0.9-301.fc30.x86_64-x86_64-with-fedora-30-Thirty'
40
    on_fedora = 'fedora' in platform.platform()
1✔
41

42
    def __init__(self, data, override_telnet_console_host, verbose):
1✔
43
        self.data = data
1✔
44
        self.OVERRIDE_TELNET_CONSOLE_HOST = override_telnet_console_host
1✔
45
        self.verbose = verbose
1✔
46
        if self.verbose:
1✔
47
            self.logger.setLevel(logging.DEBUG)
×
48
        else:
49
            self.logger.setLevel(logging.INFO)
1✔
50

51
        self.scrapy_version = self.data['latest_matches']['scrapy_version'] or '0.0.0'
1✔
52
        self.telnet_console = self.data['latest_matches']['telnet_console']
1✔
53
        self.telnet_username = self.data['latest_matches']['telnet_username'] or TELNETCONSOLE_DEFAULT_USERNAME
1✔
54
        self.telnet_password = self.data['latest_matches']['telnet_password']
1✔
55
        self.host = None
1✔
56
        self.port = None
1✔
57
        self.tn = None
1✔
58
        self.crawler_stats = {}
1✔
59
        self.crawler_engine = {}
1✔
60

61
    def main(self):
1✔
62
        try:
1✔
63
            self.run()
1✔
64
        # Cannot catch error directly in setup_pexpect()
65
        # pexpect.exceptions.EOF: End Of File (EOF). Exception style platform.
66
        # pexpect.exceptions.TIMEOUT: Timeout exceeded  # Old logfile but connected to 1.5.1
67
        # except (pexpect.exceptions.EOF, pexpect.exceptions.TIMEOUT) as err:
68
        # In setup_telnet(): # except ConnectionRefusedError as err:  # Python 2: <class 'socket.error'>
69
        except Exception as err:
1✔
70
            self.logger.error("Fail to telnet to %s:%s for %s (%s). Maybe the job was stopped: %s",
1✔
71
                              self.host, self.port, self.data['log_path'], self.scrapy_version, err)
72
            if self.verbose:
1✔
73
                self.logger.error(traceback.format_exc())
×
74
        finally:
75
            if self.tn is not None:
1✔
76
                try:
1✔
77
                    self.tn.close()
1✔
78
                except:
×
79
                    pass
×
80
            self.tn = None
1✔
81

82
        return self.crawler_stats, self.crawler_engine
1✔
83

84
    # https://stackoverflow.com/questions/18547412/python-telnetlib-to-connect-to-scrapy-telnet-to-read-stats
85
    def run(self):
1✔
86
        self.logger.debug("scrapy_version: %s", self.scrapy_version)
1✔
87
        # Telnet via pexpect would cause '[twisted] CRITICAL: Unhandled Error' in Scrapy log on Fedora:
88
        # twisted/conch/telnet.py line 585, in dataReceived
89
        # raise ValueError("Stumped", b)
90
        # builtins.ValueError: ('Stumped', b'\\xec')
91
        if (self.ON_WINDOWS or self.on_fedora) and self.scrapy_version > SUPPORTED_SCRAPY_VERSION:
1✔
92
            self.logger.error("Telnet only supports scrapy<=%s if you are running Scrapyd on Windows and Fedora, "
×
93
                              "current scrapy_version: %s", SUPPORTED_SCRAPY_VERSION, self.scrapy_version)
94
            return
×
95
        # Telnet console listening on 127.0.0.1:6023
96
        m = re.search(r'^(.+):(\d+)$', self.telnet_console)
1✔
97
        if not m:
1✔
98
            self.logger.warning("Fail to extract host and port from %s", self.telnet_console)
1✔
99
            return
1✔
100
        self.host, self.port = m.groups()
1✔
101
        self.host = self.OVERRIDE_TELNET_CONSOLE_HOST or self.host
1✔
102

103
        self.logger.debug("Try to telnet to %s:%s for %s", self.host, self.port, self.data['log_path'])
1✔
104
        if self.telnet_password or telnetlib is None:
1✔
105
            self.setup_pexpect()
1✔
106
            if self.tn is not None:
1✔
107
                self.pexpect_io()
1✔
108
        else:
109
            self.setup_telnet()
1✔
UNCOV
110
            if self.tn is not None:
×
UNCOV
111
                self.telnet_io()
×
112

113
    def setup_pexpect(self):
1✔
114
        # Cannot catch error directly here, see main()
115
        self.tn = pexpect.spawn('telnet %s %s' % (self.host, self.port), encoding='utf-8', timeout=TELNET_TIMEOUT)
1✔
116
        # logfile: <open file '<stdout>', mode 'w' at 0x7fe160149150>
117
        # logfile_read: None
118
        # logfile_send: None
119
        if self.verbose:
1✔
120
            self.tn.logfile = sys.stdout
×
121
        else:
122
            self.tn.logfile = io.open(os.path.join(self.CWD, TELNET_LOG_FILE), 'w')
1✔
123

124
    @staticmethod
1✔
125
    def telnet_callback(tn, command, option):
UNCOV
126
        if command == telnetlib.DO and option == telnetlib.TTYPE:
×
127
            tn.sendall(telnetlib.IAC + telnetlib.WILL + telnetlib.TTYPE)
×
128
            tn.sendall(telnetlib.IAC + telnetlib.SB + telnetlib.TTYPE + '\0' + 'LogParser' + telnetlib.IAC + telnetlib.SE)
×
UNCOV
129
        elif command in (telnetlib.DO, telnetlib.DONT):
×
130
            tn.sendall(telnetlib.IAC + telnetlib.WILL + option)
×
UNCOV
131
        elif command in (telnetlib.WILL, telnetlib.WONT):
×
UNCOV
132
            tn.sendall(telnetlib.IAC + telnetlib.DO + option)
×
133

134
    def setup_telnet(self):
1✔
135
        self.tn = telnetlib.Telnet(self.host, int(self.port), timeout=TELNET_TIMEOUT)
1✔
136
        # [twisted] CRITICAL: Unhandled Error
137
        # Failure: twisted.conch.telnet.OptionRefused: twisted.conch.telnet.OptionRefused
138
        # https://github.com/jookies/jasmin-web/issues/2
UNCOV
139
        self.tn.set_option_negotiation_callback(self.telnet_callback)
×
UNCOV
140
        if self.verbose:
×
141
            self.tn.set_debuglevel(logging.DEBUG)
×
142

143
    def parse_output(self, text):
1✔
UNCOV
144
        m = re.search(r'{.+}', text)
×
UNCOV
145
        if m:
×
UNCOV
146
            result = self.parse_crawler_stats(m.group())
×
147
        else:
UNCOV
148
            lines = [line for line in re.split(r'\r\n|\n|\r', text) if ':' in line]
×
UNCOV
149
            result = dict([re.split(r'\s*:\s*', line, maxsplit=1) for line in lines])
×
UNCOV
150
            for k, v in result.items():
×
UNCOV
151
                if k == 'engine.spider.name':
×
UNCOV
152
                    continue
×
UNCOV
153
                elif v == 'True':
×
154
                    result[k] = True
×
UNCOV
155
                elif v == 'False':
×
UNCOV
156
                    result[k] = False
×
157
                else:
UNCOV
158
                    try:
×
UNCOV
159
                        result[k] = int(float(v))
×
UNCOV
160
                    except (TypeError, ValueError):
×
UNCOV
161
                        pass
×
UNCOV
162
        if result:
×
UNCOV
163
            return self.get_ordered_dict(result, source='telnet')
×
164
        else:
165
            return {}
×
166

167
    def pexpect_io(self):
1✔
168
        def bytes_to_str(src):
1✔
UNCOV
169
            if self.PY2:
×
170
                return src
×
UNCOV
171
            return src.decode('utf-8')
×
172
        # TypeError: got <type 'str'> ('Username: ') as pattern,
173
        # must be one of: <type 'unicode'>, pexpect.EOF, pexpect.TIMEOUT
174
        if self.telnet_password:
1✔
175
            self.tn.expect(u'Username: ', timeout=TELNET_TIMEOUT)
1✔
UNCOV
176
            self.tn.sendline(self.telnet_username)
×
UNCOV
177
            self.tn.expect(u'Password: ', timeout=TELNET_TIMEOUT)
×
UNCOV
178
            self.tn.sendline(self.telnet_password)
×
UNCOV
179
            self.tn.expect(u'>>>', timeout=TELNET_TIMEOUT)
×
UNCOV
180
            self.logger.debug("Login successfully")
×
181
        else:
182
            self.tn.expect(u'>>>', timeout=TELNET_TIMEOUT)
×
183
            self.logger.debug("Connect successfully")
×
184

UNCOV
185
        self.tn.sendline(bytes_to_str(TELNETCONSOLE_COMMAND_MAP['log_file']))
×
UNCOV
186
        self.tn.expect(re.compile(r'[\'"].+>>>', re.S), timeout=TELNET_TIMEOUT)
×
UNCOV
187
        log_file = self.tn.after
×
UNCOV
188
        self.logger.debug("settings['LOG_FILE'] found via telnet: %s", log_file)
×
UNCOV
189
        if not self.verify_log_file_path(self.parse_log_path(self.data['log_path']), log_file):
×
190
            self.logger.warning("Skip telnet due to mismatching: %s AND %s", self.data['log_path'], log_file)
×
191
            return
×
192

UNCOV
193
        self.tn.sendline(bytes_to_str(TELNETCONSOLE_COMMAND_MAP['crawler_stats']))
×
UNCOV
194
        self.tn.expect(re.compile(r'{.+>>>', re.S), timeout=TELNET_TIMEOUT)
×
UNCOV
195
        self.crawler_stats = self.parse_output(self.tn.after)
×
196

UNCOV
197
        self.tn.sendline(bytes_to_str(TELNETCONSOLE_COMMAND_MAP['crawler_engine']))
×
UNCOV
198
        self.tn.expect(re.compile(r'Execution engine status.+>>>', re.S), timeout=TELNET_TIMEOUT)
×
UNCOV
199
        self.crawler_engine = self.parse_output(self.tn.after)
×
200

201
    def _telnet_io(self, command):
1✔
202
        # Microsoft Telnet> o
203
        # ( to )127.0.0.1 6023
204
        # >>>stats.get_stats()
205
        # >>>est()
UNCOV
206
        self.tn.write(b'%s\n' % command)
×
UNCOV
207
        content = self.tn.read_until(b'\n>>>', timeout=TELNET_TIMEOUT)
×
208
        # print(repr(content))
209
        # b"\x1bc>>> \x1b[4hstats.get_stats()\r\r\r\n{'log_count/INFO': 61,
210
        # 'start_time': datetime.datetime(2019, 1, 22, 9, 7, 14, 998126),
211
        # 'httperror/response_ignored_status_count/404': 1}\r\r\r\n>>>"
212
        # b' est()\r\r\r\nExecution engine status\r\r\r\n\r\r\r\n
213
        # time()-engine.start_time                        : 3249.7548048496246
214
        # engine.scraper.slot.needs_backout()             : False\r\r\r\n\r\r\r\n\r\r\r\n>>>'
UNCOV
215
        return content.decode('utf-8')
×
216

217
    def telnet_io(self):
1✔
218
        # spider._job, spider._version, settings.attributes["BOT_NAME"].value, JOB, SPIDER, PROJECT
219
        # '\'logs\\\\demo_persistent\\\\test\\\\2019-01-23T18_25_34.log\'\r\r\r\n>>>'
UNCOV
220
        log_file = self._telnet_io(TELNETCONSOLE_COMMAND_MAP['log_file'])
×
UNCOV
221
        self.logger.debug("settings['LOG_FILE'] found via telnet: %s", log_file)
×
222
        # Username: Password:
UNCOV
223
        if 'Username:' in log_file:
×
UNCOV
224
            self.logger.error("Telnet with auth is not supported on Windows. You can use scrapy<=%s instead: %s",
×
225
                              SUPPORTED_SCRAPY_VERSION, log_file)
UNCOV
226
            return
×
227
        if not self.verify_log_file_path(self.parse_log_path(self.data['log_path']), log_file):
×
228
            self.logger.warning("Skip telnet due to mismatching: %s vs %s", self.data['log_path'], log_file)
×
229
            return
×
230
        self.crawler_stats = self.parse_output(self._telnet_io(TELNETCONSOLE_COMMAND_MAP['crawler_stats']))
×
231
        self.crawler_engine = self.parse_output(self._telnet_io(TELNETCONSOLE_COMMAND_MAP['crawler_engine']))
×
232

233
    def verify_log_file_path(self, parts, log_file):
1✔
UNCOV
234
        for part in parts:
×
UNCOV
235
            if part not in log_file:
×
236
                self.logger.warning("%s not found in settings['LOG_FILE']: %s", part, log_file)
×
237
                return False
×
UNCOV
238
        return True
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc