26825512277

Committed 02 Jun 2026 01:57PM UTC coverage: 78.671% (+0.04%) from 78.63%

Build # 26825512277

Build Type

Pull #772

github

Committed by

ivandevp

Commit Message

fix(devtools-proxy-support): reconnect SSH tunnel after unrecoverable client error COMPASS-8355

- Add sshClient.on('error') handler in SSHAgent constructor to prevent
  unhandled 'error' events from crashing the process when the SSH session
  dies unexpectedly (e.g. after hibernate)
- Add reinitializeClient flag to initialize() to recreate the ssh2 Client
  instance when it enters an unrecoverable "Instance unusable" state
- Expand retryable error patterns to include "Instance unusable after fatal
  error", "read ECONNRESET", and "Socket closed"
- Extract client setup into createSshClient() to ensure close/error handlers
  and forwardOut binding are consistent on both initial setup and recreation

Pull Request Pull Request #772: fix(devtools-proxy-support): reconnect SSH tunnel after unrecoverable client error COMPASS-8355

Coverage Stats

1884 of 2664 branches covered (70.72%)

Branch coverage included in aggregate %.

25 of 25 new or added lines in 2 files covered. (100.0%)

2 existing lines in 2 files now uncovered.

4069 of 4903 relevant lines covered (82.99%)

636.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.08

/packages/devtools-proxy-support/src/ssh.ts

import type { AgentConnectOpts } from 'agent-base';
import { Agent as AgentBase } from 'agent-base';
import type { DevtoolsProxyOptions } from './proxy-options';
import type { AgentWithInitialize } from './agent';
import type { ClientRequest } from 'http';
import type { Duplex } from 'stream';
import type { ClientChannel, ConnectConfig, Client as SshClient } from 'ssh2';
import EventEmitter, { once } from 'events';
import { promises as fs } from 'fs';
import { promisify } from 'util';
import type { ProxyLogEmitter } from './logging';
import { connect as tlsConnect } from 'tls';
import type { Socket } from 'net';
import { getFips } from 'crypto';

// eslint-disable-next-line @typescript-eslint/consistent-type-imports
function ssh2(): typeof import('ssh2') {
  if (getFips()) {
    // ssh2 uses a WASM implementation of the non-FIPS-compliant Poly1305 hash algorithm
    throw new Error(
      'devtools-proxy-support: Using `ssh2` features in FIPS mode is currently not available',
    );
  }
  // Lazily loading this package because it uses WebAssembly and therefore cannot
  // be included in startup snapshots, and generally adds unnecessary loading time
  // to the application.
  return require('ssh2');
}

// The original version of this code was largely taken from
// https://github.com/mongodb-js/compass/tree/55a5a608713d7316d158dc66febeb6b114d8b40d/packages/ssh-tunnel/src
export class SSHAgent extends AgentBase implements AgentWithInitialize {
  public logger: ProxyLogEmitter;
  public readonly proxyOptions: Readonly<DevtoolsProxyOptions>;
  private readonly url: URL;
  private sshClient: SshClient;
  private connected = false;
  private connectingPromise?: Promise<void>;
  private closed = false;
  private forwardOut!: (
    srcIP: string,
    srcPort: number,
    dstIP: string,
    dstPort: number,
  ) => Promise<ClientChannel>;

  constructor(options: DevtoolsProxyOptions, logger?: ProxyLogEmitter) {
    super();
    (this as AgentWithInitialize).on?.('error', () => {
      // Errors here should not crash the process
    });
    this.logger = logger ?? new EventEmitter().setMaxListeners(Infinity);
    this.proxyOptions = options;
    this.url = new URL(options.proxy ?? '');
    this.sshClient = this.createSshClient();
  }

  private createSshClient(): SshClient {
    const client = new (ssh2().Client)();
    client.on('close', () => {
      this.logger.emit('ssh:client-closed');
      this.connected = false;
    });
    client.on('error', () => {
      // Errors during connection setup are handled through initialize()'s
      // connectingPromise race, and post-connection errors through _connect()'s
      // catch block. This listener prevents unhandled 'error' events from
      // crashing the process when the SSH session dies unexpectedly (e.g. after
      // the host machine resumes from hibernate).
      this.connected = false;
    });
    this.forwardOut = promisify(client.forwardOut.bind(client));
    return client;
  }

  async initialize(reinitializeClient = false): Promise<void> {
    if (this.connected && !reinitializeClient) {
      return;
    }

    if (this.connectingPromise && !reinitializeClient) {
      return this.connectingPromise;
    }

    if (this.closed) {
      // A socks5 request could come in after we deliberately closed the connection. Don't reconnect in that case.
      throw new Error('Disconnected.');
    }

    if (reinitializeClient) {
      // The previous ssh2 Client instance is in an unrecoverable state (e.g.
      // "Instance unusable after fatal error" after the TCP connection was killed
      // mid-stream during hibernate). Create a fresh client before reconnecting.
      delete this.connectingPromise;
      this.connected = false;
      this.sshClient.end();
      this.sshClient = this.createSshClient();
    }

    const sshConnectConfig: ConnectConfig = {
      readyTimeout: 20000,
      keepaliveInterval: 20000,
      host: decodeURIComponent(this.url.hostname),
      port: +this.url.port || 22,
      username: decodeURIComponent(this.url.username) || undefined,
      password: decodeURIComponent(this.url.password) || undefined,
      privateKey: this.proxyOptions.sshOptions?.identityKeyFile
        ? await fs.readFile(this.proxyOptions.sshOptions.identityKeyFile)
        : undefined,
      passphrase: this.proxyOptions.sshOptions?.identityKeyPassphrase,
      // debug: console.log.bind(null, '[client]')
    };

    this.logger.emit('ssh:establishing-conection', {
      host: sshConnectConfig.host,
      port: sshConnectConfig.port,
      password: !!sshConnectConfig.password,
      privateKey: !!sshConnectConfig.privateKey,
      passphrase: !!sshConnectConfig.passphrase,
    });

    this.connectingPromise = Promise.race([
      once(this.sshClient, 'error').then(([err]) => {
        throw err;
      }),
      (() => {
        const waitForReady = once(this.sshClient, 'ready').then(
          () => undefined,
        );
        this.sshClient.connect(sshConnectConfig);
        return waitForReady;
      })(),
    ]);

    try {
      await this.connectingPromise;
    } catch (err) {
      (this as AgentWithInitialize).emit?.('error', err);
      this.logger.emit('ssh:failed-connection', {
        error: (err as any)?.stack ?? String(err),
      });
      delete this.connectingPromise;
      throw err;
    }

    delete this.connectingPromise;
    this.connected = true;
    this.logger.emit('ssh:established-connection');
  }

  override async connect(
    req: ClientRequest,
    connectOpts: AgentConnectOpts,
  ): Promise<Duplex> {
    return await this._connect(req, connectOpts);
  }

  private async _connect(
    req: ClientRequest,
    connectOpts: AgentConnectOpts,
    retriesLeft = 1,
  ): Promise<Duplex> {
    let host = '';
    try {
      // Using the `host` header matches what proxy-agent does
      host = connectOpts.host || (req.getHeader('host') as string);
      const url = new URL(req.path, `tcp://${host}:${connectOpts.port}`);

      await this.initialize();

      let sock: Duplex & Partial<Pick<Socket, 'setTimeout'>> =
        await this.forwardOut('127.0.0.1', 0, url.hostname, +url.port);
      (sock as any).setTimeout ??= function () {
        // noop, required for node-fetch
        return this;
      };
      if (connectOpts.secureEndpoint) {
        sock = tlsConnect({
          ...this.proxyOptions,
          ...connectOpts,
          socket: sock,
        });
      }
      return sock;
    } catch (err: unknown) {
      const requiresNewClient = /Instance unusable after fatal error/.test(
        (err as Error).message,
      );
      const retryableError =
        requiresNewClient ||
        /Not connected|Channel open failure|read ECONNRESET|Socket closed/.test(
          (err as Error).message,
        );
      this.logger.emit('ssh:failed-forward', {
        host,
        error: String((err as Error).stack),
        retryableError,
        retriesLeft,
      });
      if (retryableError) {
        this.connected = false;
        if (retriesLeft > 0) {
          await this.initialize(requiresNewClient);
          return await this._connect(req, connectOpts, retriesLeft - 1);
        }
      }
      throw err;
    }
  }

  destroy(): void {
    this.closed = true;
    this.sshClient.end();
  }

  async interruptForTesting(): Promise<void> {
    this.sshClient.end();
    await once(this.sshClient, 'close');
  }
}

1	import type { AgentConnectOpts } from 'agent-base';
2	import { Agent as AgentBase } from 'agent-base';	1✔
3	import type { DevtoolsProxyOptions } from './proxy-options';
4	import type { AgentWithInitialize } from './agent';
5	import type { ClientRequest } from 'http';
6	import type { Duplex } from 'stream';
7	import type { ClientChannel, ConnectConfig, Client as SshClient } from 'ssh2';
8	import EventEmitter, { once } from 'events';	1✔
9	import { promises as fs } from 'fs';	1✔
10	import { promisify } from 'util';	1✔
11	import type { ProxyLogEmitter } from './logging';
12	import { connect as tlsConnect } from 'tls';	1✔
13	import type { Socket } from 'net';
14	import { getFips } from 'crypto';	1✔
15
16	// eslint-disable-next-line @typescript-eslint/consistent-type-imports
17	function ssh2(): typeof import('ssh2') {
18	if (getFips()) {	16!
19	// ssh2 uses a WASM implementation of the non-FIPS-compliant Poly1305 hash algorithm
20	throw new Error(	×
21	'devtools-proxy-support: Using `ssh2` features in FIPS mode is currently not available',
22	);
23	}
24	// Lazily loading this package because it uses WebAssembly and therefore cannot
25	// be included in startup snapshots, and generally adds unnecessary loading time
26	// to the application.
27	return require('ssh2');	16✔
28	}
29
30	// The original version of this code was largely taken from
31	// https://github.com/mongodb-js/compass/tree/55a5a608713d7316d158dc66febeb6b114d8b40d/packages/ssh-tunnel/src
32	export class SSHAgent extends AgentBase implements AgentWithInitialize {	1✔
33	public logger: ProxyLogEmitter;
34	public readonly proxyOptions: Readonly<DevtoolsProxyOptions>;
35	private readonly url: URL;
36	private sshClient: SshClient;
37	private connected = false;	15✔
38	private connectingPromise?: Promise<void>;
39	private closed = false;	15✔
40	private forwardOut!: (
41	srcIP: string,
42	srcPort: number,
43	dstIP: string,
44	dstPort: number,
45	) => Promise<ClientChannel>;
46
47	constructor(options: DevtoolsProxyOptions, logger?: ProxyLogEmitter) {
48	super();	15✔
49	(this as AgentWithInitialize).on?.('error', () => {	15✔
50	// Errors here should not crash the process
51	});
52	this.logger = logger ?? new EventEmitter().setMaxListeners(Infinity);	15✔
53	this.proxyOptions = options;	15✔
54	this.url = new URL(options.proxy ?? '');	15!
55	this.sshClient = this.createSshClient();	15✔
56	}
57
58	private createSshClient(): SshClient {
59	const client = new (ssh2().Client)();	16✔
60	client.on('close', () => {	16✔
61	this.logger.emit('ssh:client-closed');	20✔
62	this.connected = false;	20✔
63	});
64	client.on('error', () => {	16✔
65	// Errors during connection setup are handled through initialize()'s
66	// connectingPromise race, and post-connection errors through _connect()'s
67	// catch block. This listener prevents unhandled 'error' events from
68	// crashing the process when the SSH session dies unexpectedly (e.g. after
69	// the host machine resumes from hibernate).
70	this.connected = false;	3✔
71	});
72	this.forwardOut = promisify(client.forwardOut.bind(client));	16✔
73	return client;	16✔
74	}
75
76	async initialize(reinitializeClient = false): Promise<void> {	38✔
77	if (this.connected && !reinitializeClient) {	42✔
78	return;	20✔
79	}
80
81	if (this.connectingPromise && !reinitializeClient) {	22!
82	return this.connectingPromise;	×
83	}
84
85	if (this.closed) {	22✔
86	// A socks5 request could come in after we deliberately closed the connection. Don't reconnect in that case.
87	throw new Error('Disconnected.');	1✔
88	}
89
90	if (reinitializeClient) {	21✔
91	// The previous ssh2 Client instance is in an unrecoverable state (e.g.
92	// "Instance unusable after fatal error" after the TCP connection was killed
93	// mid-stream during hibernate). Create a fresh client before reconnecting.
94	delete this.connectingPromise;	1✔
95	this.connected = false;	1✔
96	this.sshClient.end();	1✔
97	this.sshClient = this.createSshClient();	1✔
98	}
99
100	const sshConnectConfig: ConnectConfig = {	21✔
101	readyTimeout: 20000,
102	keepaliveInterval: 20000,
103	host: decodeURIComponent(this.url.hostname),
104	port: +this.url.port \|\| 22,	21!
105	username: decodeURIComponent(this.url.username) \|\| undefined,	21!
106	password: decodeURIComponent(this.url.password) \|\| undefined,	24✔
107	privateKey: this.proxyOptions.sshOptions?.identityKeyFile	21!
108	? await fs.readFile(this.proxyOptions.sshOptions.identityKeyFile)
109	: undefined,
110	passphrase: this.proxyOptions.sshOptions?.identityKeyPassphrase,
111	// debug: console.log.bind(null, '[client]')
112	};
113
114	this.logger.emit('ssh:establishing-conection', {	21✔
115	host: sshConnectConfig.host,
116	port: sshConnectConfig.port,
117	password: !!sshConnectConfig.password,
118	privateKey: !!sshConnectConfig.privateKey,
119	passphrase: !!sshConnectConfig.passphrase,
120	});
121
122	this.connectingPromise = Promise.race([	21✔
123	once(this.sshClient, 'error').then(([err]) => {
124	throw err;	4✔
125	}),
126	(() => {
127	const waitForReady = once(this.sshClient, 'ready').then(	21✔
128	() => undefined,	19✔
129	);
130	this.sshClient.connect(sshConnectConfig);	21✔
131	return waitForReady;	21✔
132	})(),
133	]);
134
135	try {	21✔
136	await this.connectingPromise;	21✔
137	} catch (err) {
138	(this as AgentWithInitialize).emit?.('error', err);	2✔
139	this.logger.emit('ssh:failed-connection', {	2✔
140	error: (err as any)?.stack ?? String(err),	2!
141	});
142	delete this.connectingPromise;	2✔
143	throw err;	2✔
144	}
145
146	delete this.connectingPromise;	19✔
147	this.connected = true;	19✔
148	this.logger.emit('ssh:established-connection');	19✔
149	}
150
151	override async connect(
152	req: ClientRequest,
153	connectOpts: AgentConnectOpts,
154	): Promise<Duplex> {
155	return await this._connect(req, connectOpts);	19✔
156	}
157
158	private async _connect(
159	req: ClientRequest,
160	connectOpts: AgentConnectOpts,
161	retriesLeft = 1,	19✔
162	): Promise<Duplex> {
163	let host = '';	22✔
164	try {	22✔
165	// Using the `host` header matches what proxy-agent does
166	host = connectOpts.host \|\| (req.getHeader('host') as string);	22!
167	const url = new URL(req.path, `tcp://${host}:${connectOpts.port}`);	22✔
168
169	await this.initialize();	22✔
170
171	let sock: Duplex & Partial<Pick<Socket, 'setTimeout'>> =
172	await this.forwardOut('127.0.0.1', 0, url.hostname, +url.port);	20✔
173	(sock as any).setTimeout ??= function () {	16✔
174	// noop, required for node-fetch
175	return this;	14✔
176	};
177	if (connectOpts.secureEndpoint) {	16!
UNCOV 178	sock = tlsConnect({	×
179	...this.proxyOptions,
180	...connectOpts,
181	socket: sock,
182	});
183	}
184	return sock;	16✔
185	} catch (err: unknown) {
186	const requiresNewClient = /Instance unusable after fatal error/.test(	6✔
187	(err as Error).message,
188	);
189	const retryableError =
190	requiresNewClient \|\|	6✔
191	/Not connected\|Channel open failure\|read ECONNRESET\|Socket closed/.test(
192	(err as Error).message,
193	);
194	this.logger.emit('ssh:failed-forward', {	6✔
195	host,
196	error: String((err as Error).stack),
197	retryableError,
198	retriesLeft,
199	});
200	if (retryableError) {	6✔
201	this.connected = false;	5✔
202	if (retriesLeft > 0) {	5✔
203	await this.initialize(requiresNewClient);	4✔
204	return await this._connect(req, connectOpts, retriesLeft - 1);	3✔
205	}
206	}
207	throw err;	2✔
208	}
209	}
210
211	destroy(): void {
212	this.closed = true;	16✔
213	this.sshClient.end();	16✔
214	}
215
216	async interruptForTesting(): Promise<void> {
217	this.sshClient.end();	2✔
218	await once(this.sshClient, 'close');	2✔
219	}
220	}

mongodb-js / devtools-shared / 26825512277

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous