6606673856430080

Committed 15 Jul 2025 10:35PM UTC coverage: 59.8% (-0.003%) from 59.803%

Build # 6606673856430080

Build Type

push

CirrusCI

Committed by

SomberNight

Commit Message

verifier: fix off-by-one for max_checkpoint

if a wallet had a tx mined in the max_checkpoint block, in certain cases
we would leave it forever in the "unverified" state and remain stuck in "synchronizing..."

Run Details

0 of 3 new or added lines in 2 files covered. (0.0%)

10 existing lines in 5 files now uncovered.

21985 of 36764 relevant lines covered (59.8%)

2.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

41.61

/electrum/verifier.py

# Electrum - Lightweight Bitcoin Client
# Copyright (c) 2012 Thomas Voegtlin
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import asyncio
from typing import Sequence, Optional, TYPE_CHECKING

import aiorpcx

from .util import TxMinedInfo, NetworkJobOnDefaultServer
from .crypto import sha256d
from .bitcoin import hash_decode, hash_encode
from .transaction import Transaction
from .blockchain import hash_header
from .interface import GracefulDisconnect
from . import constants

if TYPE_CHECKING:
    from .network import Network
    from .address_synchronizer import AddressSynchronizer


class MerkleVerificationFailure(Exception): pass
class MissingBlockHeader(MerkleVerificationFailure): pass
class MerkleRootMismatch(MerkleVerificationFailure): pass
class InnerNodeOfSpvProofIsValidTx(MerkleVerificationFailure): pass


class SPV(NetworkJobOnDefaultServer):
    """ Simple Payment Verification """

    def __init__(self, network: 'Network', wallet: 'AddressSynchronizer'):
        self.wallet = wallet
        NetworkJobOnDefaultServer.__init__(self, network)

    def _reset(self):
        super()._reset()
        self.merkle_roots = {}  # txid -> merkle root (once it has been verified)
        self.requested_merkle = set()  # txid set of pending requests

    async def _run_tasks(self, *, taskgroup):
        await super()._run_tasks(taskgroup=taskgroup)
        async with taskgroup as group:
            await group.spawn(self.main)

    def diagnostic_name(self):
        return self.wallet.diagnostic_name()

    async def main(self):
        self.blockchain = self.network.blockchain()
        while True:
            await self._maybe_undo_verifications()
            await self._request_proofs()
            await asyncio.sleep(0.1)

    async def _request_proofs(self):
        local_height = self.blockchain.height()
        unverified = self.wallet.get_unverified_txs()

        for tx_hash, tx_height in unverified.items():
            # do not request merkle branch if we already requested it
            if tx_hash in self.requested_merkle or tx_hash in self.merkle_roots:
                continue
            # or before headers are available
            if not (0 < tx_height <= local_height):
                continue
            # if it's in the checkpoint region, we still might not have the header
            header = self.blockchain.read_header(tx_height)
            if header is None:
                if tx_height <= constants.net.max_checkpoint():
                    # FIXME these requests are not counted (self._requests_sent += 1)
                    await self.taskgroup.spawn(self.interface.request_chunk(tx_height, can_return_early=True))
                continue
            # request now
            self.logger.info(f'requested merkle {tx_hash}')
            self.requested_merkle.add(tx_hash)
            await self.taskgroup.spawn(self._request_and_verify_single_proof, tx_hash, tx_height)

    async def _request_and_verify_single_proof(self, tx_hash, tx_height):
        try:
            self._requests_sent += 1
            async with self._network_request_semaphore:
                merkle = await self.interface.get_merkle_for_transaction(tx_hash, tx_height)
        except aiorpcx.jsonrpc.RPCError:
            self.logger.info(f'tx {tx_hash} not at height {tx_height}')
            self.wallet.remove_unverified_tx(tx_hash, tx_height)
            self.requested_merkle.discard(tx_hash)
            return
        finally:
            self._requests_answered += 1
        # Verify the hash of the server-provided merkle branch to a
        # transaction matches the merkle root of its block
        if tx_height != merkle.get('block_height'):
            self.logger.info('requested tx_height {} differs from received tx_height {} for txid {}'
                             .format(tx_height, merkle.get('block_height'), tx_hash))
        tx_height = merkle.get('block_height')
        pos = merkle.get('pos')
        merkle_branch = merkle.get('merkle')
        # we need to wait if header sync/reorg is still ongoing, hence lock:
        async with self.network.bhi_lock:
            header = self.network.blockchain().read_header(tx_height)
        try:
            verify_tx_is_in_block(tx_hash, merkle_branch, pos, header, tx_height)
        except MerkleVerificationFailure as e:
            if self.network.config.NETWORK_SKIPMERKLECHECK:
                self.logger.info(f"skipping merkle proof check {tx_hash}")
            else:
                self.logger.info(repr(e))
                raise GracefulDisconnect(e) from e
        # we passed all the tests
        self.merkle_roots[tx_hash] = header.get('merkle_root')
        self.requested_merkle.discard(tx_hash)
        self.logger.info(f"verified {tx_hash}")
        header_hash = hash_header(header)
        tx_info = TxMinedInfo(height=tx_height,
                              timestamp=header.get('timestamp'),
                              txpos=pos,
                              header_hash=header_hash)
        self.wallet.add_verified_tx(tx_hash, tx_info)

    @classmethod
    def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int):
        """Return calculated merkle root."""
        try:
            h = hash_decode(tx_hash)
            merkle_branch_bytes = [hash_decode(item) for item in merkle_branch]
            leaf_pos_in_tree = int(leaf_pos_in_tree)  # raise if invalid
        except Exception as e:
            raise MerkleVerificationFailure(e)
        if leaf_pos_in_tree < 0:
            raise MerkleVerificationFailure('leaf_pos_in_tree must be non-negative')
        index = leaf_pos_in_tree
        for item in merkle_branch_bytes:
            if len(item) != 32:
                raise MerkleVerificationFailure('all merkle branch items have to be 32 bytes long')
            inner_node = (item + h) if (index & 1) else (h + item)
            cls._raise_if_valid_tx(inner_node.hex())
            h = sha256d(inner_node)
            index >>= 1
        if index != 0:
            raise MerkleVerificationFailure(f'leaf_pos_in_tree too large for branch')
        return hash_encode(h)

    @classmethod
    def _raise_if_valid_tx(cls, raw_tx: str):
        # If an inner node of the merkle proof is also a valid tx, chances are, this is an attack.
        # https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-June/016105.html
        # https://lists.linuxfoundation.org/pipermail/bitcoin-dev/attachments/20180609/9f4f5b1f/attachment-0001.pdf
        # https://bitcoin.stackexchange.com/questions/76121/how-is-the-leaf-node-weakness-in-merkle-trees-exploitable/76122#76122
        tx = Transaction(raw_tx)
        try:
            tx.deserialize()
        except Exception:
            pass
        else:
            raise InnerNodeOfSpvProofIsValidTx()

    async def _maybe_undo_verifications(self):
        old_chain = self.blockchain
        cur_chain = self.network.blockchain()
        if cur_chain != old_chain:
            self.blockchain = cur_chain
            above_height = cur_chain.get_height_of_last_common_block_with_chain(old_chain)
            self.logger.info(f"undoing verifications above height {above_height}")
            tx_hashes = self.wallet.undo_verifications(self.blockchain, above_height)
            for tx_hash in tx_hashes:
                self.logger.info(f"redoing {tx_hash}")
                self.remove_spv_proof_for_tx(tx_hash)

    def remove_spv_proof_for_tx(self, tx_hash):
        self.merkle_roots.pop(tx_hash, None)
        self.requested_merkle.discard(tx_hash)

    def is_up_to_date(self):
        return (not self.requested_merkle
                and not self.wallet.unverified_tx)


def verify_tx_is_in_block(tx_hash: str, merkle_branch: Sequence[str],
                          leaf_pos_in_tree: int, block_header: Optional[dict],
                          block_height: int) -> None:
    """Raise MerkleVerificationFailure if verification fails."""
    if not block_header:
        raise MissingBlockHeader("merkle verification failed for {} (missing header {})"
                                 .format(tx_hash, block_height))
    if len(merkle_branch) > 30:
        raise MerkleVerificationFailure(f"merkle branch too long: {len(merkle_branch)}")
    calc_merkle_root = SPV.hash_merkle_root(merkle_branch, tx_hash, leaf_pos_in_tree)
    if block_header.get('merkle_root') != calc_merkle_root:
        raise MerkleRootMismatch("merkle verification failed for {} ({} != {})".format(
            tx_hash, block_header.get('merkle_root'), calc_merkle_root))

1	# Electrum - Lightweight Bitcoin Client
2	# Copyright (c) 2012 Thomas Voegtlin
3	#
4	# Permission is hereby granted, free of charge, to any person
5	# obtaining a copy of this software and associated documentation files
6	# (the "Software"), to deal in the Software without restriction,
7	# including without limitation the rights to use, copy, modify, merge,
8	# publish, distribute, sublicense, and/or sell copies of the Software,
9	# and to permit persons to whom the Software is furnished to do so,
10	# subject to the following conditions:
11	#
12	# The above copyright notice and this permission notice shall be
13	# included in all copies or substantial portions of the Software.
14	#
15	# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16	# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17	# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18	# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
19	# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
20	# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21	# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22	# SOFTWARE.
23
24	import asyncio	5✔
25	from typing import Sequence, Optional, TYPE_CHECKING	5✔
26
27	import aiorpcx	5✔
28
29	from .util import TxMinedInfo, NetworkJobOnDefaultServer	5✔
30	from .crypto import sha256d	5✔
31	from .bitcoin import hash_decode, hash_encode	5✔
32	from .transaction import Transaction	5✔
33	from .blockchain import hash_header	5✔
34	from .interface import GracefulDisconnect	5✔
35	from . import constants	5✔
36
37	if TYPE_CHECKING:	5✔
38	from .network import Network	×
39	from .address_synchronizer import AddressSynchronizer	×
40
41
42	class MerkleVerificationFailure(Exception): pass	5✔
43	class MissingBlockHeader(MerkleVerificationFailure): pass	5✔
44	class MerkleRootMismatch(MerkleVerificationFailure): pass	5✔
45	class InnerNodeOfSpvProofIsValidTx(MerkleVerificationFailure): pass	5✔
46
47
48	class SPV(NetworkJobOnDefaultServer):	5✔
49	""" Simple Payment Verification """
50
51	def __init__(self, network: 'Network', wallet: 'AddressSynchronizer'):	5✔
52	self.wallet = wallet	5✔
53	NetworkJobOnDefaultServer.__init__(self, network)	5✔
54
55	def _reset(self):	5✔
56	super()._reset()	5✔
57	self.merkle_roots = {} # txid -> merkle root (once it has been verified)	5✔
58	self.requested_merkle = set() # txid set of pending requests	5✔
59
60	async def _run_tasks(self, *, taskgroup):	5✔
61	await super()._run_tasks(taskgroup=taskgroup)	×
62	async with taskgroup as group:	×
63	await group.spawn(self.main)	×
64
65	def diagnostic_name(self):	5✔
66	return self.wallet.diagnostic_name()	5✔
67
68	async def main(self):	5✔
69	self.blockchain = self.network.blockchain()	×
70	while True:	×
71	await self._maybe_undo_verifications()	×
72	await self._request_proofs()	×
73	await asyncio.sleep(0.1)	×
74
75	async def _request_proofs(self):	5✔
76	local_height = self.blockchain.height()	×
77	unverified = self.wallet.get_unverified_txs()	×
78
79	for tx_hash, tx_height in unverified.items():	×
80	# do not request merkle branch if we already requested it
81	if tx_hash in self.requested_merkle or tx_hash in self.merkle_roots:	×
82	continue	×
83	# or before headers are available
84	if not (0 < tx_height <= local_height):	×
85	continue	×
86	# if it's in the checkpoint region, we still might not have the header
87	header = self.blockchain.read_header(tx_height)	×
88	if header is None:	×
NEW 89	if tx_height <= constants.net.max_checkpoint():	×
90	# FIXME these requests are not counted (self._requests_sent += 1)
91	await self.taskgroup.spawn(self.interface.request_chunk(tx_height, can_return_early=True))	×
92	continue	×
93	# request now
94	self.logger.info(f'requested merkle {tx_hash}')	×
95	self.requested_merkle.add(tx_hash)	×
96	await self.taskgroup.spawn(self._request_and_verify_single_proof, tx_hash, tx_height)	×
97
98	async def _request_and_verify_single_proof(self, tx_hash, tx_height):	5✔
99	try:	×
100	self._requests_sent += 1	×
101	async with self._network_request_semaphore:	×
102	merkle = await self.interface.get_merkle_for_transaction(tx_hash, tx_height)	×
103	except aiorpcx.jsonrpc.RPCError:	×
104	self.logger.info(f'tx {tx_hash} not at height {tx_height}')	×
105	self.wallet.remove_unverified_tx(tx_hash, tx_height)	×
106	self.requested_merkle.discard(tx_hash)	×
107	return	×
108	finally:
109	self._requests_answered += 1	×
110	# Verify the hash of the server-provided merkle branch to a
111	# transaction matches the merkle root of its block
112	if tx_height != merkle.get('block_height'):	×
113	self.logger.info('requested tx_height {} differs from received tx_height {} for txid {}'	×
114	.format(tx_height, merkle.get('block_height'), tx_hash))
115	tx_height = merkle.get('block_height')	×
116	pos = merkle.get('pos')	×
117	merkle_branch = merkle.get('merkle')	×
118	# we need to wait if header sync/reorg is still ongoing, hence lock:
119	async with self.network.bhi_lock:	×
120	header = self.network.blockchain().read_header(tx_height)	×
121	try:	×
122	verify_tx_is_in_block(tx_hash, merkle_branch, pos, header, tx_height)	×
123	except MerkleVerificationFailure as e:	×
124	if self.network.config.NETWORK_SKIPMERKLECHECK:	×
125	self.logger.info(f"skipping merkle proof check {tx_hash}")	×
126	else:
127	self.logger.info(repr(e))	×
128	raise GracefulDisconnect(e) from e	×
129	# we passed all the tests
130	self.merkle_roots[tx_hash] = header.get('merkle_root')	×
131	self.requested_merkle.discard(tx_hash)	×
132	self.logger.info(f"verified {tx_hash}")	×
133	header_hash = hash_header(header)	×
134	tx_info = TxMinedInfo(height=tx_height,	×
135	timestamp=header.get('timestamp'),
136	txpos=pos,
137	header_hash=header_hash)
138	self.wallet.add_verified_tx(tx_hash, tx_info)	×
139
140	@classmethod	5✔
141	def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int):	5✔
142	"""Return calculated merkle root."""
143	try:	5✔
144	h = hash_decode(tx_hash)	5✔
145	merkle_branch_bytes = [hash_decode(item) for item in merkle_branch]	5✔
146	leaf_pos_in_tree = int(leaf_pos_in_tree) # raise if invalid	5✔
147	except Exception as e:	×
148	raise MerkleVerificationFailure(e)	×
149	if leaf_pos_in_tree < 0:	5✔
150	raise MerkleVerificationFailure('leaf_pos_in_tree must be non-negative')	×
151	index = leaf_pos_in_tree	5✔
152	for item in merkle_branch_bytes:	5✔
153	if len(item) != 32:	5✔
154	raise MerkleVerificationFailure('all merkle branch items have to be 32 bytes long')	×
155	inner_node = (item + h) if (index & 1) else (h + item)	5✔
156	cls._raise_if_valid_tx(inner_node.hex())	5✔
157	h = sha256d(inner_node)	5✔
158	index >>= 1	5✔
159	if index != 0:	5✔
160	raise MerkleVerificationFailure(f'leaf_pos_in_tree too large for branch')	×
161	return hash_encode(h)	5✔
162
163	@classmethod	5✔
164	def _raise_if_valid_tx(cls, raw_tx: str):	5✔
165	# If an inner node of the merkle proof is also a valid tx, chances are, this is an attack.
166	# https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-June/016105.html
167	# https://lists.linuxfoundation.org/pipermail/bitcoin-dev/attachments/20180609/9f4f5b1f/attachment-0001.pdf
168	# https://bitcoin.stackexchange.com/questions/76121/how-is-the-leaf-node-weakness-in-merkle-trees-exploitable/76122#76122
169	tx = Transaction(raw_tx)	5✔
170	try:	5✔
171	tx.deserialize()	5✔
172	except Exception:	5✔
173	pass	5✔
174	else:
175	raise InnerNodeOfSpvProofIsValidTx()	5✔
176
177	async def _maybe_undo_verifications(self):	5✔
178	old_chain = self.blockchain	×
179	cur_chain = self.network.blockchain()	×
180	if cur_chain != old_chain:	×
181	self.blockchain = cur_chain	×
182	above_height = cur_chain.get_height_of_last_common_block_with_chain(old_chain)	×
183	self.logger.info(f"undoing verifications above height {above_height}")	×
184	tx_hashes = self.wallet.undo_verifications(self.blockchain, above_height)	×
185	for tx_hash in tx_hashes:	×
186	self.logger.info(f"redoing {tx_hash}")	×
187	self.remove_spv_proof_for_tx(tx_hash)	×
188
189	def remove_spv_proof_for_tx(self, tx_hash):	5✔
190	self.merkle_roots.pop(tx_hash, None)	×
191	self.requested_merkle.discard(tx_hash)	×
192
193	def is_up_to_date(self):	5✔
194	return (not self.requested_merkle	×
195	and not self.wallet.unverified_tx)
196
197
198	def verify_tx_is_in_block(tx_hash: str, merkle_branch: Sequence[str],	5✔
199	leaf_pos_in_tree: int, block_header: Optional[dict],
200	block_height: int) -> None:
201	"""Raise MerkleVerificationFailure if verification fails."""
202	if not block_header:	×
203	raise MissingBlockHeader("merkle verification failed for {} (missing header {})"	×
204	.format(tx_hash, block_height))
205	if len(merkle_branch) > 30:	×
206	raise MerkleVerificationFailure(f"merkle branch too long: {len(merkle_branch)}")	×
207	calc_merkle_root = SPV.hash_merkle_root(merkle_branch, tx_hash, leaf_pos_in_tree)	×
208	if block_header.get('merkle_root') != calc_merkle_root:	×
209	raise MerkleRootMismatch("merkle verification failed for {} ({} != {})".format(	×
210	tx_hash, block_header.get('merkle_root'), calc_merkle_root))

spesmilo / electrum / 6606673856430080

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous