5758403852

pending completion

Build # 5758403852

Build Type

push

github

Committed by

evanlinjin

Commit Message

Merge bitcoindevkit/bdk#1023: Update rust bitcoin

1da3b304b ci: Pin rustls to keep the MSRV (Daniela Brozzoni)
792b39fa9 Explicitly deny multipath keys (Daniela Brozzoni)
b73385dbd Update wallet_electrum to rust-bitcoin 0.30.0 (Daniela Brozzoni)
3dac3f9bb Update example_electrum to rust-bitcoin 0.30.0 (Daniela Brozzoni)
2949bdc7b Update example_cli to rust-bitcoin 0.30.0 (Daniela Brozzoni)
468d2a0a3 Update tmp_plan to rust-bitcoin 0.30.0 (Daniela Brozzoni)
b8ac16d03 Update coin_select to rust-bitcoin 0.30.0 (Daniela Brozzoni)
6c29e53ee Update wallet_esplora and wallet_esplora_async to... ...rust-bitcoin 0.30.0 (Daniela Brozzoni)
6eb079576 Update crates/esplora to rust-bitcoin 0.30.0 (Daniela Brozzoni)
91b0f0ba2 Update crates/electrum to bitcoin 0.30.0 (Daniela Brozzoni)
f4e3ba326 Update bdk to bitcoin 0.30.0 (Daniela Brozzoni)
853d36175 Update bdk_chain to bitcoin 0.30.0 (Daniela Brozzoni)

Pull request description:

  ### Description

  Updates to rust-bitcoin 0.30.0 and miniscript 0.10.0

  Not covered in this PR:
  - https://github.com/bitcoindevkit/bdk/issues/1036. Although the latter is deprecated, I think it's better if I update it in a separate PR, as this one is pretty big already.
  - https://github.com/bitcoindevkit/bdk/issues/1037
  - https://github.com/bitcoindevkit/bdk/issues/1038

  Heads up, I'm explicitly denying multipath descriptors until we have better tests for them. See the commit message of 23fba7aee

  ### Changelog notice

  - Update to `rust-bitcoin` 0.30.0 and `miniscript` 10.0.0

  ### Checklists

  #### All Submissions:

  * [x... (continued)

Run Details

239 of 239 new or added lines in 20 files covered. (100.0%)

7007 of 9033 relevant lines covered (77.57%)

5623.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/crates/electrum/src/electrum_ext.rs

use bdk_chain::{
    bitcoin::{OutPoint, ScriptBuf, Transaction, Txid},
    keychain::LocalUpdate,
    local_chain::{self, CheckPoint},
    tx_graph::{self, TxGraph},
    Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeAnchor,
};
use electrum_client::{Client, ElectrumApi, Error, HeaderNotification};
use std::{
    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
    fmt::Debug,
    str::FromStr,
};

/// We assume that a block of this depth and deeper cannot be reorged.
const ASSUME_FINAL_DEPTH: u32 = 8;

/// Represents an update fetched from an Electrum server, but excludes full transactions.
///
/// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to
/// determine the full transactions missing from [`TxGraph`]. Then call [`Self::finalize`] to fetch
/// the full transactions from Electrum and finalize the update.
#[derive(Debug, Clone)]
pub struct ElectrumUpdate<K, A> {
    /// Map of [`Txid`]s to associated [`Anchor`]s.
    pub graph_update: HashMap<Txid, BTreeSet<A>>,
    /// The latest chain tip, as seen by the Electrum server.
    pub new_tip: local_chain::CheckPoint,
    /// Last-used index update for [`KeychainTxOutIndex`](bdk_chain::keychain::KeychainTxOutIndex).
    pub keychain_update: BTreeMap<K, u32>,
}

impl<K, A: Anchor> ElectrumUpdate<K, A> {
    fn new(new_tip: local_chain::CheckPoint) -> Self {
        Self {
            new_tip,
            graph_update: HashMap::new(),
            keychain_update: BTreeMap::new(),
        }
    }

    /// Determine the full transactions that are missing from `graph`.
    ///
    /// Refer to [`ElectrumUpdate`].
    pub fn missing_full_txs<A2>(&self, graph: &TxGraph<A2>) -> Vec<Txid> {
        self.graph_update
            .keys()
            .filter(move |&&txid| graph.as_ref().get_tx(txid).is_none())
            .cloned()
            .collect()
    }

    /// Finalizes update with `missing` txids to fetch from `client`.
    ///
    /// Refer to [`ElectrumUpdate`].
    pub fn finalize(
        self,
        client: &Client,
        seen_at: Option<u64>,
        missing: Vec<Txid>,
    ) -> Result<LocalUpdate<K, A>, Error> {
        let new_txs = client.batch_transaction_get(&missing)?;
        let mut graph_update = TxGraph::<A>::new(new_txs);
        for (txid, anchors) in self.graph_update {
            if let Some(seen_at) = seen_at {
                let _ = graph_update.insert_seen_at(txid, seen_at);
            }
            for anchor in anchors {
                let _ = graph_update.insert_anchor(txid, anchor);
            }
        }
        Ok(LocalUpdate {
            last_active_indices: self.keychain_update,
            graph: graph_update,
            chain: local_chain::Update {
                tip: self.new_tip,
                introduce_older_blocks: true,
            },
        })
    }
}

impl<K> ElectrumUpdate<K, ConfirmationHeightAnchor> {
    /// Finalizes the [`ElectrumUpdate`] with `new_txs` and anchors of type
    /// [`ConfirmationTimeAnchor`].
    ///
    /// **Note:** The confirmation time might not be precisely correct if there has been a reorg.
    /// Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to
    /// use it.
    pub fn finalize_as_confirmation_time(
        self,
        client: &Client,
        seen_at: Option<u64>,
        missing: Vec<Txid>,
    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error> {
        let update = self.finalize(client, seen_at, missing)?;

        let relevant_heights = {
            let mut visited_heights = HashSet::new();
            update
                .graph
                .all_anchors()
                .iter()
                .map(|(a, _)| a.confirmation_height_upper_bound())
                .filter(move |&h| visited_heights.insert(h))
                .collect::<Vec<_>>()
        };

        let height_to_time = relevant_heights
            .clone()
            .into_iter()
            .zip(
                client
                    .batch_block_header(relevant_heights)?
                    .into_iter()
                    .map(|bh| bh.time as u64),
            )
            .collect::<HashMap<u32, u64>>();

        let graph_additions = {
            let old_additions = TxGraph::default().determine_additions(&update.graph);
            tx_graph::Additions {
                txs: old_additions.txs,
                txouts: old_additions.txouts,
                last_seen: old_additions.last_seen,
                anchors: old_additions
                    .anchors
                    .into_iter()
                    .map(|(height_anchor, txid)| {
                        let confirmation_height = height_anchor.confirmation_height;
                        let confirmation_time = height_to_time[&confirmation_height];
                        let time_anchor = ConfirmationTimeAnchor {
                            anchor_block: height_anchor.anchor_block,
                            confirmation_height,
                            confirmation_time,
                        };
                        (time_anchor, txid)
                    })
                    .collect(),
            }
        };

        Ok(LocalUpdate {
            last_active_indices: update.last_active_indices,
            graph: {
                let mut graph = TxGraph::default();
                graph.apply_additions(graph_additions);
                graph
            },
            chain: update.chain,
        })
    }
}

/// Trait to extend [`Client`] functionality.
pub trait ElectrumExt<A> {
    /// Scan the blockchain (via electrum) for the data specified and returns a [`ElectrumUpdate`].
    ///
    /// - `prev_tip`: the most recent blockchain tip present locally
    /// - `keychain_spks`: keychains that we want to scan transactions for
    /// - `txids`: transactions for which we want updated [`Anchor`]s
    /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
    ///     want to included in the update
    ///
    /// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
    /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a
    /// single batch request.
    fn scan<K: Ord + Clone>(
        &self,
        prev_tip: Option<CheckPoint>,
        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
        txids: impl IntoIterator<Item = Txid>,
        outpoints: impl IntoIterator<Item = OutPoint>,
        stop_gap: usize,
        batch_size: usize,
    ) -> Result<ElectrumUpdate<K, A>, Error>;

    /// Convenience method to call [`scan`] without requiring a keychain.
    ///
    /// [`scan`]: ElectrumExt::scan
    fn scan_without_keychain(
        &self,
        prev_tip: Option<CheckPoint>,
        misc_spks: impl IntoIterator<Item = ScriptBuf>,
        txids: impl IntoIterator<Item = Txid>,
        outpoints: impl IntoIterator<Item = OutPoint>,
        batch_size: usize,
    ) -> Result<ElectrumUpdate<(), A>, Error> {
        let spk_iter = misc_spks
            .into_iter()
            .enumerate()
            .map(|(i, spk)| (i as u32, spk));

        self.scan(
            prev_tip,
            [((), spk_iter)].into(),
            txids,
            outpoints,
            usize::MAX,
            batch_size,
        )
    }
}

impl ElectrumExt<ConfirmationHeightAnchor> for Client {
    fn scan<K: Ord + Clone>(
        &self,
        prev_tip: Option<CheckPoint>,
        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
        txids: impl IntoIterator<Item = Txid>,
        outpoints: impl IntoIterator<Item = OutPoint>,
        stop_gap: usize,
        batch_size: usize,
    ) -> Result<ElectrumUpdate<K, ConfirmationHeightAnchor>, Error> {
        let mut request_spks = keychain_spks
            .into_iter()
            .map(|(k, s)| (k, s.into_iter()))
            .collect::<BTreeMap<K, _>>();
        let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new();

        let txids = txids.into_iter().collect::<Vec<_>>();
        let outpoints = outpoints.into_iter().collect::<Vec<_>>();

        let update = loop {
            let (tip, _) = construct_update_tip(self, prev_tip.clone())?;
            let mut update = ElectrumUpdate::<K, ConfirmationHeightAnchor>::new(tip.clone());
            let cps = update
                .new_tip
                .iter()
                .take(10)
                .map(|cp| (cp.height(), cp))
                .collect::<BTreeMap<u32, CheckPoint>>();

            if !request_spks.is_empty() {
                if !scanned_spks.is_empty() {
                    scanned_spks.append(&mut populate_with_spks(
                        self,
                        &cps,
                        &mut update,
                        &mut scanned_spks
                            .iter()
                            .map(|(i, (spk, _))| (i.clone(), spk.clone())),
                        stop_gap,
                        batch_size,
                    )?);
                }
                for (keychain, keychain_spks) in &mut request_spks {
                    scanned_spks.extend(
                        populate_with_spks(
                            self,
                            &cps,
                            &mut update,
                            keychain_spks,
                            stop_gap,
                            batch_size,
                        )?
                        .into_iter()
                        .map(|(spk_i, spk)| ((keychain.clone(), spk_i), spk)),
                    );
                }
            }

            populate_with_txids(self, &cps, &mut update, &mut txids.iter().cloned())?;

            let _txs =
                populate_with_outpoints(self, &cps, &mut update, &mut outpoints.iter().cloned())?;

            // check for reorgs during scan process
            let server_blockhash = self.block_header(tip.height() as usize)?.block_hash();
            if tip.hash() != server_blockhash {
                continue; // reorg
            }

            update.keychain_update = request_spks
                .into_keys()
                .filter_map(|k| {
                    scanned_spks
                        .range((k.clone(), u32::MIN)..=(k.clone(), u32::MAX))
                        .rev()
                        .find(|(_, (_, active))| *active)
                        .map(|((_, i), _)| (k, *i))
                })
                .collect::<BTreeMap<_, _>>();
            break update;
        };

        Ok(update)
    }
}

/// Return a [`CheckPoint`] of the latest tip, that connects with `prev_tip`.
fn construct_update_tip(
    client: &Client,
    prev_tip: Option<CheckPoint>,
) -> Result<(CheckPoint, Option<u32>), Error> {
    let HeaderNotification { height, .. } = client.block_headers_subscribe()?;
    let new_tip_height = height as u32;

    // If electrum returns a tip height that is lower than our previous tip, then checkpoints do
    // not need updating. We just return the previous tip and use that as the point of agreement.
    if let Some(prev_tip) = prev_tip.as_ref() {
        if new_tip_height < prev_tip.height() {
            return Ok((prev_tip.clone(), Some(prev_tip.height())));
        }
    }

    // Atomically fetch the latest `ASSUME_FINAL_DEPTH` count of blocks from Electrum. We use this
    // to construct our checkpoint update.
    let mut new_blocks = {
        let start_height = new_tip_height.saturating_sub(ASSUME_FINAL_DEPTH);
        let hashes = client
            .block_headers(start_height as _, ASSUME_FINAL_DEPTH as _)?
            .headers
            .into_iter()
            .map(|h| h.block_hash());
        (start_height..).zip(hashes).collect::<BTreeMap<u32, _>>()
    };

    // Find the "point of agreement" (if any).
    let agreement_cp = {
        let mut agreement_cp = Option::<CheckPoint>::None;
        for cp in prev_tip.iter().flat_map(CheckPoint::iter) {
            let cp_block = cp.block_id();
            let hash = match new_blocks.get(&cp_block.height) {
                Some(&hash) => hash,
                None => {
                    assert!(
                        new_tip_height >= cp_block.height,
                        "already checked that electrum's tip cannot be smaller"
                    );
                    let hash = client.block_header(cp_block.height as _)?.block_hash();
                    new_blocks.insert(cp_block.height, hash);
                    hash
                }
            };
            if hash == cp_block.hash {
                agreement_cp = Some(cp);
                break;
            }
        }
        agreement_cp
    };

    let agreement_height = agreement_cp.as_ref().map(CheckPoint::height);

    let new_tip = new_blocks
        .into_iter()
        // Prune `new_blocks` to only include blocks that are actually new.
        .filter(|(height, _)| Some(*height) > agreement_height)
        .map(|(height, hash)| BlockId { height, hash })
        .fold(agreement_cp, |prev_cp, block| {
            Some(match prev_cp {
                Some(cp) => cp.push(block).expect("must extend checkpoint"),
                None => CheckPoint::new(block),
            })
        })
        .expect("must have at least one checkpoint");

    Ok((new_tip, agreement_height))
}

/// A [tx status] comprises of a concatenation of `tx_hash:height:`s. We transform a single one of
/// these concatenations into a [`ConfirmationHeightAnchor`] if possible.
///
/// We use the lowest possible checkpoint as the anchor block (from `cps`). If an anchor block
/// cannot be found, or the transaction is unconfirmed, [`None`] is returned.
///
/// [tx status](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status)
fn determine_tx_anchor(
    cps: &BTreeMap<u32, CheckPoint>,
    raw_height: i32,
    txid: Txid,
) -> Option<ConfirmationHeightAnchor> {
    // The electrum API has a weird quirk where an unconfirmed transaction is presented with a
    // height of 0. To avoid invalid representation in our data structures, we manually set
    // transactions residing in the genesis block to have height 0, then interpret a height of 0 as
    // unconfirmed for all other transactions.
    if txid
        == Txid::from_str("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")
            .expect("must deserialize genesis coinbase txid")
    {
        let anchor_block = cps.values().next()?.block_id();
        return Some(ConfirmationHeightAnchor {
            anchor_block,
            confirmation_height: 0,
        });
    }
    match raw_height {
        h if h <= 0 => {
            debug_assert!(h == 0 || h == -1, "unexpected height ({}) from electrum", h);
            None
        }
        h => {
            let h = h as u32;
            let anchor_block = cps.range(h..).next().map(|(_, cp)| cp.block_id())?;
            if h > anchor_block.height {
                None
            } else {
                Some(ConfirmationHeightAnchor {
                    anchor_block,
                    confirmation_height: h,
                })
            }
        }
    }
}

fn populate_with_outpoints<K>(
    client: &Client,
    cps: &BTreeMap<u32, CheckPoint>,
    update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,
    outpoints: &mut impl Iterator<Item = OutPoint>,
) -> Result<HashMap<Txid, Transaction>, Error> {
    let mut full_txs = HashMap::new();
    for outpoint in outpoints {
        let txid = outpoint.txid;
        let tx = client.transaction_get(&txid)?;
        debug_assert_eq!(tx.txid(), txid);
        let txout = match tx.output.get(outpoint.vout as usize) {
            Some(txout) => txout,
            None => continue,
        };
        // attempt to find the following transactions (alongside their chain positions), and
        // add to our sparsechain `update`:
        let mut has_residing = false; // tx in which the outpoint resides
        let mut has_spending = false; // tx that spends the outpoint
        for res in client.script_get_history(&txout.script_pubkey)? {
            if has_residing && has_spending {
                break;
            }

            if res.tx_hash == txid {
                if has_residing {
                    continue;
                }
                has_residing = true;
                full_txs.insert(res.tx_hash, tx.clone());
            } else {
                if has_spending {
                    continue;
                }
                let res_tx = match full_txs.get(&res.tx_hash) {
                    Some(tx) => tx,
                    None => {
                        let res_tx = client.transaction_get(&res.tx_hash)?;
                        full_txs.insert(res.tx_hash, res_tx);
                        full_txs.get(&res.tx_hash).expect("just inserted")
                    }
                };
                has_spending = res_tx
                    .input
                    .iter()
                    .any(|txin| txin.previous_output == outpoint);
                if !has_spending {
                    continue;
                }
            };

            let anchor = determine_tx_anchor(cps, res.height, res.tx_hash);

            let tx_entry = update.graph_update.entry(res.tx_hash).or_default();
            if let Some(anchor) = anchor {
                tx_entry.insert(anchor);
            }
        }
    }
    Ok(full_txs)
}

fn populate_with_txids<K>(
    client: &Client,
    cps: &BTreeMap<u32, CheckPoint>,
    update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,
    txids: &mut impl Iterator<Item = Txid>,
) -> Result<(), Error> {
    for txid in txids {
        let tx = match client.transaction_get(&txid) {
            Ok(tx) => tx,
            Err(electrum_client::Error::Protocol(_)) => continue,
            Err(other_err) => return Err(other_err),
        };

        let spk = tx
            .output
            .get(0)
            .map(|txo| &txo.script_pubkey)
            .expect("tx must have an output");

        let anchor = match client
            .script_get_history(spk)?
            .into_iter()
            .find(|r| r.tx_hash == txid)
        {
            Some(r) => determine_tx_anchor(cps, r.height, txid),
            None => continue,
        };

        let tx_entry = update.graph_update.entry(txid).or_default();
        if let Some(anchor) = anchor {
            tx_entry.insert(anchor);
        }
    }
    Ok(())
}

fn populate_with_spks<K, I: Ord + Clone>(
    client: &Client,
    cps: &BTreeMap<u32, CheckPoint>,
    update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,
    spks: &mut impl Iterator<Item = (I, ScriptBuf)>,
    stop_gap: usize,
    batch_size: usize,
) -> Result<BTreeMap<I, (ScriptBuf, bool)>, Error> {
    let mut unused_spk_count = 0_usize;
    let mut scanned_spks = BTreeMap::new();

    loop {
        let spks = (0..batch_size)
            .map_while(|_| spks.next())
            .collect::<Vec<_>>();
        if spks.is_empty() {
            return Ok(scanned_spks);
        }

        let spk_histories =
            client.batch_script_get_history(spks.iter().map(|(_, s)| s.as_script()))?;

        for ((spk_index, spk), spk_history) in spks.into_iter().zip(spk_histories) {
            if spk_history.is_empty() {
                scanned_spks.insert(spk_index, (spk, false));
                unused_spk_count += 1;
                if unused_spk_count > stop_gap {
                    return Ok(scanned_spks);
                }
                continue;
            } else {
                scanned_spks.insert(spk_index, (spk, true));
                unused_spk_count = 0;
            }

            for tx in spk_history {
                let tx_entry = update.graph_update.entry(tx.tx_hash).or_default();
                if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) {
                    tx_entry.insert(anchor);
                }
            }
        }
    }
}

1	use bdk_chain::{
2	bitcoin::{OutPoint, ScriptBuf, Transaction, Txid},
3	keychain::LocalUpdate,
4	local_chain::{self, CheckPoint},
5	tx_graph::{self, TxGraph},
6	Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeAnchor,
7	};
8	use electrum_client::{Client, ElectrumApi, Error, HeaderNotification};
9	use std::{
10	collections::{BTreeMap, BTreeSet, HashMap, HashSet},
11	fmt::Debug,
12	str::FromStr,
13	};
14
15	/// We assume that a block of this depth and deeper cannot be reorged.
16	const ASSUME_FINAL_DEPTH: u32 = 8;
17
18	/// Represents an update fetched from an Electrum server, but excludes full transactions.
19	///
20	/// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to
21	/// determine the full transactions missing from [`TxGraph`]. Then call [`Self::finalize`] to fetch
22	/// the full transactions from Electrum and finalize the update.
23	#[derive(Debug, Clone)]	×
24	pub struct ElectrumUpdate<K, A> {
25	/// Map of [`Txid`]s to associated [`Anchor`]s.
26	pub graph_update: HashMap<Txid, BTreeSet<A>>,
27	/// The latest chain tip, as seen by the Electrum server.
28	pub new_tip: local_chain::CheckPoint,
29	/// Last-used index update for [`KeychainTxOutIndex`](bdk_chain::keychain::KeychainTxOutIndex).
30	pub keychain_update: BTreeMap<K, u32>,
31	}
32
33	impl<K, A: Anchor> ElectrumUpdate<K, A> {
34	fn new(new_tip: local_chain::CheckPoint) -> Self {	×
35	Self {	×
36	new_tip,	×
37	graph_update: HashMap::new(),	×
38	keychain_update: BTreeMap::new(),	×
39	}	×
40	}	×
41
42	/// Determine the full transactions that are missing from `graph`.
43	///
44	/// Refer to [`ElectrumUpdate`].
45	pub fn missing_full_txs<A2>(&self, graph: &TxGraph<A2>) -> Vec<Txid> {	×
46	self.graph_update	×
47	.keys()	×
48	.filter(move \|&&txid\| graph.as_ref().get_tx(txid).is_none())	×
49	.cloned()	×
50	.collect()	×
51	}	×
52
53	/// Finalizes update with `missing` txids to fetch from `client`.
54	///
55	/// Refer to [`ElectrumUpdate`].
56	pub fn finalize(	×
57	self,	×
58	client: &Client,	×
59	seen_at: Option<u64>,	×
60	missing: Vec<Txid>,	×
61	) -> Result<LocalUpdate<K, A>, Error> {	×
62	let new_txs = client.batch_transaction_get(&missing)?;	×
63	let mut graph_update = TxGraph::<A>::new(new_txs);	×
64	for (txid, anchors) in self.graph_update {	×
65	if let Some(seen_at) = seen_at {	×
66	let _ = graph_update.insert_seen_at(txid, seen_at);	×
67	}	×
68	for anchor in anchors {	×
69	let _ = graph_update.insert_anchor(txid, anchor);	×
70	}	×
71	}
72	Ok(LocalUpdate {	×
73	last_active_indices: self.keychain_update,	×
74	graph: graph_update,	×
75	chain: local_chain::Update {	×
76	tip: self.new_tip,	×
77	introduce_older_blocks: true,	×
78	},	×
79	})	×
80	}	×
81	}
82
83	impl<K> ElectrumUpdate<K, ConfirmationHeightAnchor> {
84	/// Finalizes the [`ElectrumUpdate`] with `new_txs` and anchors of type
85	/// [`ConfirmationTimeAnchor`].
86	///
87	/// Note: The confirmation time might not be precisely correct if there has been a reorg.
88	/// Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to
89	/// use it.
90	pub fn finalize_as_confirmation_time(	×
91	self,	×
92	client: &Client,	×
93	seen_at: Option<u64>,	×
94	missing: Vec<Txid>,	×
95	) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error> {	×
96	let update = self.finalize(client, seen_at, missing)?;	×
97
98	let relevant_heights = {	×
99	let mut visited_heights = HashSet::new();	×
100	update	×
101	.graph	×
102	.all_anchors()	×
103	.iter()	×
104	.map(\|(a, _)\| a.confirmation_height_upper_bound())	×
105	.filter(move \|&h\| visited_heights.insert(h))	×
106	.collect::<Vec<_>>()	×
107	};
108
109	let height_to_time = relevant_heights	×
110	.clone()	×
111	.into_iter()	×
112	.zip(	×
113	client	×
114	.batch_block_header(relevant_heights)?	×
115	.into_iter()	×
116	.map(\|bh\| bh.time as u64),	×
117	)	×
118	.collect::<HashMap<u32, u64>>();	×
119		×
120	let graph_additions = {	×
121	let old_additions = TxGraph::default().determine_additions(&update.graph);	×
122	tx_graph::Additions {	×
123	txs: old_additions.txs,	×
124	txouts: old_additions.txouts,	×
125	last_seen: old_additions.last_seen,	×
126	anchors: old_additions	×
127	.anchors	×
128	.into_iter()	×
129	.map(\|(height_anchor, txid)\| {	×
130	let confirmation_height = height_anchor.confirmation_height;	×
131	let confirmation_time = height_to_time[&confirmation_height];	×
132	let time_anchor = ConfirmationTimeAnchor {	×
133	anchor_block: height_anchor.anchor_block,	×
134	confirmation_height,	×
135	confirmation_time,	×
136	};	×
137	(time_anchor, txid)	×
138	})	×
139	.collect(),	×
140	}	×
141	};	×
142		×
143	Ok(LocalUpdate {	×
144	last_active_indices: update.last_active_indices,	×
145	graph: {	×
146	let mut graph = TxGraph::default();	×
147	graph.apply_additions(graph_additions);	×
148	graph	×
149	},	×
150	chain: update.chain,	×
151	})	×
152	}	×
153	}
154
155	/// Trait to extend [`Client`] functionality.
156	pub trait ElectrumExt<A> {
157	/// Scan the blockchain (via electrum) for the data specified and returns a [`ElectrumUpdate`].
158	///
159	/// - `prev_tip`: the most recent blockchain tip present locally
160	/// - `keychain_spks`: keychains that we want to scan transactions for
161	/// - `txids`: transactions for which we want updated [`Anchor`]s
162	/// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
163	/// want to included in the update
164	///
165	/// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
166	/// transactions. `batch_size` specifies the max number of script pubkeys to request for in a
167	/// single batch request.
168	fn scan<K: Ord + Clone>(
169	&self,
170	prev_tip: Option<CheckPoint>,
171	keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
172	txids: impl IntoIterator<Item = Txid>,
173	outpoints: impl IntoIterator<Item = OutPoint>,
174	stop_gap: usize,
175	batch_size: usize,
176	) -> Result<ElectrumUpdate<K, A>, Error>;
177
178	/// Convenience method to call [`scan`] without requiring a keychain.
179	///
180	/// [`scan`]: ElectrumExt::scan
181	fn scan_without_keychain(	×
182	&self,	×
183	prev_tip: Option<CheckPoint>,	×
184	misc_spks: impl IntoIterator<Item = ScriptBuf>,	×
185	txids: impl IntoIterator<Item = Txid>,	×
186	outpoints: impl IntoIterator<Item = OutPoint>,	×
187	batch_size: usize,	×
188	) -> Result<ElectrumUpdate<(), A>, Error> {	×
189	let spk_iter = misc_spks	×
190	.into_iter()	×
191	.enumerate()	×
192	.map(\|(i, spk)\| (i as u32, spk));	×
193		×
194	self.scan(	×
195	prev_tip,	×
196	[((), spk_iter)].into(),	×
197	txids,	×
198	outpoints,	×
199	usize::MAX,	×
200	batch_size,	×
201	)	×
202	}	×
203	}
204
205	impl ElectrumExt<ConfirmationHeightAnchor> for Client {
206	fn scan<K: Ord + Clone>(	×
207	&self,	×
208	prev_tip: Option<CheckPoint>,	×
209	keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,	×
210	txids: impl IntoIterator<Item = Txid>,	×
211	outpoints: impl IntoIterator<Item = OutPoint>,	×
212	stop_gap: usize,	×
213	batch_size: usize,	×
214	) -> Result<ElectrumUpdate<K, ConfirmationHeightAnchor>, Error> {	×
215	let mut request_spks = keychain_spks	×
216	.into_iter()	×
217	.map(\|(k, s)\| (k, s.into_iter()))	×
218	.collect::<BTreeMap<K, _>>();	×
219	let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new();	×
220		×
221	let txids = txids.into_iter().collect::<Vec<_>>();	×
222	let outpoints = outpoints.into_iter().collect::<Vec<_>>();	×
223
224	let update = loop {	×
225	let (tip, _) = construct_update_tip(self, prev_tip.clone())?;	×
226	let mut update = ElectrumUpdate::<K, ConfirmationHeightAnchor>::new(tip.clone());	×
227	let cps = update	×
228	.new_tip	×
229	.iter()	×
230	.take(10)	×
231	.map(\|cp\| (cp.height(), cp))	×
232	.collect::<BTreeMap<u32, CheckPoint>>();	×
233		×
234	if !request_spks.is_empty() {	×
235	if !scanned_spks.is_empty() {	×
236	scanned_spks.append(&mut populate_with_spks(	×
237	self,	×
238	&cps,	×
239	&mut update,	×
240	&mut scanned_spks	×
241	.iter()	×
242	.map(\|(i, (spk, _))\| (i.clone(), spk.clone())),	×
243	stop_gap,	×
244	batch_size,	×
245	)?);	×
246	}	×
247	for (keychain, keychain_spks) in &mut request_spks {	×
248	scanned_spks.extend(
249	populate_with_spks(	×
250	self,	×
251	&cps,	×
252	&mut update,	×
253	keychain_spks,	×
254	stop_gap,	×
255	batch_size,	×
256	)?	×
257	.into_iter()	×
258	.map(\|(spk_i, spk)\| ((keychain.clone(), spk_i), spk)),	×
259	);
260	}
261	}	×
262
263	populate_with_txids(self, &cps, &mut update, &mut txids.iter().cloned())?;	×
264
265	let _txs =	×
266	populate_with_outpoints(self, &cps, &mut update, &mut outpoints.iter().cloned())?;	×
267
268	// check for reorgs during scan process
269	let server_blockhash = self.block_header(tip.height() as usize)?.block_hash();	×
270	if tip.hash() != server_blockhash {	×
271	continue; // reorg	×
272	}	×
273		×
274	update.keychain_update = request_spks	×
275	.into_keys()	×
276	.filter_map(\|k\| {	×
277	scanned_spks	×
278	.range((k.clone(), u32::MIN)..=(k.clone(), u32::MAX))	×
279	.rev()	×
280	.find(\|(_, (_, active))\| *active)	×
281	.map(\|((_, i), _)\| (k, *i))	×
282	})	×
283	.collect::<BTreeMap<_, _>>();	×
284	break update;	×
285	};	×
286		×
287	Ok(update)	×
288	}	×
289	}
290
291	/// Return a [`CheckPoint`] of the latest tip, that connects with `prev_tip`.
292	fn construct_update_tip(	×
293	client: &Client,	×
294	prev_tip: Option<CheckPoint>,	×
295	) -> Result<(CheckPoint, Option<u32>), Error> {	×
296	let HeaderNotification { height, .. } = client.block_headers_subscribe()?;	×
297	let new_tip_height = height as u32;	×
298
299	// If electrum returns a tip height that is lower than our previous tip, then checkpoints do
300	// not need updating. We just return the previous tip and use that as the point of agreement.
301	if let Some(prev_tip) = prev_tip.as_ref() {	×
302	if new_tip_height < prev_tip.height() {	×
303	return Ok((prev_tip.clone(), Some(prev_tip.height())));	×
304	}	×
305	}	×
306
307	// Atomically fetch the latest `ASSUME_FINAL_DEPTH` count of blocks from Electrum. We use this
308	// to construct our checkpoint update.
309	let mut new_blocks = {	×
310	let start_height = new_tip_height.saturating_sub(ASSUME_FINAL_DEPTH);	×
311	let hashes = client	×
312	.block_headers(start_height as _, ASSUME_FINAL_DEPTH as _)?	×
313	.headers
314	.into_iter()	×
315	.map(\|h\| h.block_hash());	×
316	(start_height..).zip(hashes).collect::<BTreeMap<u32, _>>()	×
317	};
318
319	// Find the "point of agreement" (if any).
320	let agreement_cp = {	×
321	let mut agreement_cp = Option::<CheckPoint>::None;	×
322	for cp in prev_tip.iter().flat_map(CheckPoint::iter) {	×
323	let cp_block = cp.block_id();	×
324	let hash = match new_blocks.get(&cp_block.height) {	×
325	Some(&hash) => hash,	×
326	None => {
327	assert!(	×
328	new_tip_height >= cp_block.height,	×
329	"already checked that electrum's tip cannot be smaller"	×
330	);
331	let hash = client.block_header(cp_block.height as _)?.block_hash();	×
332	new_blocks.insert(cp_block.height, hash);	×
333	hash	×
334	}
335	};
336	if hash == cp_block.hash {	×
337	agreement_cp = Some(cp);	×
338	break;	×
339	}	×
340	}
341	agreement_cp	×
342	};	×
343		×
344	let agreement_height = agreement_cp.as_ref().map(CheckPoint::height);	×
345		×
346	let new_tip = new_blocks	×
347	.into_iter()	×
348	// Prune `new_blocks` to only include blocks that are actually new.	×
349	.filter(\|(height, _)\| Some(*height) > agreement_height)	×
350	.map(\|(height, hash)\| BlockId { height, hash })	×
351	.fold(agreement_cp, \|prev_cp, block\| {	×
352	Some(match prev_cp {	×
353	Some(cp) => cp.push(block).expect("must extend checkpoint"),	×
354	None => CheckPoint::new(block),	×
355	})
356	})	×
357	.expect("must have at least one checkpoint");	×
358		×
359	Ok((new_tip, agreement_height))	×
360	}	×
361
362	/// A [tx status] comprises of a concatenation of `tx_hash:height:`s. We transform a single one of
363	/// these concatenations into a [`ConfirmationHeightAnchor`] if possible.
364	///
365	/// We use the lowest possible checkpoint as the anchor block (from `cps`). If an anchor block
366	/// cannot be found, or the transaction is unconfirmed, [`None`] is returned.
367	///
368	/// [tx status](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status)
369	fn determine_tx_anchor(	×
370	cps: &BTreeMap<u32, CheckPoint>,	×
371	raw_height: i32,	×
372	txid: Txid,	×
373	) -> Option<ConfirmationHeightAnchor> {	×
374	// The electrum API has a weird quirk where an unconfirmed transaction is presented with a	×
375	// height of 0. To avoid invalid representation in our data structures, we manually set	×
376	// transactions residing in the genesis block to have height 0, then interpret a height of 0 as	×
377	// unconfirmed for all other transactions.	×
378	if txid	×
379	== Txid::from_str("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")	×
380	.expect("must deserialize genesis coinbase txid")	×
381	{
382	let anchor_block = cps.values().next()?.block_id();	×
383	return Some(ConfirmationHeightAnchor {	×
384	anchor_block,	×
385	confirmation_height: 0,	×
386	});	×
387	}	×
388	match raw_height {	×
389	h if h <= 0 => {	×
390	debug_assert!(h == 0 \|\| h == -1, "unexpected height ({}) from electrum", h);	×
391	None	×
392	}
393	h => {	×
394	let h = h as u32;	×
395	let anchor_block = cps.range(h..).next().map(\|(_, cp)\| cp.block_id())?;	×
396	if h > anchor_block.height {	×
397	None	×
398	} else {
399	Some(ConfirmationHeightAnchor {	×
400	anchor_block,	×
401	confirmation_height: h,	×
402	})	×
403	}
404	}
405	}
406	}	×
407
408	fn populate_with_outpoints<K>(	×
409	client: &Client,	×
410	cps: &BTreeMap<u32, CheckPoint>,	×
411	update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,	×
412	outpoints: &mut impl Iterator<Item = OutPoint>,	×
413	) -> Result<HashMap<Txid, Transaction>, Error> {	×
414	let mut full_txs = HashMap::new();	×
415	for outpoint in outpoints {	×
416	let txid = outpoint.txid;	×
417	let tx = client.transaction_get(&txid)?;	×
418	debug_assert_eq!(tx.txid(), txid);	×
419	let txout = match tx.output.get(outpoint.vout as usize) {	×
420	Some(txout) => txout,	×
421	None => continue,	×
422	};
423	// attempt to find the following transactions (alongside their chain positions), and
424	// add to our sparsechain `update`:
425	let mut has_residing = false; // tx in which the outpoint resides	×
426	let mut has_spending = false; // tx that spends the outpoint	×
427	for res in client.script_get_history(&txout.script_pubkey)? {	×
428	if has_residing && has_spending {	×
429	break;	×
430	}	×
431		×
432	if res.tx_hash == txid {	×
433	if has_residing {	×
434	continue;	×
435	}	×
436	has_residing = true;	×
437	full_txs.insert(res.tx_hash, tx.clone());	×
438	} else {
439	if has_spending {	×
440	continue;	×
441	}	×
442	let res_tx = match full_txs.get(&res.tx_hash) {	×
443	Some(tx) => tx,	×
444	None => {
445	let res_tx = client.transaction_get(&res.tx_hash)?;	×
446	full_txs.insert(res.tx_hash, res_tx);	×
447	full_txs.get(&res.tx_hash).expect("just inserted")	×
448	}
449	};
450	has_spending = res_tx	×
451	.input	×
452	.iter()	×
453	.any(\|txin\| txin.previous_output == outpoint);	×
454	if !has_spending {	×
455	continue;	×
456	}	×
457	};
458
459	let anchor = determine_tx_anchor(cps, res.height, res.tx_hash);	×
460		×
461	let tx_entry = update.graph_update.entry(res.tx_hash).or_default();	×
462	if let Some(anchor) = anchor {	×
463	tx_entry.insert(anchor);	×
464	}	×
465	}
466	}
467	Ok(full_txs)	×
468	}	×
469
470	fn populate_with_txids<K>(	×
471	client: &Client,	×
472	cps: &BTreeMap<u32, CheckPoint>,	×
473	update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,	×
474	txids: &mut impl Iterator<Item = Txid>,	×
475	) -> Result<(), Error> {	×
476	for txid in txids {	×
477	let tx = match client.transaction_get(&txid) {	×
478	Ok(tx) => tx,	×
479	Err(electrum_client::Error::Protocol(_)) => continue,	×
480	Err(other_err) => return Err(other_err),	×
481	};
482
483	let spk = tx	×
484	.output	×
485	.get(0)	×
486	.map(\|txo\| &txo.script_pubkey)	×
487	.expect("tx must have an output");	×
488
489	let anchor = match client	×
490	.script_get_history(spk)?	×
491	.into_iter()	×
492	.find(\|r\| r.tx_hash == txid)	×
493	{
494	Some(r) => determine_tx_anchor(cps, r.height, txid),	×
495	None => continue,	×
496	};
497
498	let tx_entry = update.graph_update.entry(txid).or_default();	×
499	if let Some(anchor) = anchor {	×
500	tx_entry.insert(anchor);	×
501	}	×
502	}
503	Ok(())	×
504	}	×
505
506	fn populate_with_spks<K, I: Ord + Clone>(	×
507	client: &Client,	×
508	cps: &BTreeMap<u32, CheckPoint>,	×
509	update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,	×
510	spks: &mut impl Iterator<Item = (I, ScriptBuf)>,	×
511	stop_gap: usize,	×
512	batch_size: usize,	×
513	) -> Result<BTreeMap<I, (ScriptBuf, bool)>, Error> {	×
514	let mut unused_spk_count = 0_usize;	×
515	let mut scanned_spks = BTreeMap::new();	×
516
517	loop {	×
518	let spks = (0..batch_size)	×
519	.map_while(\|_\| spks.next())	×
520	.collect::<Vec<_>>();	×
521	if spks.is_empty() {	×
522	return Ok(scanned_spks);	×
523	}	×
524
525	let spk_histories =	×
526	client.batch_script_get_history(spks.iter().map(\|(_, s)\| s.as_script()))?;	×
527
528	for ((spk_index, spk), spk_history) in spks.into_iter().zip(spk_histories) {	×
529	if spk_history.is_empty() {	×
530	scanned_spks.insert(spk_index, (spk, false));	×
531	unused_spk_count += 1;	×
532	if unused_spk_count > stop_gap {	×
533	return Ok(scanned_spks);	×
534	}	×
535	continue;	×
536	} else {	×
537	scanned_spks.insert(spk_index, (spk, true));	×
538	unused_spk_count = 0;	×
539	}	×
540
541	for tx in spk_history {	×
542	let tx_entry = update.graph_update.entry(tx.tx_hash).or_default();	×
543	if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) {	×
544	tx_entry.insert(anchor);	×
545	}	×
546	}
547	}
548	}
549	}	×

bitcoindevkit / bdk / 5758403852

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous