Better fee estimation

sdk/README.md

@@ -24,16 +24,19 @@ or
 npm i @gobob/bob-sdk
 ```
 
-## Building BOB sdk
+## Building BOB SDK
 
 ### Clone the repository
+
 Clone the repository and change to the `sdk` subfolder.
+
 ```shell
 git clone [email protected]:bob-collective/bob.git
 cd bob/sdk
 ```
 
 ### Install dependencies
+
 We use `pnpm` in the examples below. But the steps below should also work when using `yarn` or `npm` instead.
 
 ```shell

sdk/src/wallet/utxo.ts

@@ -39,6 +39,7 @@ export interface Input {
  * @param publicKey Optional public key needed if using P2SH-P2WPKH.
  * @param opReturnData Optional OP_RETURN data to include in an output.
  * @param confirmationTarget The number of blocks to include this tx (for fee estimation).
+ * @param utxoSelectionStrategy The strategy to use for selecting UTXOs. See https://github.com/paulmillr/scure-btc-signer/tree/main#utxo-selection for options.
  * @returns {Promise<string>} The Base64 encoded PSBT.
  */
 export async function createBitcoinPsbt(
@@ -47,11 +48,12 @@ export async function createBitcoinPsbt(
     amount: number,
     publicKey?: string,
     opReturnData?: string,
-    confirmationTarget: number = 3
+    confirmationTarget: number = 3,
+    utxoSelectionStrategy: string = 'default'
 ): Promise<string> {
     const addressInfo = getAddressInfo(fromAddress);
-    const network = addressInfo.network;
-    if (network === 'regtest') {
+
+    if (addressInfo.network === 'regtest') {
         throw new Error('Bitcoin regtest not supported');
     }
 
@@ -80,7 +82,7 @@ export async function createBitcoinPsbt(
         confirmedUtxos.map(async (utxo) => {
             const hex = await esploraClient.getTransactionHex(utxo.txid);
             const transaction = Transaction.fromRaw(Buffer.from(hex, 'hex'), { allowUnknownOutputs: true });
-            const input = getInputFromUtxoAndTx(network, utxo, transaction, addressInfo.type, publicKey);
+            const input = getInputFromUtxoAndTx(addressInfo.network, utxo, transaction, addressInfo.type, publicKey);
             possibleInputs.push(input);
         })
     );
@@ -108,12 +110,12 @@ export async function createBitcoinPsbt(
     // https://github.com/paulmillr/scure-btc-signer?tab=readme-ov-file#utxo-selection
     // default = exactBiggest/accumBiggest creates tx with smallest fees, but it breaks
     // big outputs to small ones, which in the end will create a lot of outputs close to dust.
-    const transaction = selectUTXO(possibleInputs, outputs, 'default', {
+    const transaction = selectUTXO(possibleInputs, outputs, utxoSelectionStrategy, {
         changeAddress: fromAddress, // Refund surplus to the payment address
         feePerByte: BigInt(Math.ceil(feeRate)), // round up to the nearest integer
         bip69: true, // Sort inputs and outputs according to BIP69
         createTx: true, // Create the transaction
-        network: getBtcNetwork(network),
+        network: getBtcNetwork(addressInfo.network),
         allowUnknownOutputs: true, // Required for OP_RETURN
         allowLegacyWitnessUtxo: true, // Required for P2SH-P2WPKH
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -179,3 +181,122 @@ export function getInputFromUtxoAndTx(
 
     return input;
 }
+
+
+/**
+ * Estimate the tx inclusion fee for a given address or public key with an optional OP_RETURN output.
+ *
+ * @param fromAddress The Bitcoin address which is sending to the `toAddress`.
+ * @param amount The amount of BTC (as satoshis) to send. If no amount is specified, the fee is estimated for all UTXOs, i.e., the max amount.
+ * @param publicKey Optional public key needed if using P2SH-P2WPKH.
+ * @param opReturnData Optional OP_RETURN data to include in an output.
+ * @param confirmationTarget The number of blocks to include this tx (for fee estimation).
+ * @param utxoSelectionStrategy The strategy to use for selecting UTXOs. See https://github.com/paulmillr/scure-btc-signer/tree/main#utxo-selection for options.
+ * @returns {Promise<number>} The fee amount for estiamted transaction inclusion in satoshis.
+ * 
+ * @dev Wtih no amount set, we estimate the fee for all UTXOs by trying to spend all inputs using strategy 'all'. If an amount is set, we use the 'default
+ * strategy to select the UTXOs.
+ */
+export async function estimateTxFee(
+    fromAddress: string,
+    amount?: number,
+    publicKey?: string,
+    opReturnData?: string,
+    confirmationTarget: number = 3,
+    utxoSelectionStrategy: string = 'default'
+): Promise<number> {
+    const addressInfo = getAddressInfo(fromAddress);
+
+    if (addressInfo.network === 'regtest') {
+        throw new Error('Bitcoin regtest not supported');
+    }
+
+    // We need the public key to generate the redeem and witness script to spend the scripts
+    if (addressInfo.type === (AddressType.p2sh || AddressType.p2wsh)) {
+        if (!publicKey) {
+            throw new Error('Public key is required to spend from the selected address type');
+        }
+    }
+
+    // Use the from address as the toAddress for the fee estimate
+    const toAddress = fromAddress;
+
+    // TODO: allow submitting the UTXOs, fee estimate and confirmed transactions
+    const esploraClient = new EsploraClient(addressInfo.network);
+
+    const [confirmedUtxos, feeRate] = await Promise.all([
+        esploraClient.getAddressUtxos(fromAddress),
+        esploraClient.getFeeEstimate(confirmationTarget),
+    ]);
+
+    if (confirmedUtxos.length === 0) {
+        throw new Error('No confirmed UTXOs');
+    }
+
+    // To construct the spending transaction and estimate the fee, we need the transactions for the UTXOs
+    const possibleInputs: Input[] = [];
+
+    await Promise.all(
+        confirmedUtxos.map(async (utxo) => {
+            const hex = await esploraClient.getTransactionHex(utxo.txid);
+            const transaction = Transaction.fromRaw(Buffer.from(hex, 'hex'), { allowUnknownOutputs: true });
+            const input = getInputFromUtxoAndTx(addressInfo.network, utxo, transaction, addressInfo.type, publicKey);
+            possibleInputs.push(input);
+        })
+    );
+
+    // Create transaction without outputs
+    const targetOutputs: Output[] = [
+        {
+            address: toAddress,
+            amount: BigInt(amount? amount : 0),
+        },
+    ];
+
+    if (opReturnData) {
+        // Strip 0x prefix from opReturn
+        if (opReturnData.startsWith('0x')) {
+            opReturnData = opReturnData.slice(2);
+        }
+        targetOutputs.push({
+            // OP_RETURN https://github.com/paulmillr/scure-btc-signer/issues/26
+            script: Script.encode(['RETURN', hex.decode(opReturnData)]),
+            amount: BigInt(0),
+        });
+    }
+
+    let outputs: Output[] = [];
+    if (amount === undefined) {
+        // Select all UTXOs if no amount is specified
+        // Add outputs to the transaction after all UTXOs are selected to prevent tx creation failures
+        utxoSelectionStrategy = 'all';
+    } else {
+        // Add the target outputs to the transaction
+        // Tx creation might fail if the requested amount is more than the available balance plus fees
+        outputs = targetOutputs;
+    }
+
+    // Outsource UTXO selection to btc-signer
+    // https://github.com/paulmillr/scure-btc-signer?tab=readme-ov-file#utxo-selection
+    // default = exactBiggest/accumBiggest creates tx with smallest fees, but it breaks
+    // big outputs to small ones, which in the end will create a lot of outputs close to dust.
+    const transaction = selectUTXO(possibleInputs, outputs, utxoSelectionStrategy, {
+        changeAddress: fromAddress, // Refund surplus to the payment address
+        feePerByte: BigInt(Math.ceil(feeRate)), // round up to the nearest integer
+        bip69: true, // Sort inputs and outputs according to BIP69
+        createTx: true, // Create the transaction
+        network: getBtcNetwork(addressInfo.network),
+        allowUnknownOutputs: true, // Required for OP_RETURN
+        allowLegacyWitnessUtxo: true, // Required for P2SH-P2WPKH
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        dust: BigInt(546) as any, // TODO: update scure-btc-signer
+    });
+
+    // Add the target outputs after the fact
+    if (amount === undefined) {
+        transaction.tx.addOutput(targetOutputs[0]);
+        transaction.tx.addOutput(targetOutputs[1]);
+    }
+
+    return transaction.fee;
+}

sdk/test/esplora.test.ts

@@ -130,9 +130,9 @@ describe('Esplora Tests', () => {
     });
 
     it('should get fee rate', async () => {
-        const client = new EsploraClient('testnet');
+        const client = new EsploraClient('mainnet');
         const feeRate = await client.getFeeEstimate(1);
-        assert.isAtLeast(feeRate, 1);
+        assert(feeRate > 0);
     });
 
     it('should get balance', async () => {

sdk/test/ordinal-api.test.ts

@@ -72,7 +72,7 @@ describe('Ordinal API Tests', () => {
         const expectedOutputJson: OutputJson = {
             value: 10737,
             script_pubkey:
-                'OP_PUSHNUM_1 OP_PUSHBYTES_32 e18a5367c5d11ee31d10bf4c53e743a7479c70e3336e70dbdea1fd927305c022',
+                '5120e18a5367c5d11ee31d10bf4c53e743a7479c70e3336e70dbdea1fd927305c022',
             address: 'bc1pux99xe796y0wx8gshax98e6r5arecu8rxdh8pk77587eyuc9cq3q2e3nng',
             transaction: 'dfe942a58b7e29a3952d8d1ed6608086c66475d20bc7bdbc3d784d616f9a6a7a',
             sat_ranges: null,

sdk/test/utxo.test.ts

@@ -2,7 +2,7 @@ import { describe, it, assert } from 'vitest';
 import { AddressType, getAddressInfo, Network } from 'bitcoin-address-validation';
 import { Address, NETWORK, OutScript, Script, Transaction, p2sh, p2wpkh, selectUTXO } from '@scure/btc-signer';
 import { hex, base64 } from '@scure/base';
-import { createBitcoinPsbt, getInputFromUtxoAndTx } from '../src/wallet/utxo';
+import { createBitcoinPsbt, getInputFromUtxoAndTx, estimateTxFee } from '../src/wallet/utxo';
 import { TransactionOutput } from '@scure/btc-signer/psbt';
 
 // TODO: Add more tests using https://github.com/paulmillr/scure-btc-signer/tree/5ead71ea9a873d8ba1882a9cd6aa561ad410d0d1/test/bitcoinjs-test/fixtures/bitcoinjs
@@ -49,6 +49,8 @@ describe('UTXO Tests', () => {
                             // Use a random public key for P2SH-P2WPKH
                             pubkey = '03b366c69e8237d9be7c4f1ac2a7abc6a79932fbf3de4e2f6c04797d7ef27abfe1';
                         }
+                        // Note: it is possible that the above addresses have spent all of their funds
+                        // and the transaction will fail.
                         const psbtBase64 = await createBitcoinPsbt(paymentAddress, toAddress, amount, pubkey, opReturn);
                         const transaction = Transaction.fromPSBT(base64.decode(psbtBase64));
 
@@ -258,4 +260,41 @@ describe('UTXO Tests', () => {
 
         assert.isDefined(transaction);
     });
+
+    it('should estimate the fee for a transaction', async () => {
+        // Addresses where randomly picked from blockstream.info
+        const paymentAddresses = [
+            // P2WPKH: https://blockstream.info/address/bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq
+            'bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq',
+            // P2SH-P2WPKH: https://blockstream.info/address/3DFVKuT9Ft4rWpysAZ1bHpg55EBy1HVPcr
+            '3DFVKuT9Ft4rWpysAZ1bHpg55EBy1HVPcr',
+            // P2PKH: https://blockstream.info/address/1Kr6QSydW9bFQG1mXiPNNu6WpJGmUa9i1g
+            '1Kr6QSydW9bFQG1mXiPNNu6WpJGmUa9i1g',
+        ];
+
+        const amounts = [undefined, 2000, 3000];
+
+        // EVM address for OP return
+        let opReturn = '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2';
+
+        // Refactor to execute in parallel
+        await Promise.all(
+            amounts.map(async (amount) => Promise.all(
+                paymentAddresses.map(async (paymentAddress) => {
+                    const paymentAddressType = getAddressInfo(paymentAddress).type;
+
+                    let pubkey: string | undefined;
+
+                    if (paymentAddressType === AddressType.p2sh) {
+                        // Use a random public key for P2SH-P2WPKH
+                        pubkey = '03b366c69e8237d9be7c4f1ac2a7abc6a79932fbf3de4e2f6c04797d7ef27abfe1';
+                    }
+
+                    // If the amount is undefined, the fee should be estimated
+                    const fee = await estimateTxFee(paymentAddress, undefined, pubkey, opReturn);
+                    assert(fee > 0, 'Fee should be greater than 0');
+                }
+            )
+        )))
+    });
 });