Better fee estimation

sdk/README.md

@@ -24,16 +24,19 @@ or
 npm i @gobob/bob-sdk
 ```
 
-## Building BOB sdk
+## Building BOB SDK
 
 ### Clone the repository
+
 Clone the repository and change to the `sdk` subfolder.
+
 ```shell
 git clone [email protected]:bob-collective/bob.git
 cd bob/sdk
 ```
 
 ### Install dependencies
+
 We use `pnpm` in the examples below. But the steps below should also work when using `yarn` or `npm` instead.
 
 ```shell

sdk/src/gateway/client.ts

@@ -203,6 +203,7 @@ export class GatewayApiClient {
                 gatewayQuote.satoshis,
                 params.fromUserPublicKey,
                 data.opReturnHash,
+                params.feeRate,
                 gatewayQuote.txProofDifficultyFactor
             );
         }

sdk/src/gateway/types.ts

@@ -60,6 +60,8 @@ export interface GatewayQuoteParams {
     /** @description The percentage of fee charged by partners in Basis Points (BPS) units. This will override the default fee rate configured via platform. 1 BPS = 0.01%. The maximum value is 1000 (which equals 10%). The minimum value is 1 (which equals 0.01%). */
     fee?: number;
 
+    feeRate?: number;
+
     // NOTE: the following are new fields added by us
     /** @description Amount of satoshis to swap for ETH */
     gasRefill?: number;

sdk/src/utils.ts

@@ -14,11 +14,6 @@ import { hash256 } from 'bitcoinjs-lib/src/crypto';
  * @ignore
  */
 import { Output, Transaction } from 'bitcoinjs-lib/src/transaction';
-/**
- * @ignore
- */
-import * as bitcoin from 'bitcoinjs-lib';
-
 /**
  * @ignore
  */
@@ -187,25 +182,3 @@ export function getMerkleProof(block: Block, txHash: string, forWitness?: boolea
         root: merkleAndRoot.root.toString('hex'),
     };
 }
-
-/**
- * Estimate the tx inclusion fee for N P2WPKH inputs and 3 P2WPKH outputs.
- *
- * @param feeRate - The current rate for inclusion, satoshi per byte.
- * @param numInputs - The number of inputs to estimate for.
- * @returns The estimated fee for transaction inclusion.
- */
-export function estimateTxFee(feeRate: number, numInputs: number = 1) {
-    const tx = new bitcoin.Transaction();
-    for (let i = 0; i < numInputs; i++) {
-        tx.addInput(Buffer.alloc(32, 0), 0, 0xfffffffd, Buffer.alloc(0));
-    }
-    // https://github.com/interlay/interbtc-clients/blob/6bd3e81d695b93180c5aeae4f33910ad4395ff1a/bitcoin/src/light/wallet.rs#L80
-    tx.ins.map((tx_input) => (tx_input.witness = [Buffer.alloc(33 + 32 + 7, 0), Buffer.alloc(33, 0)]));
-    tx.addOutput(Buffer.alloc(22, 0), 1000); // P2WPKH
-    tx.addOutput(Buffer.alloc(22, 0), 1000); // P2WPKH (change)
-    tx.addOutput(bitcoin.script.compile([bitcoin.opcodes.OP_RETURN, Buffer.alloc(20, 0)]), 0);
-    const vsize = tx.virtualSize();
-    const satoshis = feeRate * vsize;
-    return satoshis;
-}

sdk/src/wallet/utxo.ts

@@ -38,20 +38,40 @@ export interface Input {
  * @param amount The amount of BTC (as satoshis) to send.
  * @param publicKey Optional public key needed if using P2SH-P2WPKH.
  * @param opReturnData Optional OP_RETURN data to include in an output.
+ * @param feeRate Optional fee rate in satoshis per byte.
  * @param confirmationTarget The number of blocks to include this tx (for fee estimation).
  * @returns {Promise<string>} The Base64 encoded PSBT.
+ *
+ * @example
+ * ```typescript
+ * const fromAddress = 'bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq';
+ * const toAddress = 'bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq';
+ * const amount = 100000;
+ * const publicKey = '02d4...`; // only for P2SH
+ * const opReturnData = 'Hello, World!'; // optional
+ * const confirmationTarget = 3; // optional
+ *
+ * const psbt = await createBitcoinPsbt(fromAddress, toAddress, amount, publicKey, opReturnData, confirmationTarget);
+ * console.log(psbt);
+ *
+ * // The PSBT can then be signed with the private key using sats-wagmi, sats-connect, ...
+ * ```
  */
 export async function createBitcoinPsbt(
     fromAddress: string,
     toAddress: string,
     amount: number,
     publicKey?: string,
     opReturnData?: string,
+    feeRate?: number,
     confirmationTarget: number = 3
 ): Promise<string> {
     const addressInfo = getAddressInfo(fromAddress);
-    const network = addressInfo.network;
-    if (network === 'regtest') {
+
+    // TODO: possibly, allow other strategies to be passed to this function
+    const utxoSelectionStrategy = 'default';
+
+    if (addressInfo.network === 'regtest') {
         throw new Error('Bitcoin regtest not supported');
     }
 
@@ -64,10 +84,16 @@ export async function createBitcoinPsbt(
 
     const esploraClient = new EsploraClient(addressInfo.network);
 
-    const [confirmedUtxos, feeRate] = await Promise.all([
-        esploraClient.getAddressUtxos(fromAddress),
-        esploraClient.getFeeEstimate(confirmationTarget),
-    ]);
+    let confirmedUtxos: UTXO[] = [];
+
+    if (feeRate) {
+        confirmedUtxos = await esploraClient.getAddressUtxos(fromAddress);
+    } else {
+        [confirmedUtxos, feeRate] = await Promise.all([
+            esploraClient.getAddressUtxos(fromAddress),
+            esploraClient.getFeeEstimate(confirmationTarget),
+        ]);
+    }
 
     if (confirmedUtxos.length === 0) {
         throw new Error('No confirmed UTXOs');
@@ -80,7 +106,13 @@ export async function createBitcoinPsbt(
         confirmedUtxos.map(async (utxo) => {
             const hex = await esploraClient.getTransactionHex(utxo.txid);
             const transaction = Transaction.fromRaw(Buffer.from(hex, 'hex'), { allowUnknownOutputs: true });
-            const input = getInputFromUtxoAndTx(network, utxo, transaction, addressInfo.type, publicKey);
+            const input = getInputFromUtxoAndTx(
+                addressInfo.network as BitcoinNetworkName,
+                utxo,
+                transaction,
+                addressInfo.type,
+                publicKey
+            );
             possibleInputs.push(input);
         })
     );
@@ -108,19 +140,22 @@ export async function createBitcoinPsbt(
     // https://github.com/paulmillr/scure-btc-signer?tab=readme-ov-file#utxo-selection
     // default = exactBiggest/accumBiggest creates tx with smallest fees, but it breaks
     // big outputs to small ones, which in the end will create a lot of outputs close to dust.
-    const transaction = selectUTXO(possibleInputs, outputs, 'default', {
+    const transaction = selectUTXO(possibleInputs, outputs, utxoSelectionStrategy, {
         changeAddress: fromAddress, // Refund surplus to the payment address
         feePerByte: BigInt(Math.ceil(feeRate)), // round up to the nearest integer
         bip69: true, // Sort inputs and outputs according to BIP69
         createTx: true, // Create the transaction
-        network: getBtcNetwork(network),
+        network: getBtcNetwork(addressInfo.network),
         allowUnknownOutputs: true, // Required for OP_RETURN
         allowLegacyWitnessUtxo: true, // Required for P2SH-P2WPKH
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         dust: BigInt(546) as any, // TODO: update scure-btc-signer
     });
 
     if (!transaction || !transaction.tx) {
+        console.debug(`fromAddress: ${fromAddress}, toAddress: ${toAddress}, amount: ${amount}`);
+        console.debug(`publicKey: ${publicKey}, opReturnData: ${opReturnData}`);
+        console.debug(`feeRate: ${feeRate}, confirmationTarget: ${confirmationTarget}`);
         throw new Error('Failed to create transaction. Do you have enough funds?');
     }
 
@@ -179,3 +214,165 @@ export function getInputFromUtxoAndTx(
 
     return input;
 }
+
+/**
+ * Estimate the tx inclusion fee for a given address or public key with an optional OP_RETURN output.
+ *
+ * @param fromAddress The Bitcoin address which is sending to the `toAddress`.
+ * @param amount The amount of BTC (as satoshis) to send. If no amount is specified, the fee is estimated for all UTXOs, i.e., the max amount.
+ * @param publicKey Optional public key needed if using P2SH-P2WPKH.
+ * @param opReturnData Optional OP_RETURN data to include in an output.
+ * @param feeRate Optional fee rate in satoshis per byte.
+ * @param confirmationTarget The number of blocks to include this tx (for fee estimation).
+ * @returns {Promise<bigint>} The fee amount for estimated transaction inclusion in satoshis.
+ *
+ * @example
+ * ```typescript
+ * // Using a target amount (call might fail if amount is larger than balance plus fees)
+ * const fromAddress = 'bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq';
+ * const amount = 100000;
+ * const publicKey = '02d4...`; // only for P2SH
+ * const opReturnData = 'Hello, World!'; // optional
+ * const feeRate = 1; // optional
+ * const confirmationTarget = 3; // optional
+ *
+ * const fee = await estimateTxFee(fromAddress, amount, publicKey, opReturnData, feeRate, confirmationTarget);
+ * console.log(fee);
+ *
+ * // Using all UTXOs without a target amount (max fee for spending all UTXOs)
+ * const fromAddress = 'bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq';
+ * const publicKey = '02d4...`; // only for P2SH
+ * const opReturnData = 'Hello, World!'; // optional
+ * const feeRate = 1; // optional
+ *
+ * const fee = await estimateTxFee(fromAddress, undefined, publicKey, opReturnData, feeRate);
+ * console.log(fee);
+ * ```
+ *
+ * @dev Wtih no amount set, we estimate the fee for all UTXOs by trying to spend all inputs using strategy 'all'. If an amount is set, we use the 'default
+ * strategy to select the UTXOs.
+ */
+export async function estimateTxFee(
+    fromAddress: string,
+    amount?: number,
+    publicKey?: string,
+    opReturnData?: string,
+    feeRate?: number,
+    confirmationTarget: number = 3
+): Promise<bigint> {
+    const addressInfo = getAddressInfo(fromAddress);
+
+    if (addressInfo.network === 'regtest') {
+        throw new Error('Bitcoin regtest not supported');
+    }
+
+    // We need the public key to generate the redeem and witness script to spend the scripts
+    if (addressInfo.type === (AddressType.p2sh || AddressType.p2wsh)) {
+        if (!publicKey) {
+            throw new Error('Public key is required to spend from the selected address type');
+        }
+    }
+
+    // Use the from address as the toAddress for the fee estimate
+    const toAddress = fromAddress;
+
+    // TODO: allow submitting the UTXOs, fee estimate and confirmed transactions
+    // to avoid fetching them again.
+    const esploraClient = new EsploraClient(addressInfo.network);
+
+    let confirmedUtxos: UTXO[] = [];
+    if (feeRate) {
+        confirmedUtxos = await esploraClient.getAddressUtxos(fromAddress);
+    } else {
+        [confirmedUtxos, feeRate] = await Promise.all([
+            esploraClient.getAddressUtxos(fromAddress),
+            esploraClient.getFeeEstimate(confirmationTarget),
+        ]);
+    }
+
+    if (confirmedUtxos.length === 0) {
+        throw new Error('No confirmed UTXOs');
+    }
+
+    // To construct the spending transaction and estimate the fee, we need the transactions for the UTXOs
+    const possibleInputs: Input[] = [];
+
+    await Promise.all(
+        confirmedUtxos.map(async (utxo) => {
+            const hex = await esploraClient.getTransactionHex(utxo.txid);
+            const transaction = Transaction.fromRaw(Buffer.from(hex, 'hex'), { allowUnknownOutputs: true });
+            const input = getInputFromUtxoAndTx(
+                addressInfo.network as BitcoinNetworkName,
+                utxo,
+                transaction,
+                addressInfo.type,
+                publicKey
+            );
+            possibleInputs.push(input);
+        })
+    );
+
+    // Create transaction without outputs
+    const targetOutputs: Output[] = [
+        {
+            address: toAddress,
+            amount: BigInt(amount ? amount : 0),
+        },
+    ];
+
+    if (opReturnData) {
+        // Strip 0x prefix from opReturn
+        if (opReturnData.startsWith('0x')) {
+            opReturnData = opReturnData.slice(2);
+        }
+        targetOutputs.push({
+            // OP_RETURN https://github.com/paulmillr/scure-btc-signer/issues/26
+            script: Script.encode(['RETURN', hex.decode(opReturnData)]),
+            amount: BigInt(0),
+        });
+    }
+
+    let outputs: Output[] = [];
+    // Select all UTXOs if no amount is specified
+    // Add outputs to the transaction after all UTXOs are selected to prevent tx creation failures
+    let utxoSelectionStrategy = 'all';
+    if (amount) {
+        // Add the target outputs to the transaction
+        // Tx creation might fail if the requested amount is more than the available balance plus fees
+        // TODO: allow passing other UTXO selection strategies for fee etimates
+        utxoSelectionStrategy = 'default';
+        outputs = targetOutputs;
+    }
+
+    // Outsource UTXO selection to btc-signer
+    // https://github.com/paulmillr/scure-btc-signer?tab=readme-ov-file#utxo-selection
+    // default = exactBiggest/accumBiggest creates tx with smallest fees, but it breaks
+    // big outputs to small ones, which in the end will create a lot of outputs close to dust.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const transaction = selectUTXO(possibleInputs, outputs, utxoSelectionStrategy as any, {
+        changeAddress: fromAddress, // Refund surplus to the payment address
+        feePerByte: BigInt(Math.ceil(feeRate)), // round up to the nearest integer
+        bip69: true, // Sort inputs and outputs according to BIP69
+        createTx: true, // Create the transaction
+        network: getBtcNetwork(addressInfo.network),
+        allowUnknownOutputs: true, // Required for OP_RETURN
+        allowLegacyWitnessUtxo: true, // Required for P2SH-P2WPKH
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        dust: BigInt(546) as any, // TODO: update scure-btc-signer
+    });
+
+    if (!transaction || !transaction.tx) {
+        console.debug(`fromAddress: ${fromAddress}, amount: ${amount}`);
+        console.debug(`publicKey: ${publicKey}, opReturnData: ${opReturnData}`);
+        console.debug(`feeRate: ${feeRate}, confirmationTarget: ${confirmationTarget}`);
+        throw new Error('Failed to create transaction. Do you have enough funds?');
+    }
+
+    // Add the target outputs after the fact
+    if (amount === undefined) {
+        transaction.tx.addOutput(targetOutputs[0]);
+        transaction.tx.addOutput(targetOutputs[1]);
+    }
+
+    return transaction.fee;
+}

sdk/test/esplora.test.ts

@@ -130,9 +130,9 @@ describe('Esplora Tests', () => {
     });
 
     it('should get fee rate', async () => {
-        const client = new EsploraClient('testnet');
+        const client = new EsploraClient('mainnet');
         const feeRate = await client.getFeeEstimate(1);
-        assert.isAtLeast(feeRate, 1);
+        assert(feeRate > 0);
     });
 
     it('should get balance', async () => {

sdk/test/ordinal-api.test.ts

@@ -71,8 +71,7 @@ describe('Ordinal API Tests', () => {
         });
         const expectedOutputJson: OutputJson = {
             value: 10737,
-            script_pubkey:
-                'OP_PUSHNUM_1 OP_PUSHBYTES_32 e18a5367c5d11ee31d10bf4c53e743a7479c70e3336e70dbdea1fd927305c022',
+            script_pubkey: '5120e18a5367c5d11ee31d10bf4c53e743a7479c70e3336e70dbdea1fd927305c022',
             address: 'bc1pux99xe796y0wx8gshax98e6r5arecu8rxdh8pk77587eyuc9cq3q2e3nng',
             transaction: 'dfe942a58b7e29a3952d8d1ed6608086c66475d20bc7bdbc3d784d616f9a6a7a',
             sat_ranges: null,

sdk/test/utils.test.ts

@@ -1,7 +1,7 @@
+import { Block } from 'bitcoinjs-lib';
 import { assert, describe, it } from 'vitest';
 import { MAINNET_ESPLORA_BASE_PATH } from '../src/esplora';
-import { Block } from 'bitcoinjs-lib';
-import { estimateTxFee, getMerkleProof } from '../src/utils';
+import { getMerkleProof } from '../src/utils';
 
 describe('Utils Tests', () => {
     // NOTE: this is a bit flaky due to slow response times from electrs
@@ -19,8 +19,4 @@ describe('Utils Tests', () => {
             root: '7cee5e99c8f0fc25fb115b7d7d00befca61f59a8544adaf3980f52132baf61ae',
         });
     });
-
-    it('should estimate fee', async () => {
-        assert.equal(estimateTxFee(1), 172);
-    });
 });