Skip to content

Commit

Permalink
feat: xml reader + ibkr flex query importer
Browse files Browse the repository at this point in the history
redstreet committed Jul 15, 2024
1 parent 1f1943c commit be7f43a
Showing 3 changed files with 228 additions and 1 deletion.
162 changes: 162 additions & 0 deletions beancount_reds_importers/importers/ibkr/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
"""IBKR Flex Query importer for beancount.
Activity Flex Query Details
Query ID XXX
Query Name XXX
Sections
========
Account Information
-------------------
1.ClientAccountID
2.CurrencyPrimary
Cash Transactions
-----------------
Options: Dividends, Payment in Lieu of Dividends, Withholding Tax, 871(m) Withholding, Advisor Fees, Other Fees, Deposits & Withdrawals, Carbon Credits, Bill Pay, Broker Interest Paid, Broker Interest Received, Broker Fees, Bond Interest Paid, Bond Interest Received, Price Adjustments, Commission Adjustments, Detail
1.Date/Time
2.Amount
3.Type
4.CurrencyPrimary
5.Symbol
6.CommodityType
7.ISIN
Net Stock Position Summary
--------------------------
1.Symbol
2.CUSIP
Open Dividend Accruals
----------------------
1.Symbol
2.GrossAmount
3.NetAmount
4.PayDate
5.Quantity
6.ISIN
Trades
------
Options: Execution
1.SecurityID
2.DateTime
3.TransactionType
4.Quantity
5.TradePrice
6.TradeMoney
7.Proceeds
8.IBCommission
9.IBCommissionCurrency
10.NetCash
11.CostBasis
12.FifoPnlRealized
13.Buy/Sell
14.CurrencyPrimary
15.ISIN
Delivery Configuration
----------------------
Accounts
Format XML
Period Last N Calendar Days
Number of Days 120
General Configuration
---------------------
Profit and Loss Default
Include Canceled Trades? No
Include Currency Rates? No
Include Audit Trail Fields? No
Display Account Alias in Place of Account ID? No
Breakout by Day? No
Date Format yyyy-MM-dd
Time Format HH:mm:ss TimeZone
Date/Time Separator ' ' (single-space)
"""

import datetime
from beancount_reds_importers.libreader import xmlreader
from beancount_reds_importers.libtransactionbuilder import investments
from beancount.core.number import D

class DictToObject:
def __init__(self, dictionary):
for key, value in dictionary.items():
setattr(self, key, value)

# xml on left, ofx on right
ofx_type_map = {
'BUY': 'buystock',
'SELL': 'selltock',
}


class Importer(investments.Importer, xmlreader.Importer):
IMPORTER_NAME = "IBKR Flex Query"

def custom_init(self):
if not self.custom_init_run:
self.max_rounding_error = 0.04
self.filename_pattern_def = "Transaction_report"
self.custom_init_run = True
self.date_format = '%Y-%m-%d'
self.get_ticker_info = self.get_ticker_info_from_id

def set_currency(self):
self.currency = list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['currency']

# fixup dates
def convert_date(self, d):
d = d.split(' ')[0]
return datetime.datetime.strptime(d, self.date_format)

def trade_to_ofx_dict(self, xml_data):
# Mapping the input dictionary to the OFX dictionary format
ofx_dict = {
'security': xml_data['isin'],
'tradeDate': self.convert_date(xml_data['dateTime']),
'memo': xml_data['transactionType'],
'type': ofx_type_map[xml_data['buySell']],
'units': D(xml_data['quantity']),
'unit_price': D(xml_data['tradePrice']),
'commission': D(xml_data['ibCommission']),
'total': D(xml_data['proceeds']),
}
return ofx_dict

def cash_to_ofx_dict(self, xml_data):
# Mapping the input dictionary to the OFX dictionary format
ofx_dict = {
'tradeDate': self.convert_date(xml_data['dateTime']),
'amount': D(xml_data['amount']),
'security': getattr(xml_data, 'isin', None),
'type': 'cash',
'memo': xml_data['type'],
}

if xml_data['type'] == 'Dividends':
ofx_dict['type'] = 'dividends'
ofx_dict['total'] = ofx_dict['amount']

return ofx_dict

def xml_trade_interpreter(self, element):
ot = self.trade_to_ofx_dict(element)
return DictToObject(ot)

def xml_cash_interpreter(self, element):
ot = self.cash_to_ofx_dict(element)
return DictToObject(ot)

def get_transactions(self):
yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade',
xml_interpreter=self.xml_trade_interpreter)
yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction',
xml_interpreter=self.xml_cash_interpreter)
9 changes: 8 additions & 1 deletion beancount_reds_importers/libreader/reader.py
Original file line number Diff line number Diff line change
@@ -24,9 +24,13 @@ def identify(self, file):
return False
self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED")
self.initialize_reader(file)
# print("reader_ready:", self.reader_ready)
# print("reader_ready:", self.reader_ready, self.IMPORTER_NAME)
return self.reader_ready

def set_currency(self):
"""For overriding"""
self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED")

def file_name(self, file):
return "{}".format(ntpath.basename(file.name))

@@ -55,5 +59,8 @@ def get_balance_statement(self, file=None):
def get_balance_positions(self):
return []

def get_balance_assertion_date(self):
return None

def get_available_cash(self, settlement_fund_balance=0):
return None
58 changes: 58 additions & 0 deletions beancount_reds_importers/libreader/xmlreader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
"""XML reader for beancount-reds-importers.
XML files have widely varying specifications, and thus, this is a very generic reader, and most of
the logic will have to be the institution specific readers.
"""

import datetime
import warnings
from collections import namedtuple
from lxml import etree

from beancount.ingest import importer
from beancount_reds_importers.libreader import reader



class Importer(reader.Reader, importer.ImporterProtocol):
FILE_EXTS = ["xml"]

def initialize_reader(self, file):
if getattr(self, "file", None) != file:
self.file = file
self.reader_ready = False
try:
self.xmltree = etree.parse(file.name)
except:
return
self.reader_ready = self.deep_identify()
if self.reader_ready:
self.set_currency()

def deep_identify(self):
"""For overriding by institution specific importer which can check if an account name
matches, and oother such things."""
return True

def file_date(self, file):
"""Get the ending date of the statement."""
if not getattr(self, "xmltree", None):
self.initialize(file)
# TODO:
return None

def read_file(self, file):
self.xmltree = etree.parse(file.name)

def get_xpath_elements(self, xpath_expr, xml_interpreter=lambda x: x):
"""Extract a list of elements in the XML file at the given XPath expression. Typically,
transactions are stored in an xml path, and this extracts them."""
elements = self.xmltree.xpath(xpath_expr)
for elem in elements:
yield xml_interpreter(elem.attrib)

def get_transactions(self):
"""/Transactions/Transaction is a dummy default path for transactions that needs to be
overriden in the institution specific importer."""
yield from self.get_xpath_elements("/Transactions/Transaction")

0 comments on commit be7f43a

Please sign in to comment.