Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support JSON File #107

Merged
merged 3 commits into from
Oct 6, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
refactor: formatting whole project
DEVNODEREACT committed Oct 6, 2024
commit 85080cd461aa021dba464d209003b1f55d71e26f
Original file line number Diff line number Diff line change
@@ -14,42 +14,43 @@ def custom_init(self):
self.filename_pattern_def = ".*History"
self.date_format = "%m/%d/%Y"
self.header_identifier = ""
self.column_labels_line = (
"Run Date,Action,Symbol,Description,Type,Quantity,Price ($),Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date"
)
self.column_labels_line = "Run Date,Action,Symbol,Description,Type,Quantity,Price ($),Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date"
self.header_map = {
"Run Date": "date",
"Action": "memo",
"Symbol": "security",
"Amount ($)": "amount",
"Settlement Date": "settleDate",
"Quantity": "units",
"Run Date": "date",
"Action": "memo",
"Symbol": "security",
"Amount ($)": "amount",
"Settlement Date": "settleDate",
"Quantity": "units",
"Accrued Interest ($)": "accrued_interest",
"Fees ($)": "fees",
"Commission ($)": "commission",
"Cash Balance ($)": "balance",
"Price ($)": "unit_price",
"Fees ($)": "fees",
"Commission ($)": "commission",
"Cash Balance ($)": "balance",
"Price ($)": "unit_price",
}
self.transaction_type_map = {
"DIVIDEND RECEIVED": "dividends",
"TRANSFERRED FROM": "cash",
"YOU BOUGHT": "buystock",
"YOU SOLD": "sellstock",
"DIVIDEND RECEIVED": "dividends",
"TRANSFERRED FROM": "cash",
"YOU BOUGHT": "buystock",
"YOU SOLD": "sellstock",
}
self.skip_transaction_types = []
# fmt: on

def deep_identify(self, file):
last_four = self.config.get("account_number", "")[-4:]
return re.match(self.header_identifier, file.head(), flags=re.DOTALL) and f"{last_four}" in file.name
return (
re.match(self.header_identifier, file.head(), flags=re.DOTALL)
and f"{last_four}" in file.name
)

def prepare_table(self, rdr):
for field in ["Action", "Symbol", "Description"]:
rdr = rdr.convert(field, lambda x: x.lstrip())

rdr = rdr.addfield("total", lambda x: x["Amount ($)"])
rdr = rdr.addfield("tradeDate", lambda x: x["Run Date"])
rdr = rdr.cutout('Type')
rdr = rdr.cutout("Type")
rdr = rdr.capture("Action", "(\\S+(?:\\s+\\S+)?)", ["type"], include_original=True)

# for field in ["memo"]:
113 changes: 71 additions & 42 deletions beancount_reds_importers/importers/ibkr/__init__.py
Original file line number Diff line number Diff line change
@@ -93,9 +93,11 @@
"""

import datetime

from beancount.core.number import D

from beancount_reds_importers.libreader import xmlreader
from beancount_reds_importers.libtransactionbuilder import investments
from beancount.core.number import D


class DictToObject:
@@ -106,8 +108,8 @@ def __init__(self, dictionary):

# xml on left, ofx on right
ofx_type_map = {
'BUY': 'buystock',
'SELL': 'selltock',
"BUY": "buystock",
"SELL": "selltock",
}


@@ -119,92 +121,119 @@ def custom_init(self):
self.max_rounding_error = 0.04
self.filename_pattern_def = "ibkr"
self.custom_init_run = True
self.date_format = '%Y-%m-%d'
self.date_format = "%Y-%m-%d"
self.get_ticker_info = self.get_ticker_info_from_id

def deep_identify(self, file):
try:
if self.config.get('account_number', None):
if self.config.get("account_number", None):
# account number specific matching
return self.config['account_number'] == list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['accountId']
return (
self.config["account_number"]
== list(
self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"
)
)[0]["accountId"]
)
else:
# base check: simply ensure this looks like a valid IBKR Flex Query file
return list(self.get_xpath_elements("/FlexQueryResponse"))[0] is not None
except IndexError:
return False

def set_currency(self):
self.currency = list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['currency']
self.currency = list(
self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"
)
)[0]["currency"]

# fixup dates
def convert_date(self, d):
d = d.split(' ')[0]
d = d.split(" ")[0]
return datetime.datetime.strptime(d, self.date_format)

def xml_transfer_interpreter(self, xml_data):
# map, with ofx fields on the left and xml fields on the right
ofx_dict = {
'security': xml_data['isin'],
'tradeDate': self.convert_date(xml_data['dateTime']),
'units': D(xml_data['quantity']),
'memo': 'Transfer in kind',
'type': 'transfer',
"security": xml_data["isin"],
"tradeDate": self.convert_date(xml_data["dateTime"]),
"units": D(xml_data["quantity"]),
"memo": "Transfer in kind",
"type": "transfer",
}
return DictToObject(ofx_dict)

def xml_trade_interpreter(self, xml_data):
# map, with ofx fields on the left and xml fields on the right
ofx_dict = {
'security': xml_data['isin'],
'tradeDate': self.convert_date(xml_data['dateTime']),
'memo': xml_data['transactionType'],
'type': ofx_type_map[xml_data['buySell']],
'units': D(xml_data['quantity']),
'unit_price': D(xml_data['tradePrice']),
'commission': -1 * D(xml_data['ibCommission']),
'total': D(xml_data['netCash']),
"security": xml_data["isin"],
"tradeDate": self.convert_date(xml_data["dateTime"]),
"memo": xml_data["transactionType"],
"type": ofx_type_map[xml_data["buySell"]],
"units": D(xml_data["quantity"]),
"unit_price": D(xml_data["tradePrice"]),
"commission": -1 * D(xml_data["ibCommission"]),
"total": D(xml_data["netCash"]),
}
return DictToObject(ofx_dict)

def xml_cash_interpreter(self, xml_data):
# map, with ofx fields on the left and xml fields on the right
ofx_dict = {
'tradeDate': self.convert_date(xml_data['dateTime']),
'amount': D(xml_data['amount']),
'security': xml_data.get('isin', None),
'type': 'cash',
'memo': xml_data['type'],
"tradeDate": self.convert_date(xml_data["dateTime"]),
"amount": D(xml_data["amount"]),
"security": xml_data.get("isin", None),
"type": "cash",
"memo": xml_data["type"],
}

if xml_data['type'] == 'Dividends':
ofx_dict['type'] = 'dividends'
ofx_dict['total'] = ofx_dict['amount']
if xml_data["type"] == "Dividends":
ofx_dict["type"] = "dividends"
ofx_dict["total"] = ofx_dict["amount"]

return DictToObject(ofx_dict)

def get_transactions(self):
yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade',
xml_interpreter=self.xml_trade_interpreter)
yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction',
xml_interpreter=self.xml_cash_interpreter)
yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Transfers/Transfer',
xml_interpreter=self.xml_transfer_interpreter)
yield from self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade",
xml_interpreter=self.xml_trade_interpreter,
)
yield from self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction",
xml_interpreter=self.xml_cash_interpreter,
)
yield from self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/Transfers/Transfer",
xml_interpreter=self.xml_transfer_interpreter,
)

def get_balance_assertion_date(self):
ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0]
return self.convert_date(ac['toDate']).date()
ac = list(
self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency"
)
)[0]
return self.convert_date(ac["toDate"]).date()

def get_available_cash(self, settlement_fund_balance=0):
"""Assumes there's only one cash currency.
TODO: get investments transaction builder to accept date from get_available_cash
"""
ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0]
return D(ac['slbNetCash'])
ac = list(
self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency"
)
)[0]
return D(ac["slbNetCash"])

def get_balance_positions(self):
for pos in self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/OpenPositions/OpenPosition'):
for pos in self.get_xpath_elements(
"/FlexQueryResponse/FlexStatements/FlexStatement/OpenPositions/OpenPosition"
):
balance = {
'security': pos['isin'],
'units': D(pos['position']),
"security": pos["isin"],
"units": D(pos["position"]),
}
yield DictToObject(balance)
32 changes: 16 additions & 16 deletions beancount_reds_importers/importers/ibkr/flexquery_download.py
Original file line number Diff line number Diff line change
@@ -1,33 +1,32 @@
#!/usr/bin/env python3
"""IBKR Flex Query Downloader"""

import requests
import click
import requests


@click.command()
@click.argument('token', required=True)
@click.argument('query_id', required=True)
@click.argument("token", required=True)
@click.argument("query_id", required=True)
def flexquery_download(token, query_id):
url = "https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.SendRequest"

url = (
"https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.SendRequest"
)

# Request Flex Query
request_payload = {
"v": "3",
"t": token,
"q": query_id
}

request_payload = {"v": "3", "t": token, "q": query_id}

response = requests.post(url, data=request_payload)

if response.status_code == 200:
request_id = response.text.split("<ReferenceCode>")[1].split("</ReferenceCode>")[0]
# print(f"Request ID: {request_id}")

# Construct URL to get the query result
result_url = f"https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.GetStatement?q={request_id}&t={token}&v=3"

result_response = requests.get(result_url)

if result_response.status_code == 200:
print(result_response.text)
else:
@@ -37,5 +36,6 @@ def flexquery_download(token, query_id):
print(f"Failed to request the query. Status Code: {response.status_code}")
return None

if __name__ == '__main__':

if __name__ == "__main__":
flexquery_download()
2 changes: 1 addition & 1 deletion beancount_reds_importers/importers/vanguard/__init__.py
Original file line number Diff line number Diff line change
@@ -45,7 +45,7 @@ def cleanup_memo(self, ot):
# some vanguard files have memos repeated like this:
# 'DIVIDEND REINVESTMENTDIVIDEND REINVESTMENT'
retval = ot.memo
if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2):]:
if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2) :]:
retval = ot.memo[: int(len(ot.memo) / 2)]
return retval

8 changes: 5 additions & 3 deletions beancount_reds_importers/libreader/jsonreader.py
Original file line number Diff line number Diff line change
@@ -10,16 +10,18 @@
import json

from beancount.ingest import importer

from beancount_reds_importers.libreader import reader


class Importer(reader.Reader, importer.ImporterProtocol):
FILE_EXTS = ["json"]

def initialize_reader(self, file):
if getattr(self, "file", None) != file:
self.file = file
self.reader_ready = False
with open(file.name, 'r') as f:
with open(file.name, "r") as f:
self.json_data = json.load(f)
self.reader_ready = self.deep_identify(file)
if self.reader_ready:
@@ -39,14 +41,14 @@ def file_date(self, file):
return None

def read_file(self, file):
with open(file.name, 'r') as f:
with open(file.name, "r") as f:
self.json_data = json.load(f)

def get_json_elements(self, json_path, json_interpreter=lambda x: x):
"""Extract a list of elements in the JSON file at the given JSON path. Typically,
transactions are stored in a JSON path, and this extracts them."""
elements = self.json_data
for key in json_path.split('.'):
for key in json_path.split("."):
if key in elements:
elements = elements[key]
else:
2 changes: 1 addition & 1 deletion beancount_reds_importers/libreader/xmlreader.py
Original file line number Diff line number Diff line change
@@ -5,9 +5,9 @@

"""

from beancount.ingest import importer
from lxml import etree

from beancount.ingest import importer
from beancount_reds_importers.libreader import reader


4 changes: 2 additions & 2 deletions beancount_reds_importers/libtransactionbuilder/investments.py
Original file line number Diff line number Diff line change
@@ -163,8 +163,8 @@ def get_ticker_info_from_id(self, security_id):
except IndexError:
print(f"Error: fund info not found for {security_id}", file=sys.stderr)
securities = self.get_security_list()
if '' in securities:
securities.remove('')
if "" in securities:
securities.remove("")
securities_missing = list(securities)
for s in securities:
for k in self.funds_db: