From 53e4d488f94c98af24e713123a8ceb684cc76dd8 Mon Sep 17 00:00:00 2001 From: Jacob Farkas Date: Wed, 13 Sep 2023 22:53:25 -0700 Subject: [PATCH 1/5] test: Add a unit test for the Schwab balance extractor --- .../importers/schwab/schwab_csv_balances.py | 2 +- .../schwab_csv_Balances_test.py | 59 +++++++++++++++++++ .../schwab_csv_brokerage_Balances_123.csv | 18 ++++++ ...wab_csv_brokerage_Balances_123.csv.extract | 11 ++++ ...sv_brokerage_Balances_123.csv.file_account | 1 + ...b_csv_brokerage_Balances_123.csv.file_date | 1 + ...b_csv_brokerage_Balances_123.csv.file_name | 1 + 7 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.extract create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_account create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_date create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_name diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py index 561edc0..21661e6 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py @@ -13,7 +13,7 @@ class Importer(investments.Importer, csv_multitable_reader.Importer): def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = '.*_Balances_' - self.header_identifier = 'Balances for account' + self.header_identifier = '"Balances for account.*' self.get_ticker_info = self.get_ticker_info_from_id self.date_format = '%m/%d/%Y' self.funds_db_txt = 'funds_by_ticker' diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py new file mode 100644 index 0000000..9138a25 --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py @@ -0,0 +1,59 @@ +# flake8: noqa + +from os import path +from beancount.ingest import regression_pytest as regtest +from beancount_reds_importers.importers.schwab import schwab_csv_balances + + +fund_data = [ + ('MMM', '123', '3M INC'), + ('BND', '789', 'Vanguard Total Bond Market Index Fund'), + ('PP', '456', 'PIED PIPER INC'), + ('VMMX', '789', 'VANGUARD MONEY MARKET'), + ('VMMX2', '901', 'VANGUARD MONEY MARKET MONEY SHARES'), + ('HOOLI', '234', 'HOOLI MONEY MARKET'), +] + +# list of money_market accounts. These will not be held at cost, and instead will use price conversions +money_market = ['VMMX','VMMX2','HOOLI'] + +fund_info = { + 'fund_data': fund_data, + 'money_market': money_market, + } + + +def build_config(): + acct = "Assets:Investments:Schwab" + root = 'Investments' + taxability = 'Taxable' + leaf = 'Schwab' + currency = 'USD' + config = { + 'account_number' : 9876, + 'main_account' : acct + ':{ticker}', + 'cash_account' : f'{acct}:{{currency}}', + 'transfer' : 'Assets:Zero-Sum-Accounts:Transfers:Bank-Account', + 'dividends' : f'Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}', + 'interest' : f'Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}', + 'cg' : f'Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}', + 'capgainsd_lt' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}', + 'capgainsd_st' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}', + 'fees' : f'Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}', + 'invexpense' : f'Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}', + 'rounding_error' : 'Equity:Rounding-Errors:Imports', + 'fund_info' : fund_info, + 'currency' : currency, + 'section_headers': ['Stocks', 'Bonds', 'Money Market'] + } + return config + + +@regtest.with_importer( + schwab_csv_balances.Importer( + build_config() + ) +) +@regtest.with_testdir(path.dirname(__file__)) +class TestSchwabCSV(regtest.ImporterTestBase): + pass diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv new file mode 100644 index 0000000..ccb981d --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv @@ -0,0 +1,18 @@ +"Balances for account General Investing ...XXX as of 05/03/2023" +"Stocks" +"Symbol","Description","Quantity","Price" +"MMM","3M INC","656","$16,516.92" + +"Bonds" +"Symbol","Description","Quantity","Price" +"BND","VANGUARD TOTAL BOND MARKET ETF","45","$3320.05" + +"Ignored" +"Symbol","Description","Quantity","Price" +"PP","PIED PIPER INC","62","$51.95" + +"Money Market" +"Symbol","Description","Quantity","Price" +"VMMX","VANGUARD MONEY MARKET","6225","$6225" +"VMMX2","VANGUARD MONEY MARKET MONEY SHARES","129","$5952" +"HOOLI","HOOLI MONEY MARKET","4591","$4591" \ No newline at end of file diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.extract b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.extract new file mode 100644 index 0000000..d84bf0c --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.extract @@ -0,0 +1,11 @@ +2023-05-03 price MMM 16516.92 USD +2023-05-03 price BND 3320.05 USD +2023-05-03 price VMMX 6225 USD +2023-05-03 price VMMX2 5952 USD +2023-05-03 price HOOLI 4591 USD + +2023-05-04 balance Assets:Investments:Schwab:MMM 656 MMM +2023-05-04 balance Assets:Investments:Schwab:BND 45 BND +2023-05-04 balance Assets:Investments:Schwab:VMMX 6225 VMMX +2023-05-04 balance Assets:Investments:Schwab:VMMX2 129 VMMX2 +2023-05-04 balance Assets:Investments:Schwab:HOOLI 4591 HOOLI diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_account b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_account new file mode 100644 index 0000000..4e2b2c2 --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_account @@ -0,0 +1 @@ +Assets:Investments:Schwab diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_date b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_date new file mode 100644 index 0000000..c285090 --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_date @@ -0,0 +1 @@ +2023-05-03 diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_name b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_name new file mode 100644 index 0000000..df463c2 --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_brokerage_Balances_123.csv.file_name @@ -0,0 +1 @@ +schwab_csv_brokerage_Balances_123.csv From bf193bd956286ce743bfdcc6015ab8fa145a25d5 Mon Sep 17 00:00:00 2001 From: Jacob Farkas Date: Wed, 13 Sep 2023 01:35:30 -0700 Subject: [PATCH 2/5] feat(minor): Add decimal precision to whole numbers to avoid rounding errors in beancount --- beancount_reds_importers/libreader/csvreader.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index c03974a..078fc1b 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -108,10 +108,16 @@ def convert_columns(self, rdr): # fixup currencies def remove_non_numeric(x): return re.sub("[^0-9\.-]", "", str(x).strip()) # noqa: W605 + def add_decimal(x): + if '.' not in x: + return x+".00" + return x currencies = getattr(self, 'currency_fields', []) + ['unit_price', 'fees', 'total', 'amount', 'balance'] for i in currencies: if i in rdr.header(): rdr = rdr.convert(i, remove_non_numeric) + if self.config.get('add_currency_precision', False): + rdr = rdr.convert(i, add_decimal) rdr = rdr.convert(i, D) # fixup dates From 4904480686546f0fcb1686ea5db4b5508fb420ec Mon Sep 17 00:00:00 2001 From: Jacob Farkas Date: Wed, 13 Sep 2023 00:15:24 -0700 Subject: [PATCH 3/5] feat: Add an importer for Vanguard 529 CSV data --- .../importers/schwab/schwab_csv_balances.py | 15 ++-- .../schwab_csv_Balances_test.py | 2 - .../importers/vanguard/vanguard_529.py | 80 +++++++++++++++++++ .../libreader/csv_multitable_reader.py | 16 ++-- .../libreader/csvreader.py | 17 +++- 5 files changed, 110 insertions(+), 20 deletions(-) create mode 100644 beancount_reds_importers/importers/vanguard/vanguard_529.py diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py index 21661e6..789f3f5 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py @@ -48,21 +48,18 @@ def file_date(self, file): def get_max_transaction_date(self): return self.date.date() + + def prepare_processed_table(self, rdr): + rdr = rdr.cut('memo', 'security', 'units', 'unit_price') + rdr = rdr.selectne('memo', '--') # we don't need total rows + rdr = rdr.addfield('date', self.date) + return rdr def prepare_tables(self): # first row has date d = self.raw_rdr[0][0].rsplit(' ', 1)[1] self.date = datetime.datetime.strptime(d, self.date_format) - for section, table in self.alltables.items(): - if section in self.config['section_headers']: - table = table.rename(self.header_map) - table = self.convert_columns(table) - table = table.cut('memo', 'security', 'units', 'unit_price') - table = table.selectne('memo', '--') # we don't need total rows - table = table.addfield('date', self.date) - self.alltables[section] = table - def get_balance_positions(self): for section in self.config['section_headers']: yield from self.alltables[section].namedtuples() diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py index 9138a25..efc5d18 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py @@ -1,5 +1,3 @@ -# flake8: noqa - from os import path from beancount.ingest import regression_pytest as regtest from beancount_reds_importers.importers.schwab import schwab_csv_balances diff --git a/beancount_reds_importers/importers/vanguard/vanguard_529.py b/beancount_reds_importers/importers/vanguard/vanguard_529.py new file mode 100644 index 0000000..9bb1da2 --- /dev/null +++ b/beancount_reds_importers/importers/vanguard/vanguard_529.py @@ -0,0 +1,80 @@ +""" Vanguard 529 csv importer.""" + +import petl as etl +import sys +import re +import datetime + +from beancount.core.number import D + +from beancount_reds_importers.libreader import csv_multitable_reader +from beancount_reds_importers.libtransactionbuilder import investments + +class Importer(investments.Importer, csv_multitable_reader.Importer): + IMPORTER_NAME = 'Vanguard 529' + + def custom_init(self): + self.max_rounding_error = 0.04 + # Vanguard only gives a csv download option for 529 accounts, but they name it "ofxdownload" to tease you + self.filename_pattern_def = '.*ofxdownload.*' + self.header_identifier = 'Fund Account Number,Fund Name,Price,Shares,Total Value.*' + self.get_ticker_info = self.get_ticker_info_from_id + self.date_format = '%m/%d/%Y' + self.funds_db_txt = 'funds_by_ticker' + self.header_map = { + "Process Date": 'date', + "Trade Date": 'tradeDate', + "Transaction Type": 'type', + "Transaction Description": 'memo', + "Shares": 'units', + "Share Price": 'unit_price', + "Gross Amount": 'amount', + "Net Amount": 'total', + "Price": 'unit_price', + } + self.transaction_type_map = { + 'Contribution AIP': 'buystock', + 'Contribution EBT': 'buystock', + } + self.skip_transaction_types = [] + self.section_titles_are_headers = True + self.config['add_currency_precision'] = self.config.get('add_currency_precision', True) + + def deep_identify(self, file): + account_number = self.config.get('account_number', '') + return super().deep_identify(file) and account_number in file.head() + + def file_date(self, file): + return datetime.datetime.now() + + def prepare_tables(self): + ticker_by_desc = {desc: ticker for ticker, _, desc in self.fund_data} + + alltables = {} + maxdate = None + for section, table in self.alltables.items(): + if section == 'Fund Account Number': + section = 'Balance Positions' + table = table.addfield('security', lambda x: ticker_by_desc.get(x['Fund Name'], x['Fund Name'])) + # We need to add a date field but we can't do that yet because we need to make sure + # the transactions section has been processed and set + elif section == 'Account Number': + section = 'Transactions' + table = table.addfield('security', lambda x: ticker_by_desc.get(x['Investment Name'], x['Investment Name'])) + # We have to do our own finding of the max date because the table data hasn't been cleaned up yet + maxdate = max(datetime.datetime.strptime(d[0], self.date_format) for d in table.cut('Trade Date').rename('Trade Date', 'date').namedtuples()).date().strftime(self.date_format) + alltables[section] = table + self.alltables = alltables + + self.alltables['Balance Positions'] = self.alltables['Balance Positions'].addfield('date', maxdate) + + def is_section_title(self, row): + if len(row) == 0: + return False + return row[0] == 'Fund Account Number' or row[0] == 'Account Number' + + def get_transactions(self): + yield from self.alltables['Transactions'].namedtuples() + + def get_balance_positions(self): + yield from self.alltables['Balance Positions'].namedtuples() diff --git a/beancount_reds_importers/libreader/csv_multitable_reader.py b/beancount_reds_importers/libreader/csv_multitable_reader.py index 8a8f4ef..bb31ce7 100644 --- a/beancount_reds_importers/libreader/csv_multitable_reader.py +++ b/beancount_reds_importers/libreader/csv_multitable_reader.py @@ -41,9 +41,6 @@ def file_date(self, file): raise "Not yet implemented" pass - def convert_columns(self, rdr): - pass - def is_section_title(self, row): # Match against rows that contain section titles. Eg: 'section1', 'section2', ... return len(row) == 1 @@ -59,6 +56,10 @@ def read_file(self, file): self.raw_rdr = rdr = self.read_raw(file) + skip_offset = 1 + if getattr(self, 'section_titles_are_headers', False): + skip_offset = 0 + rdr = rdr.skip(getattr(self, 'skip_head_rows', 0)) # chop unwanted file header rows rdr = rdr.head(len(rdr) - getattr(self, 'skip_tail_rows', 0) - 1) # chop unwanted file footer rows @@ -73,8 +74,8 @@ def read_file(self, file): for (s, e) in table_indexes: if s == e: continue - table = rdr.skip(s+1) # skip past start index and header row - table = table.head(e-s-1) # chop lines after table section data + table = rdr.skip(s+skip_offset) # skip past start index and header row + table = table.head(e-s-skip_offset) # chop lines after table section data self.alltables[rdr[s][0]] = table for section, table in self.alltables.items(): @@ -83,6 +84,11 @@ def read_file(self, file): self.alltables[section] = table self.prepare_tables() # to be overridden by importer + + for section, table in self.alltables.items(): + table = self.process_table(table) + self.alltables[section] = table + self.file_read_done = True def get_transactions(self): diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 078fc1b..534d29a 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -192,14 +192,23 @@ def read_file(self, file): rdr = self.prepare_table(rdr) # process table - rdr = rdr.rename(self.header_map) - rdr = self.convert_columns(rdr) - rdr = self.fix_column_names(rdr) - rdr = self.prepare_processed_table(rdr) + rdr = self.process_table(rdr) self.rdr = rdr self.ifile = file self.file_read_done = True + def process_table(self, rdr): + # Filter out any header mappings that don't exist in this table, since petl doesn't do this for us + # and will complain if we try to rename a header that doesn't exist + existing_headers = {key: value for key, value in self.header_map.items() if key in rdr.header()} + rdr = rdr.rename(existing_headers) + + rdr = self.convert_columns(rdr) + rdr = self.fix_column_names(rdr) + rdr = self.prepare_processed_table(rdr) + return rdr + + def get_transactions(self): for ot in self.rdr.namedtuples(): if self.skip_transaction(ot): From 007d71cbf095c6091f6fa31f33a15c5c09560502 Mon Sep 17 00:00:00 2001 From: Jacob Farkas Date: Thu, 14 Sep 2023 12:05:11 -0700 Subject: [PATCH 4/5] test: Add a unit test for the Vanguard_529 importer --- .../tests/{ => vanguard}/OfxDownload-401k.qfx | 0 .../OfxDownload-401k.qfx.extract | 4 +-- .../OfxDownload-401k.qfx.file_account | 0 .../OfxDownload-401k.qfx.file_date | 0 .../OfxDownload-401k.qfx.file_name | 0 .../tests/{ => vanguard}/vanguard_test.py | 0 .../vanguard_529/ofxdownload_09102023.csv | 12 +++++++ .../ofxdownload_09102023.csv.extract | 36 +++++++++++++++++++ .../ofxdownload_09102023.csv.file_account | 1 + .../ofxdownload_09102023.csv.file_date | 1 + .../ofxdownload_09102023.csv.file_name | 1 + .../tests/vanguard_529/vanguard_529_test.py | 32 +++++++++++++++++ .../importers/vanguard/vanguard_529.py | 16 ++++++--- 13 files changed, 97 insertions(+), 6 deletions(-) rename beancount_reds_importers/importers/vanguard/tests/{ => vanguard}/OfxDownload-401k.qfx (100%) rename beancount_reds_importers/importers/vanguard/tests/{ => vanguard}/OfxDownload-401k.qfx.extract (94%) rename beancount_reds_importers/importers/vanguard/tests/{ => vanguard}/OfxDownload-401k.qfx.file_account (100%) rename beancount_reds_importers/importers/vanguard/tests/{ => vanguard}/OfxDownload-401k.qfx.file_date (100%) rename beancount_reds_importers/importers/vanguard/tests/{ => vanguard}/OfxDownload-401k.qfx.file_name (100%) rename beancount_reds_importers/importers/vanguard/tests/{ => vanguard}/vanguard_test.py (100%) create mode 100644 beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv create mode 100644 beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.extract create mode 100644 beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_account create mode 100644 beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_date create mode 100644 beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_name create mode 100644 beancount_reds_importers/importers/vanguard/tests/vanguard_529/vanguard_529_test.py diff --git a/beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx b/beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx similarity index 100% rename from beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx rename to beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx diff --git a/beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx.extract b/beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx.extract similarity index 94% rename from beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx.extract rename to beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx.extract index eb71d94..7c88a20 100644 --- a/beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx.extract +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx.extract @@ -18,6 +18,6 @@ 2023-03-07 * "Investment Expense" "[V7743] Vanguard Target Retirement 2050 Trust" Assets:Vanguard:401k:Pretax:V7743 -0.000349 V7743 -2023-05-25 balance Assets:Vanguard:401k:V7743 113.718 V7743 - 2023-05-26 price V7743 117.71 USD + +2023-05-27 balance Assets:Vanguard:401k:V7743 113.718 V7743 diff --git a/beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx.file_account b/beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx.file_account similarity index 100% rename from beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx.file_account rename to beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx.file_account diff --git a/beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx.file_date b/beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx.file_date similarity index 100% rename from beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx.file_date rename to beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx.file_date diff --git a/beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx.file_name b/beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx.file_name similarity index 100% rename from beancount_reds_importers/importers/vanguard/tests/OfxDownload-401k.qfx.file_name rename to beancount_reds_importers/importers/vanguard/tests/vanguard/OfxDownload-401k.qfx.file_name diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py b/beancount_reds_importers/importers/vanguard/tests/vanguard/vanguard_test.py similarity index 100% rename from beancount_reds_importers/importers/vanguard/tests/vanguard_test.py rename to beancount_reds_importers/importers/vanguard/tests/vanguard/vanguard_test.py diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv new file mode 100644 index 0000000..f14224a --- /dev/null +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv @@ -0,0 +1,12 @@ +Fund Account Number,Fund Name,Price,Shares,Total Value +535672845-01,Vanguard Target Enrollment 2040/2041 Portfolio,$9.45,348.5265,$3293.58 + +Account Number,Trade Date,Process Date,Transaction Type,Transaction Description,Investment Name,Share Price,Shares,Gross Amount,Net Amount +535672845-01,06/30/2023,06/30/2023,Contribution AIP,Contribution AIP,Vanguard Target Enrollment 2040/2041 Portfolio,$9.51,26.2881,$250,$250 +535672845-01,05/31/2023,05/31/2023,Contribution AIP,Contribution AIP,Vanguard Target Enrollment 2040/2041 Portfolio,$9.01,27.7469,$250,$250 +535672845-01,04/28/2023,04/28/2023,Contribution AIP,Contribution AIP,Vanguard Target Enrollment 2040/2041 Portfolio,$9.11,27.4424,$250,$250 +535672845-01,03/31/2023,03/31/2023,Contribution AIP,Contribution AIP,Vanguard Target Enrollment 2040/2041 Portfolio,$9,27.7778,$250,$250 +535672845-01,02/28/2023,02/28/2023,Contribution AIP,Contribution AIP,Vanguard Target Enrollment 2040/2041 Portfolio,$8.76,28.5388,$250,$250 +535672845-01,02/17/2023,02/17/2023,Contribution EBT,Contribution EBT,Vanguard Target Enrollment 2040/2041 Portfolio,$8.99,22.2469,$200,$200 +535672845-01,01/31/2023,01/31/2023,Contribution AIP,Contribution AIP,Vanguard Target Enrollment 2040/2041 Portfolio,$9.03,27.6855,$250,$250 +535672845-01,01/03/2023,01/03/2023,Contribution AIP,Contribution AIP,Vanguard Target Enrollment 2040/2041 Portfolio,$8.42,29.6912,$250,$250 diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.extract b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.extract new file mode 100644 index 0000000..893b2a0 --- /dev/null +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.extract @@ -0,0 +1,36 @@ + +2023-01-03 * "Contribution AIP" "[VTE2040] Vanguard Target Enrollment 2040/2041 Portfolio" + Assets:Vanguard:529:VTE2040 29.6912 VTE2040 {8.42 USD} + Assets:Vanguard:529:Cash -250.00 USD + +2023-01-31 * "Contribution AIP" "[VTE2040] Vanguard Target Enrollment 2040/2041 Portfolio" + Assets:Vanguard:529:VTE2040 27.6855 VTE2040 {9.03 USD} + Assets:Vanguard:529:Cash -250.00 USD + +2023-02-17 * "Contribution EBT" "[VTE2040] Vanguard Target Enrollment 2040/2041 Portfolio" + Assets:Vanguard:529:VTE2040 22.2469 VTE2040 {8.99 USD} + Assets:Vanguard:529:Cash -200.00 USD + +2023-02-28 * "Contribution AIP" "[VTE2040] Vanguard Target Enrollment 2040/2041 Portfolio" + Assets:Vanguard:529:VTE2040 28.5388 VTE2040 {8.76 USD} + Assets:Vanguard:529:Cash -250.00 USD + +2023-03-31 * "Contribution AIP" "[VTE2040] Vanguard Target Enrollment 2040/2041 Portfolio" + Assets:Vanguard:529:VTE2040 27.7778 VTE2040 {9.00 USD} + Assets:Vanguard:529:Cash -250.00 USD + +2023-04-28 * "Contribution AIP" "[VTE2040] Vanguard Target Enrollment 2040/2041 Portfolio" + Assets:Vanguard:529:VTE2040 27.4424 VTE2040 {9.11 USD} + Assets:Vanguard:529:Cash -250.00 USD + +2023-05-31 * "Contribution AIP" "[VTE2040] Vanguard Target Enrollment 2040/2041 Portfolio" + Assets:Vanguard:529:VTE2040 27.7469 VTE2040 {9.01 USD} + Assets:Vanguard:529:Cash -250.00 USD + +2023-06-30 * "Contribution AIP" "[VTE2040] Vanguard Target Enrollment 2040/2041 Portfolio" + Assets:Vanguard:529:VTE2040 26.2881 VTE2040 {9.51 USD} + Assets:Vanguard:529:Cash -250.00 USD + +2023-06-30 price VTE2040 9.45 USD + +2023-07-01 balance Assets:Vanguard:529:VTE2040 348.5265 VTE2040 diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_account b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_account new file mode 100644 index 0000000..e2db6d4 --- /dev/null +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_account @@ -0,0 +1 @@ +Assets:Vanguard:529 diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_date b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_date new file mode 100644 index 0000000..6253d73 --- /dev/null +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_date @@ -0,0 +1 @@ +2023-09-10 diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_name b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_name new file mode 100644 index 0000000..58d8e49 --- /dev/null +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/ofxdownload_09102023.csv.file_name @@ -0,0 +1 @@ +ofxdownload_09102023.csv diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_529/vanguard_529_test.py b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/vanguard_529_test.py new file mode 100644 index 0000000..89775bc --- /dev/null +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/vanguard_529_test.py @@ -0,0 +1,32 @@ +from os import path +from beancount.ingest import regression_pytest as regtest +from beancount_reds_importers.importers.vanguard import vanguard_529 + + +@regtest.with_importer( + vanguard_529.Importer( + { + "account_number": "535672845-01", + "main_account": "Assets:Vanguard:529:{ticker}", + "cash_account": "Assets:Vanguard:529:Cash", + "dividends": "Income:Dividends:Vanguard:529:{ticker}", + "interest": "Income:Interest:Vanguard:529:{ticker}", + "cg": "Income:CapitalGains:529:{ticker}", + "capgainsd_lt": "Income:CapitalGains:Long:Vanguard:529:{ticker}", + "capgainsd_st": "Income:CapitalGains:Short:Vanguard:529:{ticker}", + "fees": "Expenses:Fees:Vanguard:529", + "invexpense": "Expenses:Expenses:Vanguard:529", + "rounding_error": "Equity:Rounding-Errors:Imports", + "fund_info": { + "fund_data": [ + ('VTE2040', '00000000', 'Vanguard Target Enrollment 2040/2041 Portfolio'), + ], + "money_market": [], + }, + "currency" : 'USD', + } + ) +) +@regtest.with_testdir(path.dirname(__file__)) +class TestVanguard529(regtest.ImporterTestBase): + pass diff --git a/beancount_reds_importers/importers/vanguard/vanguard_529.py b/beancount_reds_importers/importers/vanguard/vanguard_529.py index 9bb1da2..2adc9c4 100644 --- a/beancount_reds_importers/importers/vanguard/vanguard_529.py +++ b/beancount_reds_importers/importers/vanguard/vanguard_529.py @@ -3,7 +3,7 @@ import petl as etl import sys import re -import datetime +from datetime import datetime from beancount.core.number import D @@ -45,7 +45,15 @@ def deep_identify(self, file): return super().deep_identify(file) and account_number in file.head() def file_date(self, file): - return datetime.datetime.now() + date = None + # Use the date in the file name. If that doesn't exist, fall back to the maximum date we found in the transactions + match = re.search(r'\d{8}', file.name) + if match: + date_str = match.group() + date = datetime.strptime(date_str, "%m%d%Y").date() + else: + date = self.maxdate + return date def prepare_tables(self): ticker_by_desc = {desc: ticker for ticker, _, desc in self.fund_data} @@ -62,11 +70,11 @@ def prepare_tables(self): section = 'Transactions' table = table.addfield('security', lambda x: ticker_by_desc.get(x['Investment Name'], x['Investment Name'])) # We have to do our own finding of the max date because the table data hasn't been cleaned up yet - maxdate = max(datetime.datetime.strptime(d[0], self.date_format) for d in table.cut('Trade Date').rename('Trade Date', 'date').namedtuples()).date().strftime(self.date_format) + self.maxdate = max(datetime.strptime(d[0], self.date_format) for d in table.cut('Trade Date').rename('Trade Date', 'date').namedtuples()).date().strftime(self.date_format) alltables[section] = table self.alltables = alltables - self.alltables['Balance Positions'] = self.alltables['Balance Positions'].addfield('date', maxdate) + self.alltables['Balance Positions'] = self.alltables['Balance Positions'].addfield('date', self.maxdate) def is_section_title(self, row): if len(row) == 0: From cc3e2a646fbf1e6334fa2c4cb6a9dbad5bf71fbc Mon Sep 17 00:00:00 2001 From: Jacob Farkas Date: Thu, 14 Sep 2023 20:26:43 -0700 Subject: [PATCH 5/5] fix(minor): Fix all flake8 warnings and errors --- .../importers/schwab/schwab_csv_balances.py | 2 +- .../schwab_csv_Balances_test.py | 32 ++++++++--------- .../tests/vanguard_529/vanguard_529_test.py | 36 +++++++++---------- .../importers/vanguard/vanguard_529.py | 22 ++++++------ .../libreader/csvreader.py | 4 +-- 5 files changed, 48 insertions(+), 48 deletions(-) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py index 789f3f5..2a02c4c 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py @@ -48,7 +48,7 @@ def file_date(self, file): def get_max_transaction_date(self): return self.date.date() - + def prepare_processed_table(self, rdr): rdr = rdr.cut('memo', 'security', 'units', 'unit_price') rdr = rdr.selectne('memo', '--') # we don't need total rows diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py index efc5d18..1741199 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_balances/schwab_csv_Balances_test.py @@ -13,7 +13,7 @@ ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMMX','VMMX2','HOOLI'] +money_market = ['VMMX', 'VMMX2', 'HOOLI'] fund_info = { 'fund_data': fund_data, @@ -28,21 +28,21 @@ def build_config(): leaf = 'Schwab' currency = 'USD' config = { - 'account_number' : 9876, - 'main_account' : acct + ':{ticker}', - 'cash_account' : f'{acct}:{{currency}}', - 'transfer' : 'Assets:Zero-Sum-Accounts:Transfers:Bank-Account', - 'dividends' : f'Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}', - 'interest' : f'Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}', - 'cg' : f'Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}', - 'capgainsd_lt' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}', - 'capgainsd_st' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}', - 'fees' : f'Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}', - 'invexpense' : f'Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}', - 'rounding_error' : 'Equity:Rounding-Errors:Imports', - 'fund_info' : fund_info, - 'currency' : currency, - 'section_headers': ['Stocks', 'Bonds', 'Money Market'] + 'account_number': 9876, + 'main_account': acct + ':{ticker}', + 'cash_account': f'{acct}:{{currency}}', + 'transfer': ' Assets:Zero-Sum-Accounts:Transfers:Bank-Account', + 'dividends': f'Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}', + 'interest': f'Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}', + 'cg': f'Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}', + 'capgainsd_lt': f'Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}', + 'capgainsd_st': f'Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}', + 'fees': f'Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}', + 'invexpense': f'Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}', + 'rounding_error': 'Equity:Rounding-Errors:Imports', + 'fund_info': fund_info, + 'currency': currency, + 'section_headers': ['Stocks', 'Bonds', 'Money Market'] } return config diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_529/vanguard_529_test.py b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/vanguard_529_test.py index 89775bc..61ce454 100644 --- a/beancount_reds_importers/importers/vanguard/tests/vanguard_529/vanguard_529_test.py +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_529/vanguard_529_test.py @@ -6,24 +6,24 @@ @regtest.with_importer( vanguard_529.Importer( { - "account_number": "535672845-01", - "main_account": "Assets:Vanguard:529:{ticker}", - "cash_account": "Assets:Vanguard:529:Cash", - "dividends": "Income:Dividends:Vanguard:529:{ticker}", - "interest": "Income:Interest:Vanguard:529:{ticker}", - "cg": "Income:CapitalGains:529:{ticker}", - "capgainsd_lt": "Income:CapitalGains:Long:Vanguard:529:{ticker}", - "capgainsd_st": "Income:CapitalGains:Short:Vanguard:529:{ticker}", - "fees": "Expenses:Fees:Vanguard:529", - "invexpense": "Expenses:Expenses:Vanguard:529", - "rounding_error": "Equity:Rounding-Errors:Imports", - "fund_info": { - "fund_data": [ - ('VTE2040', '00000000', 'Vanguard Target Enrollment 2040/2041 Portfolio'), - ], - "money_market": [], - }, - "currency" : 'USD', + "account_number": "535672845-01", + "main_account": "Assets:Vanguard:529:{ticker}", + "cash_account": "Assets:Vanguard:529:Cash", + "dividends": "Income:Dividends:Vanguard:529:{ticker}", + "interest": "Income:Interest:Vanguard:529:{ticker}", + "cg": "Income:CapitalGains:529:{ticker}", + "capgainsd_lt": "Income:CapitalGains:Long:Vanguard:529:{ticker}", + "capgainsd_st": "Income:CapitalGains:Short:Vanguard:529:{ticker}", + "fees": "Expenses:Fees:Vanguard:529", + "invexpense": "Expenses:Expenses:Vanguard:529", + "rounding_error": "Equity:Rounding-Errors:Imports", + "fund_info": { + "fund_data": [ + ('VTE2040', '00000000', 'Vanguard Target Enrollment 2040/2041 Portfolio'), + ], + "money_market": [], + }, + "currency": 'USD', } ) ) diff --git a/beancount_reds_importers/importers/vanguard/vanguard_529.py b/beancount_reds_importers/importers/vanguard/vanguard_529.py index 2adc9c4..f48512b 100644 --- a/beancount_reds_importers/importers/vanguard/vanguard_529.py +++ b/beancount_reds_importers/importers/vanguard/vanguard_529.py @@ -1,15 +1,12 @@ """ Vanguard 529 csv importer.""" -import petl as etl -import sys import re from datetime import datetime -from beancount.core.number import D - from beancount_reds_importers.libreader import csv_multitable_reader from beancount_reds_importers.libtransactionbuilder import investments + class Importer(investments.Importer, csv_multitable_reader.Importer): IMPORTER_NAME = 'Vanguard 529' @@ -39,11 +36,11 @@ def custom_init(self): self.skip_transaction_types = [] self.section_titles_are_headers = True self.config['add_currency_precision'] = self.config.get('add_currency_precision', True) - + def deep_identify(self, file): account_number = self.config.get('account_number', '') return super().deep_identify(file) and account_number in file.head() - + def file_date(self, file): date = None # Use the date in the file name. If that doesn't exist, fall back to the maximum date we found in the transactions @@ -54,23 +51,26 @@ def file_date(self, file): else: date = self.maxdate return date - + def prepare_tables(self): ticker_by_desc = {desc: ticker for ticker, _, desc in self.fund_data} alltables = {} - maxdate = None + self.maxdate = None for section, table in self.alltables.items(): if section == 'Fund Account Number': section = 'Balance Positions' table = table.addfield('security', lambda x: ticker_by_desc.get(x['Fund Name'], x['Fund Name'])) - # We need to add a date field but we can't do that yet because we need to make sure + # We need to add a date field but we can't do that yet because we need to make sure # the transactions section has been processed and set elif section == 'Account Number': section = 'Transactions' table = table.addfield('security', lambda x: ticker_by_desc.get(x['Investment Name'], x['Investment Name'])) # We have to do our own finding of the max date because the table data hasn't been cleaned up yet - self.maxdate = max(datetime.strptime(d[0], self.date_format) for d in table.cut('Trade Date').rename('Trade Date', 'date').namedtuples()).date().strftime(self.date_format) + self.maxdate = max(datetime.strptime(d[0], self.date_format) + for d in table.cut('Trade Date').rename('Trade Date', 'date').namedtuples()) \ + .date().strftime(self.date_format) + alltables[section] = table self.alltables = alltables @@ -80,7 +80,7 @@ def is_section_title(self, row): if len(row) == 0: return False return row[0] == 'Fund Account Number' or row[0] == 'Account Number' - + def get_transactions(self): yield from self.alltables['Transactions'].namedtuples() diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 534d29a..acbda70 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -108,6 +108,7 @@ def convert_columns(self, rdr): # fixup currencies def remove_non_numeric(x): return re.sub("[^0-9\.-]", "", str(x).strip()) # noqa: W605 + def add_decimal(x): if '.' not in x: return x+".00" @@ -202,13 +203,12 @@ def process_table(self, rdr): # and will complain if we try to rename a header that doesn't exist existing_headers = {key: value for key, value in self.header_map.items() if key in rdr.header()} rdr = rdr.rename(existing_headers) - + rdr = self.convert_columns(rdr) rdr = self.fix_column_names(rdr) rdr = self.prepare_processed_table(rdr) return rdr - def get_transactions(self): for ot in self.rdr.namedtuples(): if self.skip_transaction(ot):