From 78f918223e674d07813a3d95e504a3ade02453aa Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 3 Feb 2024 00:39:08 -0800 Subject: [PATCH 01/59] feat(minor): overridable add_custom_postings() --- beancount_reds_importers/libtransactionbuilder/investments.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 8d2759e..f0b9646 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -380,6 +380,7 @@ def extract_transactions(self, file, counter): print("ERROR: unknown entry type:", ot.type) raise Exception('Unknown entry type') self.add_fee_postings(entry, ot) + self.add_custom_postings(entry, ot) new_entries.append(entry) return new_entries @@ -436,6 +437,9 @@ def add_fee_postings(self, entry, ot): if getattr(ot, 'commission', 0) != 0: data.create_simple_posting(entry, config['fees'], ot.commission, self.currency) + def add_custom_postings(self, entry, ot): + pass + def extract_custom_entries(self, file, counter): """For custom importers to override""" return [] From c33e02fb4c5db5c9b2e4c356c22d55131ff2d9ab Mon Sep 17 00:00:00 2001 From: Red S Date: Thu, 8 Feb 2024 23:25:38 -0800 Subject: [PATCH 02/59] feat: add identification based on filename to schwab importers --- .../importers/schwab/schwab_csv_brokerage.py | 5 +++++ .../importers/schwab/schwab_csv_checking.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index 63ab8fa..b448808 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -1,5 +1,6 @@ """ Schwab Brokerage .csv importer.""" +import re from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import investments @@ -51,6 +52,10 @@ def custom_init(self): 'Cash In Lieu': 'cash', } + def deep_identify(self, file): + last_three = self.config.get('account_number', '')[-3:] + return re.match(self.header_identifier, file.head()) and f'XX{last_three}' in file.name + def skip_transaction(self, ot): return ot.type in ['', 'Journal'] diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py index d3132f4..7a3072c 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py @@ -30,6 +30,10 @@ def custom_init(self): } self.skip_transaction_types = ['Journal'] + def deep_identify(self, file): + last_three = self.config.get('account_number', '')[-3:] + return self.column_labels_line in file.head() and f'XX{last_three}' in file.name + def prepare_table(self, rdr): rdr = rdr.addfield('amount', lambda x: "-" + x['Withdrawal'] if x['Withdrawal'] != '' else x['Deposit']) From 1d379183497a7cff73c0041f200d4b13df54f1d0 Mon Sep 17 00:00:00 2001 From: Red S Date: Thu, 8 Feb 2024 23:43:31 -0800 Subject: [PATCH 03/59] feat: add schwab csv credit line importer --- .../importers/schwab/schwab_csv_creditline.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 beancount_reds_importers/importers/schwab/schwab_csv_creditline.py diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py new file mode 100644 index 0000000..5f452de --- /dev/null +++ b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py @@ -0,0 +1,11 @@ +""" Schwab Credit Line (eg: Pledged Asset Line) .csv importer.""" + +from beancount_reds_importers.importers.schwab import schwab_csv_checking + +class Importer(schwab_csv_checking.Importer): + IMPORTER_NAME = 'Schwab Line of Credit CSV' + + def custom_init(self): + super().custom_init() + self.filename_pattern_def = '.*_Transactions_' + self.column_labels_line = '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' From bee9b44758fb37d1ccac5a94f98b96e25825aa5c Mon Sep 17 00:00:00 2001 From: Red S Date: Fri, 9 Feb 2024 01:37:15 -0800 Subject: [PATCH 04/59] fix: schwab doesn't use a header in their csv any more - also: add "Journaled Shares" as a buy type. it's actually an in-kind transfer that needs to be manually handled anyway --- .../importers/schwab/schwab_csv_brokerage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index b448808..01583fc 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -11,7 +11,7 @@ class Importer(csvreader.Importer, investments.Importer): def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = '.*_Transactions_' - self.header_identifier = '"Transactions for account.*' + self.header_identifier = '' self.column_labels_line = '"Date","Action","Symbol","Description","Quantity","Price","Fees & Comm","Amount"' self.get_ticker_info = self.get_ticker_info_from_id self.date_format = '%m/%d/%Y' @@ -32,6 +32,7 @@ def custom_init(self): 'Bank Interest': 'income', 'Bank Transfer': 'cash', 'Buy': 'buystock', + 'Journaled Shares': 'buystock', # These are in-kind tranfers 'Reinvestment Adj': 'buystock', 'Div Adjustment': 'dividends', 'Long Term Cap Gain Reinvest': 'capgainsd_lt', @@ -39,6 +40,7 @@ def custom_init(self): 'MoneyLink Deposit': 'cash', 'MoneyLink Transfer': 'cash', 'Pr Yr Div Reinvest': 'dividends', + 'Journal': 'cash', # These are transfers 'Reinvest Dividend': 'dividends', 'Qualified Dividend': 'dividends', 'Cash Dividend': 'dividends', From 41c8cbf13a8ef0a232e84da81612b26b2d1462af Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 10 Feb 2024 16:53:28 -0800 Subject: [PATCH 05/59] fix: schwab tests --- .../importers/schwab/schwab_json_brokerage.py | 18 ++++ ...v_brokerage_Transactions_123.csv.file_name | 1 - ...brokerage_XX876_Transactions_20220731.csv} | 0 ...e_XX876_Transactions_20220731.csv.extract} | 12 ++- ...76_Transactions_20220731.csv.file_account} | 0 ...XX876_Transactions_20220731.csv.file_date} | 0 ..._XX876_Transactions_20220731.csv.file_name | 1 + .../schwab_csv_brokerage_test.py | 4 +- .../schwab_Checking_Transactions_1234.csv | 12 --- ...wab_Checking_Transactions_1234.csv.extract | 14 --- ...b_Checking_Transactions_1234.csv.file_date | 1 - ...b_Checking_Transactions_1234.csv.file_name | 1 - ..._XXX234_Checking_Transactions_20220203.csv | 3 + ...Checking_Transactions_20220203.csv.extract | 7 ++ ...ng_Transactions_20220203.csv.file_account} | 0 ...ecking_Transactions_20220203.csv.file_date | 1 + ...ecking_Transactions_20220203.csv.file_name | 1 + .../libreader/csvreader.py | 6 +- .../libreader/jsonreader.py | 86 +++++++++++++++++++ 19 files changed, 130 insertions(+), 38 deletions(-) create mode 100644 beancount_reds_importers/importers/schwab/schwab_json_brokerage.py delete mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.file_name rename beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/{schwab_csv_brokerage_Transactions_123.csv => schwab_csv_brokerage_XX876_Transactions_20220731.csv} (100%) rename beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/{schwab_csv_brokerage_Transactions_123.csv.extract => schwab_csv_brokerage_XX876_Transactions_20220731.csv.extract} (76%) rename beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/{schwab_csv_brokerage_Transactions_123.csv.file_account => schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_account} (100%) rename beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/{schwab_csv_brokerage_Transactions_123.csv.file_date => schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_date} (100%) create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_name delete mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv delete mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.extract delete mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_date delete mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_name create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.extract rename beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/{schwab_Checking_Transactions_1234.csv.file_account => schwab_XXX234_Checking_Transactions_20220203.csv.file_account} (100%) create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_date create mode 100644 beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_name create mode 100644 beancount_reds_importers/libreader/jsonreader.py diff --git a/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py new file mode 100644 index 0000000..3c50ad1 --- /dev/null +++ b/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py @@ -0,0 +1,18 @@ +""" Schwab Brokerage .csv importer.""" + +from beancount_reds_importers.libreader import jsonreader +from beancount_reds_importers.libtransactionbuilder import investments + + +class Importer(jsonreader.Importer, investments.Importer): + IMPORTER_NAME = 'Schwab Brokerage JSON' + + def custom_init(self): + self.max_rounding_error = 0.04 + self.filename_pattern_def = '.*_Transactions_' + self.get_ticker_info = self.get_ticker_info_from_id + self.date_format = '%m/%d/%Y' + self.funds_db_txt = 'funds_by_ticker' + + def skip_transaction(self, ot): + return ot.type in ['', 'Journal', 'Journaled Shares'] diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.file_name b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.file_name deleted file mode 100644 index 87530f4..0000000 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.file_name +++ /dev/null @@ -1 +0,0 @@ -schwab_csv_brokerage_Transactions_123.csv diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv similarity index 100% rename from beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv rename to beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.extract b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.extract similarity index 76% rename from beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.extract rename to beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.extract index 4df6be2..03cad14 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.extract +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.extract @@ -29,12 +29,20 @@ 2023-04-18 * "Sell" "[BND] Vanguard Total Bond Market Index Fund" todo: "TODO: this entry is incomplete until lots are selected (bean-doctor context )" - Assets:Investments:Schwab:BND -10.065 BND {} @ 73.4049 USD + Assets:Investments:Schwab:BND -10.065 BND {} @ 73.4049 USD Income:Investments:Taxable:Capital-Gains:Schwab:BND Assets:Investments:Schwab:USD 738.81 USD - Equity:Rounding-Errors:Imports 0.0103185 USD + Equity:Rounding-Errors:Imports 0.0103185 USD Expenses:Fees-and-Charges:Brokerage-Fees:Taxable:Schwab 0.01 USD +2023-04-20 * "Journal" "cash" + Assets:Investments:Schwab:USD -7461.72 USD + Assets:Zero-Sum-Accounts:Transfers:Bank-Account 7461.72 USD + 2023-04-27 * "Buy" "[BND] Vanguard Total Bond Market Index Fund" Assets:Investments:Schwab:BND 45 BND {73.7789 USD} Assets:Investments:Schwab:USD -3320.05 USD + +2023-04-27 * "Journal" "cash" + Assets:Investments:Schwab:USD 7461.72 USD + Assets:Zero-Sum-Accounts:Transfers:Bank-Account -7461.72 USD diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.file_account b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_account similarity index 100% rename from beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.file_account rename to beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_account diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.file_date b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_date similarity index 100% rename from beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_Transactions_123.csv.file_date rename to beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_date diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_name b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_name new file mode 100644 index 0000000..99b871a --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_XX876_Transactions_20220731.csv.file_name @@ -0,0 +1 @@ +schwab_csv_brokerage_XX876_Transactions_20220731.csv diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py index 09ac48e..546fca6 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py @@ -27,7 +27,7 @@ def build_config(): leaf = 'Schwab' currency = 'USD' config = { - 'account_number' : 9876, + 'account_number' : '9876', 'main_account' : acct + ':{ticker}', 'cash_account' : f'{acct}:{{currency}}', 'transfer' : 'Assets:Zero-Sum-Accounts:Transfers:Bank-Account', @@ -51,5 +51,5 @@ def build_config(): ) ) @regtest.with_testdir(path.dirname(__file__)) -class TestSchwabCSV(regtest.ImporterTestBase): +class TestSchwabBrokerage(regtest.ImporterTestBase): pass diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv deleted file mode 100644 index 0649dd8..0000000 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv +++ /dev/null @@ -1,12 +0,0 @@ -"Transactions for Checking account ...000 as of 05/10/2023 01:41:36 PM ET" -"Date","Type","Check #","Description","Withdrawal (-)","Deposit (+)","RunningBalance" -"Pending Transactions" -"Total Pending Check and other Credit(s)","","","","","$0.00","" -"05/09/2023","","","APPLE CASH CUPERTINO, CA, US","$130.00","","" -"05/09/2023","","","APPLE CASH CUPERTINO, CA, US","$50.00","","" -"05/09/2023","","","APPLE CASH CUPERTINO, CA, US","$15.00","","" -"Posted Transactions" -"12/30/2022","INTADJUST","","Interest Paid","","$1.00","$3,951.00" -"12/25/2022","VISA","","Grocery Store","$50.00","","$3,950.00" -"12/20/2022","TRANSFER","","Funds Transfer to Brokerage -XXXX","$1,000.00","","$4,000.00" -"12/14/2022","ACH","","Electronic Deposit","","$5,000.00","$5,000.00" diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.extract b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.extract deleted file mode 100644 index cb3c2dd..0000000 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.extract +++ /dev/null @@ -1,14 +0,0 @@ - -2022-12-14 * "Electronic Deposit" - Assets:Banks:Schwab 5000.00 USD - -2022-12-20 * "Funds Transfer to Brokerage -XXXX" - Assets:Banks:Schwab -1000.00 USD - -2022-12-25 * "Grocery Store" - Assets:Banks:Schwab -50.00 USD - -2022-12-30 * "Interest Paid" - Assets:Banks:Schwab 1.00 USD - -2022-12-31 balance Assets:Banks:Schwab 3951.00 USD diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_date b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_date deleted file mode 100644 index 027b98f..0000000 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_date +++ /dev/null @@ -1 +0,0 @@ -2022-12-30 diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_name b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_name deleted file mode 100644 index 2b229c7..0000000 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_name +++ /dev/null @@ -1 +0,0 @@ -schwab_Checking_Transactions_1234.csv diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv new file mode 100644 index 0000000..5117fab --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv @@ -0,0 +1,3 @@ +"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance" +"07/31/2022","Posted","INTADJUST","","Interest Paid","","$1.00","$2.00" +"02/03/2022","Posted","ACH","","Electronic Withdrawal","$2.00","","$3.00" diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.extract b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.extract new file mode 100644 index 0000000..22bf2ba --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.extract @@ -0,0 +1,7 @@ +2022-02-03 * "Electronic Withdrawal" + Assets:Banks:Schwab -2.00 USD + +2022-07-31 * "Interest Paid" + Assets:Banks:Schwab 1.00 USD + +2022-08-01 balance Assets:Banks:Schwab 2.00 USD diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_account b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_account similarity index 100% rename from beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_Checking_Transactions_1234.csv.file_account rename to beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_account diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_date b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_date new file mode 100644 index 0000000..1969bb8 --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_date @@ -0,0 +1 @@ +2022-07-31 diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_name b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_name new file mode 100644 index 0000000..395cc53 --- /dev/null +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_XXX234_Checking_Transactions_20220203.csv.file_name @@ -0,0 +1 @@ +schwab_XXX234_Checking_Transactions_20220203.csv diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 457f832..84a3719 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -170,8 +170,7 @@ def skip_until_row_contains(self, rdr, value): return rdr.rowslice(start, len(rdr)) def read_file(self, file): - if not self.file_read_done: - + if not getattr(self, 'file_read_done', False): # read file rdr = self.read_raw(file) rdr = self.prepare_raw_file(rdr) @@ -200,9 +199,6 @@ def get_transactions(self): continue yield ot - def get_available_cash(self, settlement_fund_balance=0): - return None - # TOOD: custom, overridable def skip_transaction(self, row): return getattr(row, 'type', 'NO_TYPE') in self.skip_transaction_types diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py new file mode 100644 index 0000000..5889e7a --- /dev/null +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -0,0 +1,86 @@ +"""JSON importer module for beancount to be used along with investment/banking/other importer modules in +beancount_reds_importers. + +JSON schemas vary widely. This one is based on Charles Schwab's json format. In the future, the +goal is to make this reader automatically "understand" the schema of any json given to it. + +Until that happens, perhaps this file should be renamed to schwabjsonreader.py. +""" + +import datetime +import ofxparse +from collections import namedtuple +from beancount.ingest import importer +from beancount_reds_importers.libreader import reader +from bs4.builder import XMLParsedAsHTMLWarning +import json +import re +import warnings +warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) + + +class Importer(reader.Reader, importer.ImporterProtocol): + FILE_EXTS = ['json'] + + def initialize_reader(self, file): + if getattr(self, 'file', None) != file: + self.file = file + self.reader_ready = self.deep_identify(file) + if self.reader_ready: + self.file_read_done = False + + def deep_identify(self, file): + # identify based on filename + return True + + def file_date(self, file): + "Get the maximum date from the file." + self.initialize(file) # self.date_format gets set via this + self.read_file(file) + return max(ot.date for ot in self.get_transactions()).date() + + def read_file(self, file): + with open(file.name) as fh: + self.rdr = json.load(fh) + + transactions = [] + for transaction in self.rdr['BrokerageTransactions']: + raw_ot = Transaction( + date = transaction['Date'], + type = transaction['Action'], + security = transaction['Symbol'], + memo = transaction['Description'], + unit_price = transaction['Price'], + units = transaction['Quantity'], + fees = transaction['Fees & Comm'], + total = transaction['Amount'] + ) + + + + def get_transactions(self): + Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', + 'units', 'fees', 'total']) + for transaction in self.rdr['BrokerageTransactions']: + raw_ot = Transaction( + date = transaction['Date'], + type = transaction['Action'], + security = transaction['Symbol'], + memo = transaction['Description'], + unit_price = transaction['Price'], + units = transaction['Quantity'], + fees = transaction['Fees & Comm'], + total = transaction['Amount'] + ) + ot = self.fixup(ot) + import pdb; pdb.set_trace() + yield ot + + def fixup(self, ot): + ot.date = self.convert_date(ot.date) + + def convert_date(d): + return datetime.datetime.strptime(d, self.date_format) + + def get_balance_assertion_date(self): + return None From 7400c60c307adaf222194148e6571a310f3c77a0 Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 10 Feb 2024 16:58:05 -0800 Subject: [PATCH 06/59] ci: fix ruff --- .../libreader/jsonreader.py | 81 ++++++++++--------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 5889e7a..5ae1e82 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -1,20 +1,25 @@ + """JSON importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers. +------------------------------ +This is WIP and incomplete. +------------------------------ + JSON schemas vary widely. This one is based on Charles Schwab's json format. In the future, the goal is to make this reader automatically "understand" the schema of any json given to it. Until that happens, perhaps this file should be renamed to schwabjsonreader.py. """ -import datetime -import ofxparse -from collections import namedtuple +# import datetime +# import ofxparse +# from collections import namedtuple from beancount.ingest import importer from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning import json -import re +# import re import warnings warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) @@ -43,44 +48,44 @@ def read_file(self, file): with open(file.name) as fh: self.rdr = json.load(fh) - transactions = [] - for transaction in self.rdr['BrokerageTransactions']: - raw_ot = Transaction( - date = transaction['Date'], - type = transaction['Action'], - security = transaction['Symbol'], - memo = transaction['Description'], - unit_price = transaction['Price'], - units = transaction['Quantity'], - fees = transaction['Fees & Comm'], - total = transaction['Amount'] - ) - - - - def get_transactions(self): - Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', - 'units', 'fees', 'total']) - for transaction in self.rdr['BrokerageTransactions']: - raw_ot = Transaction( - date = transaction['Date'], - type = transaction['Action'], - security = transaction['Symbol'], - memo = transaction['Description'], - unit_price = transaction['Price'], - units = transaction['Quantity'], - fees = transaction['Fees & Comm'], - total = transaction['Amount'] - ) - ot = self.fixup(ot) - import pdb; pdb.set_trace() - yield ot + # transactions = [] + # for transaction in self.rdr['BrokerageTransactions']: + # raw_ot = Transaction( + # date = transaction['Date'], + # type = transaction['Action'], + # security = transaction['Symbol'], + # memo = transaction['Description'], + # unit_price = transaction['Price'], + # units = transaction['Quantity'], + # fees = transaction['Fees & Comm'], + # total = transaction['Amount'] + # ) + + + + # def get_transactions(self): + # Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', + # 'units', 'fees', 'total']) + # for transaction in self.rdr['BrokerageTransactions']: + # raw_ot = Transaction( + # date = transaction['Date'], + # type = transaction['Action'], + # security = transaction['Symbol'], + # memo = transaction['Description'], + # unit_price = transaction['Price'], + # units = transaction['Quantity'], + # fees = transaction['Fees & Comm'], + # total = transaction['Amount'] + # ) + # ot = self.fixup(ot) + # import pdb; pdb.set_trace() + # yield ot def fixup(self, ot): ot.date = self.convert_date(ot.date) - def convert_date(d): - return datetime.datetime.strptime(d, self.date_format) + # def convert_date(d): + # return datetime.datetime.strptime(d, self.date_format) def get_balance_assertion_date(self): return None From 44f04ef065ce65d77704f3b9479d0bdcc87dc91a Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 10 Feb 2024 16:59:43 -0800 Subject: [PATCH 07/59] ci: reenable pytests; add conventionalcommits for merge requests --- .github/workflows/conventionalcommits.yml | 14 ++++++++++++++ .github/workflows/pythonpackage.yml | 6 +++--- 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/conventionalcommits.yml diff --git a/.github/workflows/conventionalcommits.yml b/.github/workflows/conventionalcommits.yml new file mode 100644 index 0000000..b7fa7a1 --- /dev/null +++ b/.github/workflows/conventionalcommits.yml @@ -0,0 +1,14 @@ +name: Conventional Commits + +on: + pull_request: + branches: [ master ] + +jobs: + build: + name: Conventional Commits + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: webiny/action-conventional-commits@v1.1.0 diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index caaf647..53d3c51 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -31,6 +31,6 @@ jobs: - name: Lint with ruff run: | ruff check . --statistics - # - name: Test with pytest - # run: | - # pytest + - name: Test with pytest + run: | + pytest From 1239a5b03ed3c9a1fbe83a019d516454c002f29a Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 10 Feb 2024 17:03:15 -0800 Subject: [PATCH 08/59] ci: python 3.11 minimum --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 53d3c51..172e3d6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.11"] steps: - uses: actions/checkout@v3 From 80bae35318b07548849d8db3ebc5a4d582b3add7 Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 10 Feb 2024 17:06:55 -0800 Subject: [PATCH 09/59] ci: xlrd requirement for pytest --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 4e2ecd4..ffd5956 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ tabulate>=0.9.0 tomli>=2.0.1 tqdm>=4.65.0 typing_extensions>=4.7.1 +xlrd>=2.0.1 From c79f92c90f6a64b9e1234d6ad6101446499d419f Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 10 Feb 2024 17:09:49 -0800 Subject: [PATCH 10/59] fix: balance date on test --- .../importers/capitalonebank/tests/360Checking.qfx.extract | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beancount_reds_importers/importers/capitalonebank/tests/360Checking.qfx.extract b/beancount_reds_importers/importers/capitalonebank/tests/360Checking.qfx.extract index 3c7b92a..a75aec0 100644 --- a/beancount_reds_importers/importers/capitalonebank/tests/360Checking.qfx.extract +++ b/beancount_reds_importers/importers/capitalonebank/tests/360Checking.qfx.extract @@ -11,4 +11,4 @@ 2023-02-28 * "Monthly Interest Paid" "" Assets:Banks:CapitalOne 0.01 USD -2023-03-09 balance Assets:Banks:CapitalOne 4321.98 USD +2023-03-10 balance Assets:Banks:CapitalOne 4321.98 USD From 92b518403ad3778c9e1e5b430d3021e830ff861a Mon Sep 17 00:00:00 2001 From: Red S Date: Sun, 11 Feb 2024 00:35:55 -0800 Subject: [PATCH 11/59] fix: timestamp issue in balance assertions this was causing unit tests to pass or fail on different test servers, because of an off-by-one error on the balance assertion date, depending on the timezone of the host the test was run on. --- README.md | 8 ++++++-- beancount_reds_importers/libreader/ofxreader.py | 5 +---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c108415..95351b2 100644 --- a/README.md +++ b/README.md @@ -133,8 +133,12 @@ If you want something else, simply override this method in individual importer `smart` dates: Banks and credit cards typically have pending transactions that are not included in downloads. When we download the next statement, new transactions may appear -prior to the balance assertion date that we generate for this statement. To attempt to -avoid this, we set the balance assertion date to either two days (fudge factor to +prior to the balance assertion date that we generate for this statement, which renders +this balance assertion invalid. This problem manifests occasionally as an existing +balance statement breaking when a new statement is downloaded and is an annoyance as it +needs manual fixing. + +To minimize this, we set the balance assertion date to either two days (fudge factor to account for pending transactions) before the statement's end date or the last transaction's date, whichever is later. To choose a different fudge factor, simply set `balance_assertion_date_fudge` in your config. diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index 7b8bf9d..b40970e 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -79,11 +79,8 @@ def get_available_cash(self, settlement_fund_balance=0): def get_ofx_end_date(self, field='end_date'): end_date = getattr(self.ofx_account.statement, field, None) - # Convert end_date from utc to local timezone if needed and return - # This is needed only if there is an actual timestamp other than time(0, 0) if end_date: - if end_date.time() != datetime.time(0, 0): - end_date = end_date.replace(tzinfo=datetime.timezone.utc).astimezone() + # We don't care about timestamps, remove them so they don't affect date calculations return end_date.date() return None From 5599e6b0d771ca7cd545919daddcd1286b49597a Mon Sep 17 00:00:00 2001 From: Ammon Sarver Date: Wed, 13 Mar 2024 01:53:52 -0600 Subject: [PATCH 12/59] feat: add pdfreader libreader importer --- .../libreader/pdfreader.py | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 beancount_reds_importers/libreader/pdfreader.py diff --git a/beancount_reds_importers/libreader/pdfreader.py b/beancount_reds_importers/libreader/pdfreader.py new file mode 100644 index 0000000..f9dc6a5 --- /dev/null +++ b/beancount_reds_importers/libreader/pdfreader.py @@ -0,0 +1,185 @@ + +from pprint import pformat +import pdfplumber +import pandas as pd +import petl as etl +from beancount_reds_importers.libreader import csvreader + +LEFT = 0 +TOP = 1 +RIGHT = 2 +BOTTOM = 3 + +BLACK = (0, 0, 0) +RED = (255, 0, 0) +PURPLE = (135,0,255) +TRANSPARENT = (0, 0, 0, 0) + +class Importer(csvreader.Importer): + """ + A reader that converts a pdf with tables into a multi-petl-table format understood by transaction builders. + + + ### Attributes customized in `custom_init` + self.pdf_table_extraction_settings: `{}` + a dictionary containing settings used to extract tables, see [pdfplumber documentation](https://github.com/jsvine/pdfplumber?tab=readme-ov-file#table-extraction-settings) for what settings are available + + self.pdf_table_extraction_crop: `(int,int,int,int)` + a tuple with 4 values representing distance from left, top, right, bottom of the page respectively, + this will crop the input (each page) before searching for tables + + self.pdf_table_title_height: `int` + an integer representing how far up from the top of the table should we look for a table title. + Set to 0 to not extract table titles, in which case sections will be labelled as `table_#` in the order + they were encountered + + self.pdf_page_break_top: `int` + an integer representing the threshold where a table can be considered page-broken. If the top of a table is + lower than the provided value, it will be in consideration for amending to the previous page's last table. + Set to 0 to never consider page-broken tables + + self.debug: `boolean` + When debug is True a few images and text file are generated: + .debug-pdf-metadata-page_#.png + shows the text available in self.meta_text with table data blacked out + + .debug-pdf-table-detection-page_#.png + shows the tables detected with cells outlined in red, and the background light blue. The purple box shows where we are looking for the table title. + + .debug-pdf-data.txt + is a printout of the meta_text and table data found before being processed into petl tables, as well as some generated helper objects to add to new importers or import configs + + ### Outputs + self.meta_text: `str` + contains all text found in the document outside of tables + + self.alltables: `{'table_1': , ...}` + contains all the tables found in the document keyed by the extracted title if available, otherwise by the 1-based index in the form of `table_#` + """ + FILE_EXTS = ['pdf'] + + def initialize_reader(self, file): + if getattr(self, 'file', None) != file: + self.pdf_table_extraction_settings = {} + self.pdf_table_extraction_crop = (0, 0, 0, 0) + self.pdf_table_title_height = 20 + self.pdf_page_break_top = 45 + self.debug = False + + self.meta_text = '' + self.file = file + self.file_read_done = False + self.reader_ready = True + + def file_date(self, file): + raise "Not implemented, must overwrite, check self.alltables, or self.meta_text for the data" + pass + + def prepare_tables(self): + return + + def read_file(self, file): + tables = [] + + with pdfplumber.open(file.name) as pdf: + for page_idx, page in enumerate(pdf.pages): + # all bounding boxes are (left, top, right, bottom) + adjusted_crop = ( + min(0 + self.pdf_table_extraction_crop[LEFT], page.width), + min(0 + self.pdf_table_extraction_crop[TOP], page.height), + max(page.width - self.pdf_table_extraction_crop[RIGHT],0), + max(page.height - self.pdf_table_extraction_crop[BOTTOM],0) + ) + + # Debug image + image = page.crop(adjusted_crop).to_image() + image.debug_tablefinder(tf=self.pdf_table_extraction_settings) + + table_ref = page.crop(adjusted_crop).find_tables(table_settings=self.pdf_table_extraction_settings) + page_tables = [{'table':i.extract(), 'bbox': i.bbox} for i in table_ref] + + # Get Metadata (all data outside tables) + meta_page = page + meta_image = meta_page.to_image() + for table in page_tables: + meta_page = meta_page.outside_bbox(table['bbox']) + meta_image.draw_rect(table['bbox'], BLACK, RED) + + meta_text = meta_page.extract_text() + self.meta_text = self.meta_text + meta_text + + # Attach section headers + for table_idx, table in enumerate(page_tables): + section_title_bbox = ( + table['bbox'][LEFT], + max(table['bbox'][TOP] - self.pdf_table_title_height, 0), + table['bbox'][RIGHT], + table['bbox'][TOP] + ) + section_title = meta_page.crop(section_title_bbox).extract_text() + image.draw_rect(section_title_bbox, TRANSPARENT, PURPLE) + page_tables[table_idx]['section'] = section_title + + tables = tables + page_tables + + if self.debug: + image.save('.debug-pdf-table-detection-page_{}.png'.format(page_idx)) + meta_image.save('.debug-pdf-metadata-page_{}.png'.format(page_idx)) + + + # Find and fix page broken tables + for table_idx, table in enumerate(tables[:]): + if ( + table_idx >= 1 and # if not the first table, + table['bbox'][TOP] < self.pdf_page_break_top and # and the top of the table is close to the top of the page + table['section'] == '' and # and there is no section title + tables[table_idx - 1]['table'][0] == tables[table_idx]['table'][0] # and the header rows are the same, + ): #assume a page break + tables[table_idx - 1]['table'] = tables[table_idx - 1]['table'] + tables[table_idx]['table'][1:] + del tables[table_idx] + continue + + # if there is no table section give it one + if table['section'] == '': + tables[table_idx]['section'] = 'table_{}'.format(table_idx + 1) + + if self.debug: + # generate helpers + paycheck_template = {} + header_map = {} + for table in tables: + for header in table['table'][0]: + header_map[header]='overwrite_me' + paycheck_template[table['section']] = {} + for row_idx, row in enumerate(table['table']): + if row_idx == 0: + continue + paycheck_template[table['section']][row[0]] = 'overwrite_me' + if not hasattr(self, 'header_map'): + self.header_map = header_map + if not hasattr(self, 'paycheck_template'): + self.paycheck_template = paycheck_template + with open('.debug-pdf-data.txt', "w") as debug_file: + debug_file.write(pformat({ + '_output': { + 'tables':tables, + 'meta_text':self.meta_text + }, + '_input': { + 'table_settings': self.pdf_table_extraction_settings, + 'crop_settings': self.pdf_table_extraction_crop + }, + 'helpers': { + 'header_map':self.header_map, + 'paycheck_template':self.paycheck_template + } + })) + + + + self.alltables = {} + for table in tables: + self.alltables[table['section']] = etl.fromdataframe(pd.DataFrame(table['table'][1:], columns=table['table'][0])) + + self.prepare_tables() + self.file_read_done = True From 041e00617989fc103ad10431bca78c9fa0cde763 Mon Sep 17 00:00:00 2001 From: Ammon Sarver Date: Wed, 13 Mar 2024 01:54:13 -0600 Subject: [PATCH 13/59] feat: add bamboohr paycheck importer --- .../importers/bamboohr/__init__.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 beancount_reds_importers/importers/bamboohr/__init__.py diff --git a/beancount_reds_importers/importers/bamboohr/__init__.py b/beancount_reds_importers/importers/bamboohr/__init__.py new file mode 100644 index 0000000..2818190 --- /dev/null +++ b/beancount_reds_importers/importers/bamboohr/__init__.py @@ -0,0 +1,62 @@ +"""BambooHR paycheck importer""" + +import re +from dateparser.search import search_dates +from beancount_reds_importers.libreader import pdfreader +from beancount_reds_importers.libtransactionbuilder import paycheck + +# BambooHR exports paycheck stubs to pdf, with multiple tables across multiple pages. +# Call this importer with a config that looks like: +# +# bamboohr.Importer({"desc":"Paycheck (My Company)", +# "main_account":"Income:Employment", +# "paycheck_template": {}, # See beancount_reds_importers/libtransactionbuilder/paycheck.py for sample template +# "currency": "PENNIES", +# }), +# + +class Importer(paycheck.Importer, pdfreader.Importer): + IMPORTER_NAME = 'BambooHR Paycheck' + + def custom_init(self): + self.max_rounding_error = 0.04 + self.filename_pattern_def = 'PayStub.*\.pdf' + self.pdf_table_extraction_settings = {"join_tolerance":4, "snap_tolerance": 4} + self.pdf_table_extraction_crop = (0, 40, 0, 0) + self.debug = False + + self.funds_db_txt = 'funds_by_ticker' + self.header_map = { + 'Deduction Type': 'description', + 'Pay Type': 'description', + 'Paycheck Total': 'amount', + 'Tax Type': 'description' + } + + self.currency_fields = ['ytd_total', 'amount'] + + def paycheck_date(self, input_file): + if not self.file_read_done: + self.read_file(input_file) + dates = [date for _, date in search_dates(self.meta_text)] + return dates[2].date() + + def prepare_tables(self): + def valid_header(label): + if label in self.header_map: + return self.header_map[header] + + label = label.lower().replace(' ', '_') + return re.sub(r'20\d{2}', 'ytd', label) + + for section, table in self.alltables.items(): + # rename columns + for header in table.header(): + table = table.rename(header,valid_header(header)) + # convert columns + table = self.convert_columns(table) + + self.alltables[section] = table + + def build_metadata(self, file, metatype=None, data={}): + return {'filing_account': self.config['main_account']} \ No newline at end of file From 80afa1b0222ae776826393fc53a230db73892a22 Mon Sep 17 00:00:00 2001 From: Red S Date: Fri, 15 Mar 2024 00:55:25 -0700 Subject: [PATCH 14/59] feat: add 'show_configured' in paycheck transaction builder This lists all entries in the paycheck which are being ignored by the current configuration. This is useful when new items appear on the paycheck and the configuration needs to be appended with it. --- .../libtransactionbuilder/paycheck.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index d6709fe..24efeb7 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -3,7 +3,7 @@ from beancount.core import data from beancount.core.number import D from beancount_reds_importers.libtransactionbuilder import banking - +from collections import defaultdict # paychecks are typically transaction with many (10-40) postings including several each of income, taxes, # pre-tax and post-tax deductions, transfers, reimbursements, etc. This importer enables importing a single @@ -66,15 +66,19 @@ def build_postings(self, entry): template = self.config['paycheck_template'] currency = self.config['currency'] total = 0 + template_missing = defaultdict(set) for section, table in self.alltables.items(): if section not in template: + template_missing[section] = set() continue for row in table.namedtuples(): # TODO: 'bank' is workday specific; move it there row_description = getattr(row, 'description', getattr(row, 'bank', None)) row_pattern = next(filter(lambda ts: row_description.startswith(ts), template[section]), None) - if row_pattern: + if not row_pattern: + template_missing[section].add(row_description) + else: accounts = template[section][row_pattern] accounts = [accounts] if not isinstance(accounts, list) else accounts for account in accounts: @@ -89,6 +93,14 @@ def build_postings(self, entry): total += amount if amount: data.create_simple_posting(entry, account, amount, currency) + + if self.config.get('show_unconfigured', False): + for section in template_missing: + print(section) + if template_missing[section]: + print(' ' + '\n '.join(i for i in template_missing[section])) + print() + if total != 0: data.create_simple_posting(entry, "TOTAL:NONZERO", total, currency) From 1c9adcd32f94ce4c21f9e1e70a9f65d7c905cd4a Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 23 Mar 2024 14:10:09 -0700 Subject: [PATCH 15/59] ci: fix conventionalcommits branch name so it runs on PRs --- .github/workflows/conventionalcommits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conventionalcommits.yml b/.github/workflows/conventionalcommits.yml index b7fa7a1..add48c6 100644 --- a/.github/workflows/conventionalcommits.yml +++ b/.github/workflows/conventionalcommits.yml @@ -2,7 +2,7 @@ name: Conventional Commits on: pull_request: - branches: [ master ] + branches: [ main ] jobs: build: From b75c8e55bf0ca6a4b8e3c906a8fb5043c4910545 Mon Sep 17 00:00:00 2001 From: Rane Brown Date: Fri, 22 Mar 2024 11:36:01 -0600 Subject: [PATCH 16/59] feat: enforce formatting with ruff --- .github/workflows/pythonpackage.yml | 3 + .ruff.toml | 8 +- beancount_reds_importers/example/fund_info.py | 14 +- .../importers/alliant/__init__.py | 4 +- .../importers/ally/__init__.py | 4 +- .../importers/amazongc/__init__.py | 40 +- .../importers/amex/__init__.py | 4 +- .../importers/becu/__init__.py | 4 +- .../importers/capitalonebank/__init__.py | 4 +- .../importers/chase/__init__.py | 4 +- .../importers/citi/__init__.py | 4 +- .../importers/dcu/__init__.py | 17 +- .../importers/discover/__init__.py | 24 +- .../importers/discover/discover_ofx.py | 4 +- .../importers/etrade/__init__.py | 8 +- .../etrade/tests/etrade_qfx_brokerage_test.py | 54 ++- .../importers/fidelity/__init__.py | 18 +- .../importers/fidelity/fidelity_cma_csv.py | 46 ++- .../importers/morganstanley/__init__.py | 6 +- .../importers/schwab/schwab_csv_balances.py | 43 ++- .../importers/schwab/schwab_csv_brokerage.py | 105 ++--- .../importers/schwab/schwab_csv_checking.py | 52 +-- .../importers/schwab/schwab_csv_creditline.py | 7 +- .../importers/schwab/schwab_csv_positions.py | 31 +- .../importers/schwab/schwab_json_brokerage.py | 12 +- .../importers/schwab/schwab_ofx_bank_ofx.py | 4 +- .../importers/schwab/schwab_ofx_brokerage.py | 6 +- .../schwab_csv_brokerage_test.py | 56 ++- .../schwab_csv_checking_test.py | 7 +- .../importers/stanchart/scbbank.py | 64 ++-- .../importers/stanchart/scbcard.py | 72 ++-- .../importers/target/__init__.py | 4 +- .../importers/tdameritrade/__init__.py | 9 +- .../importers/techcubank/__init__.py | 4 +- .../unitedoverseas/tests/uobbank_test.py | 14 +- .../importers/unitedoverseas/uobbank.py | 46 ++- .../importers/unitedoverseas/uobcard.py | 60 +-- .../importers/unitedoverseas/uobsrs.py | 45 ++- .../importers/vanguard/__init__.py | 20 +- .../vanguard/vanguard_screenscrape.py | 82 ++-- .../importers/workday/__init__.py | 30 +- .../libreader/csv_multitable_reader.py | 24 +- .../libreader/csvreader.py | 54 ++- .../libreader/jsonreader.py | 9 +- .../libreader/ofxreader.py | 82 ++-- beancount_reds_importers/libreader/reader.py | 27 +- .../last_transaction_date_test.py | 12 +- .../ofx_date/ofx_date_test.py | 12 +- .../smart/smart_date_test.py | 10 +- .../libreader/tsvreader.py | 3 +- .../libreader/xlsreader.py | 10 +- .../libreader/xlsx_multitable_reader.py | 6 +- .../libreader/xlsxreader.py | 2 +- .../libtransactionbuilder/banking.py | 65 ++-- .../libtransactionbuilder/common.py | 66 +++- .../libtransactionbuilder/investments.py | 360 ++++++++++++------ .../libtransactionbuilder/paycheck.py | 54 ++- .../transactionbuilder.py | 19 +- .../util/bean_download.py | 107 ++++-- beancount_reds_importers/util/needs_update.py | 113 ++++-- .../util/ofx_summarize.py | 63 ++- setup.py | 64 ++-- 62 files changed, 1342 insertions(+), 863 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 172e3d6..af768a6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -34,3 +34,6 @@ jobs: - name: Test with pytest run: | pytest + - name: Check format with ruff + run: | + ruff format --check diff --git a/.ruff.toml b/.ruff.toml index 46121f3..03403de 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1 +1,7 @@ -line-length = 127 +line-length = 88 + +[format] +docstring-code-format = true +indent-style = "space" +line-ending = "lf" +quote-style = "double" diff --git a/beancount_reds_importers/example/fund_info.py b/beancount_reds_importers/example/fund_info.py index 807e647..65858d2 100755 --- a/beancount_reds_importers/example/fund_info.py +++ b/beancount_reds_importers/example/fund_info.py @@ -20,15 +20,15 @@ # mutual funds since those are brokerage specific. fund_data = [ - ('SCHF', '808524805', 'Schwab International Equity ETF'), - ('VGTEST', '012345678', 'Vanguard Test Fund'), - ('VMFXX', '922906300', 'Vanguard Federal Money Market Fund'), + ("SCHF", "808524805", "Schwab International Equity ETF"), + ("VGTEST", "012345678", "Vanguard Test Fund"), + ("VMFXX", "922906300", "Vanguard Federal Money Market Fund"), ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMFXX'] +money_market = ["VMFXX"] fund_info = { - 'fund_data': fund_data, - 'money_market': money_market, - } + "fund_data": fund_data, + "money_market": money_market, +} diff --git a/beancount_reds_importers/importers/alliant/__init__.py b/beancount_reds_importers/importers/alliant/__init__.py index a67681c..dd6c587 100644 --- a/beancount_reds_importers/importers/alliant/__init__.py +++ b/beancount_reds_importers/importers/alliant/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Alliant Credit Union' + IMPORTER_NAME = "Alliant Credit Union" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*alliant' + self.filename_pattern_def = ".*alliant" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/ally/__init__.py b/beancount_reds_importers/importers/ally/__init__.py index 3764d0f..08b1264 100644 --- a/beancount_reds_importers/importers/ally/__init__.py +++ b/beancount_reds_importers/importers/ally/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Ally Bank' + IMPORTER_NAME = "Ally Bank" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*transactions' + self.filename_pattern_def = ".*transactions" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/amazongc/__init__.py b/beancount_reds_importers/importers/amazongc/__init__.py index f4f486d..e1f5d3b 100644 --- a/beancount_reds_importers/importers/amazongc/__init__.py +++ b/beancount_reds_importers/importers/amazongc/__init__.py @@ -37,25 +37,25 @@ class Importer(importer.ImporterProtocol): def __init__(self, config): self.config = config - self.currency = self.config.get('currency', 'CURRENCY_NOT_CONFIGURED') - self.filename_pattern_def = 'amazon-gift-card.tsv' + self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED") + self.filename_pattern_def = "amazon-gift-card.tsv" def identify(self, file): return self.filename_pattern_def in file.name def file_name(self, file): - return '{}'.format(ntpath.basename(file.name)) + return "{}".format(ntpath.basename(file.name)) def file_account(self, _): - return self.config['main_account'] + return self.config["main_account"] def file_date(self, file): "Get the maximum date from the file." maxdate = datetime.date.min - for line in open(file.name, 'r').readlines()[1:]: - f = line.split('\t') + for line in open(file.name, "r").readlines()[1:]: + f = line.split("\t") f = [i.strip() for i in f] - date = datetime.datetime.strptime(f[0], '%B %d, %Y').date() + date = datetime.datetime.strptime(f[0], "%B %d, %Y").date() maxdate = max(date, maxdate) return maxdate @@ -65,18 +65,28 @@ def extract(self, file, existing_entries=None): new_entries = [] counter = itertools.count() - for line in open(file.name, 'r').readlines()[1:]: - f = line.split('\t') + for line in open(file.name, "r").readlines()[1:]: + f = line.split("\t") f = [i.strip() for i in f] - date = datetime.datetime.strptime(f[0], '%B %d, %Y').date() + date = datetime.datetime.strptime(f[0], "%B %d, %Y").date() description = f[1].encode("ascii", "ignore").decode() - number = D(f[2].replace('$', '')) + number = D(f[2].replace("$", "")) metadata = data.new_metadata(file.name, next(counter)) - entry = data.Transaction(metadata, date, self.FLAG, - None, description, data.EMPTY_SET, data.EMPTY_SET, []) - data.create_simple_posting(entry, config['main_account'], number, self.currency) - data.create_simple_posting(entry, config['target_account'], None, None) + entry = data.Transaction( + metadata, + date, + self.FLAG, + None, + description, + data.EMPTY_SET, + data.EMPTY_SET, + [], + ) + data.create_simple_posting( + entry, config["main_account"], number, self.currency + ) + data.create_simple_posting(entry, config["target_account"], None, None) new_entries.append(entry) return new_entries diff --git a/beancount_reds_importers/importers/amex/__init__.py b/beancount_reds_importers/importers/amex/__init__.py index 8ebae4e..086cb0a 100644 --- a/beancount_reds_importers/importers/amex/__init__.py +++ b/beancount_reds_importers/importers/amex/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'American Express' + IMPORTER_NAME = "American Express" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*amex' + self.filename_pattern_def = ".*amex" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/becu/__init__.py b/beancount_reds_importers/importers/becu/__init__.py index 3f978fe..a0a019a 100644 --- a/beancount_reds_importers/importers/becu/__init__.py +++ b/beancount_reds_importers/importers/becu/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'BECU' + IMPORTER_NAME = "BECU" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*becu' + self.filename_pattern_def = ".*becu" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/capitalonebank/__init__.py b/beancount_reds_importers/importers/capitalonebank/__init__.py index 43a93d2..48c1043 100644 --- a/beancount_reds_importers/importers/capitalonebank/__init__.py +++ b/beancount_reds_importers/importers/capitalonebank/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Capital One Bank' + IMPORTER_NAME = "Capital One Bank" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*360Checking' + self.filename_pattern_def = ".*360Checking" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/chase/__init__.py b/beancount_reds_importers/importers/chase/__init__.py index 4e70ebf..4a79edc 100644 --- a/beancount_reds_importers/importers/chase/__init__.py +++ b/beancount_reds_importers/importers/chase/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Chase' + IMPORTER_NAME = "Chase" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*[Cc]hase' + self.filename_pattern_def = ".*[Cc]hase" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/citi/__init__.py b/beancount_reds_importers/importers/citi/__init__.py index 4460ae2..c4d210f 100644 --- a/beancount_reds_importers/importers/citi/__init__.py +++ b/beancount_reds_importers/importers/citi/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Citi' + IMPORTER_NAME = "Citi" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*citi' + self.filename_pattern_def = ".*citi" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/dcu/__init__.py b/beancount_reds_importers/importers/dcu/__init__.py index d818a1b..6837ddf 100644 --- a/beancount_reds_importers/importers/dcu/__init__.py +++ b/beancount_reds_importers/importers/dcu/__init__.py @@ -14,19 +14,20 @@ def custom_init(self): self.header_identifier = "" self.column_labels_line = '"DATE","TRANSACTION TYPE","DESCRIPTION","AMOUNT","ID","MEMO","CURRENT BALANCE"' self.date_format = "%m/%d/%Y" + # fmt: off self.header_map = { - "DATE": "date", - "DESCRIPTION": "payee", - "MEMO": "memo", - "AMOUNT": "amount", - "CURRENT BALANCE": "balance", + "DATE": "date", + "DESCRIPTION": "payee", + "MEMO": "memo", + "AMOUNT": "amount", + "CURRENT BALANCE": "balance", "TRANSACTION TYPE": "type", } - self.transaction_type_map = { - "DEBIT": "transfer", - "CREDIT": "transfer", + "DEBIT": "transfer", + "CREDIT": "transfer", } + # fmt: on self.skip_transaction_types = [] def get_balance_statement(self, file=None): diff --git a/beancount_reds_importers/importers/discover/__init__.py b/beancount_reds_importers/importers/discover/__init__.py index a554e07..d06cdfc 100644 --- a/beancount_reds_importers/importers/discover/__init__.py +++ b/beancount_reds_importers/importers/discover/__init__.py @@ -1,4 +1,4 @@ -""" Discover credit card .csv importer.""" +"""Discover credit card .csv importer.""" from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking @@ -9,21 +9,23 @@ class Importer(csvreader.Importer, banking.Importer): def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'Discover.*' - self.header_identifier = 'Trans. Date,Post Date,Description,Amount,Category' - self.date_format = '%m/%d/%Y' + self.filename_pattern_def = "Discover.*" + self.header_identifier = "Trans. Date,Post Date,Description,Amount,Category" + self.date_format = "%m/%d/%Y" + # fmt: off self.header_map = { - "Category": 'payee', - "Description": 'memo', - "Trans. Date": 'date', - "Post Date": 'postDate', - "Amount": 'amount', - } + "Category": "payee", + "Description": "memo", + "Trans. Date": "date", + "Post Date": "postDate", + "Amount": "amount", + } + # fmt: on def skip_transaction(self, ot): return False def prepare_processed_table(self, rdr): # Need to invert numbers supplied by Discover - rdr = rdr.convert('amount', lambda x: -1 * x) + rdr = rdr.convert("amount", lambda x: -1 * x) return rdr diff --git a/beancount_reds_importers/importers/discover/discover_ofx.py b/beancount_reds_importers/importers/discover/discover_ofx.py index 0593a2e..abee824 100644 --- a/beancount_reds_importers/importers/discover/discover_ofx.py +++ b/beancount_reds_importers/importers/discover/discover_ofx.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Discover' + IMPORTER_NAME = "Discover" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*Discover' + self.filename_pattern_def = ".*Discover" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/etrade/__init__.py b/beancount_reds_importers/importers/etrade/__init__.py index b5eb96a..bc6d3d3 100644 --- a/beancount_reds_importers/importers/etrade/__init__.py +++ b/beancount_reds_importers/importers/etrade/__init__.py @@ -1,18 +1,18 @@ -""" ETrade Brokerage ofx importer.""" +"""ETrade Brokerage ofx importer.""" from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'ETrade Brokerage OFX' + IMPORTER_NAME = "ETrade Brokerage OFX" def custom_init(self): self.max_rounding_error = 0.11 - self.filename_pattern_def = '.*etrade' + self.filename_pattern_def = ".*etrade" self.get_ticker_info = self.get_ticker_info_from_id def skip_transaction(self, ot): - if 'JNL' in ot.memo: + if "JNL" in ot.memo: return True return False diff --git a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py index fefd32d..1ad8980 100644 --- a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py +++ b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py @@ -6,49 +6,45 @@ fund_data = [ - ('TSM', '874039100', 'Taiwan Semiconductor Mfg LTD'), - ('VISA', '92826C839', 'Visa Inc'), + ("TSM", "874039100", "Taiwan Semiconductor Mfg LTD"), + ("VISA", "92826C839", "Visa Inc"), ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMFXX'] +money_market = ["VMFXX"] fund_info = { - 'fund_data': fund_data, - 'money_market': money_market, - } + "fund_data": fund_data, + "money_market": money_market, +} def build_config(): acct = "Assets:Investments:Etrade" - root = 'Investments' - taxability = 'Taxable' - leaf = 'Etrade' - currency = 'USD' + root = "Investments" + taxability = "Taxable" + leaf = "Etrade" + currency = "USD" config = { - 'account_number' : '555555555', - 'main_account' : acct + ':{ticker}', - 'cash_account' : f'{acct}:{{currency}}', - 'transfer' : 'Assets:Zero-Sum-Accounts:Transfers:Bank-Account', - 'dividends' : f'Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}', - 'interest' : f'Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}', - 'cg' : f'Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}', - 'capgainsd_lt' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}', - 'capgainsd_st' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}', - 'fees' : f'Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}', - 'invexpense' : f'Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}', - 'rounding_error' : 'Equity:Rounding-Errors:Imports', - 'fund_info' : fund_info, - 'currency' : currency, + "account_number": "555555555", + "main_account": acct + ":{ticker}", + "cash_account": f"{acct}:{{currency}}", + "transfer": "Assets:Zero-Sum-Accounts:Transfers:Bank-Account", + "dividends": f"Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}", + "interest": f"Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}", + "cg": f"Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}", + "capgainsd_lt": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}", + "capgainsd_st": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}", + "fees": f"Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}", + "invexpense": f"Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}", + "rounding_error": "Equity:Rounding-Errors:Imports", + "fund_info": fund_info, + "currency": currency, } return config -@regtest.with_importer( - etrade.Importer( - build_config() - ) -) +@regtest.with_importer(etrade.Importer(build_config())) @regtest.with_testdir(path.dirname(__file__)) class TestEtradeQFX(regtest.ImporterTestBase): pass diff --git a/beancount_reds_importers/importers/fidelity/__init__.py b/beancount_reds_importers/importers/fidelity/__init__.py index d0438b7..3330376 100644 --- a/beancount_reds_importers/importers/fidelity/__init__.py +++ b/beancount_reds_importers/importers/fidelity/__init__.py @@ -6,27 +6,31 @@ class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Fidelity Net Benefits / Fidelity Investments OFX' + IMPORTER_NAME = "Fidelity Net Benefits / Fidelity Investments OFX" def custom_init(self): self.max_rounding_error = 0.18 - self.filename_pattern_def = '.*fidelity' + self.filename_pattern_def = ".*fidelity" self.get_ticker_info = self.get_ticker_info_from_id - self.get_payee = lambda ot: ot.memo.split(";", 1)[0] if ';' in ot.memo else ot.memo + self.get_payee = ( + lambda ot: ot.memo.split(";", 1)[0] if ";" in ot.memo else ot.memo + ) def security_narration(self, ot): ticker, ticker_long_name = self.get_ticker_info(ot.security) return f"[{ticker}]" def file_name(self, file): - return 'fidelity-{}-{}'.format(self.config['account_number'], ntpath.basename(file.name)) + return "fidelity-{}-{}".format( + self.config["account_number"], ntpath.basename(file.name) + ) def get_target_acct_custom(self, transaction, ticker=None): if transaction.memo.startswith("CONTRIBUTION"): - return self.config['transfer'] + return self.config["transfer"] if transaction.memo.startswith("FEES"): - return self.config['fees'] + return self.config["fees"] return None def get_available_cash(self, settlement_fund_balance=0): - return getattr(self.ofx_account.statement, 'available_cash', None) + return getattr(self.ofx_account.statement, "available_cash", None) diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py index c609e1c..e6abf7b 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py @@ -6,45 +6,49 @@ class Importer(banking.Importer, csvreader.Importer): - IMPORTER_NAME = 'Fidelity Cash Management Account' + IMPORTER_NAME = "Fidelity Cash Management Account" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*History' - self.date_format = '%m/%d/%Y' + self.filename_pattern_def = ".*History" + self.date_format = "%m/%d/%Y" header_s0 = ".*Run Date,Action,Symbol,Security Description,Security Type,Quantity,Price \\(\\$\\)," header_s1 = "Commission \\(\\$\\),Fees \\(\\$\\),Accrued Interest \\(\\$\\),Amount \\(\\$\\),Settlement Date" header_sum = header_s0 + header_s1 self.header_identifier = header_sum self.skip_head_rows = 5 self.skip_tail_rows = 16 + # fmt: off self.header_map = { - "Run Date": 'date', - "Action": 'description', - "Amount ($)": 'amount', - - "Settlement Date": 'settleDate', - "Accrued Interest ($)": 'accrued_interest', - "Fees ($)": 'fees', - "Security Type": 'security_type', - "Commission ($)": 'commission', - "Security Description": 'security_description', - "Symbol": 'security', - "Price ($)": 'unit_price', - } + "Run Date": "date", + "Action": "description", + "Amount ($)": "amount", + "Settlement Date": "settleDate", + "Accrued Interest ($)": "accrued_interest", + "Fees ($)": "fees", + "Security Type": "security_type", + "Commission ($)": "commission", + "Security Description": "security_description", + "Symbol": "security", + "Price ($)": "unit_price", + } + # fmt: on def deep_identify(self, file): return re.match(self.header_identifier, file.head(), flags=re.DOTALL) def prepare_raw_columns(self, rdr): - - for field in ['Action']: + for field in ["Action"]: rdr = rdr.convert(field, lambda x: x.lstrip()) - rdr = rdr.capture('Action', '(?:\\s)(?:\\w*)(.*)', ['memo'], include_original=True) - rdr = rdr.capture('Action', '(\\S+(?:\\s+\\S+)?)', ['payee'], include_original=True) + rdr = rdr.capture( + "Action", "(?:\\s)(?:\\w*)(.*)", ["memo"], include_original=True + ) + rdr = rdr.capture( + "Action", "(\\S+(?:\\s+\\S+)?)", ["payee"], include_original=True + ) - for field in ['memo', 'payee']: + for field in ["memo", "payee"]: rdr = rdr.convert(field, lambda x: x.lstrip()) return rdr diff --git a/beancount_reds_importers/importers/morganstanley/__init__.py b/beancount_reds_importers/importers/morganstanley/__init__.py index 7084e1d..5a859d8 100644 --- a/beancount_reds_importers/importers/morganstanley/__init__.py +++ b/beancount_reds_importers/importers/morganstanley/__init__.py @@ -1,13 +1,13 @@ -""" Morgan Stanley Investments ofx importer.""" +"""Morgan Stanley Investments ofx importer.""" from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Morgan Stanley Investments' + IMPORTER_NAME = "Morgan Stanley Investments" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*morganstanley' + self.filename_pattern_def = ".*morganstanley" self.get_ticker_info = self.get_ticker_info_from_id diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py index 5f52426..1ea5361 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py @@ -1,4 +1,4 @@ -""" Schwab csv importer.""" +"""Schwab csv importer.""" import datetime import re @@ -8,35 +8,38 @@ class Importer(investments.Importer, csv_multitable_reader.Importer): - IMPORTER_NAME = 'Schwab Brokerage Balances CSV' + IMPORTER_NAME = "Schwab Brokerage Balances CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Balances_' - self.header_identifier = 'Balances for account' + self.filename_pattern_def = ".*_Balances_" + self.header_identifier = "Balances for account" self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" + # fmt: off self.header_map = { - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - } + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + } + # fmt: on def prepare_table(self, rdr): return rdr def convert_columns(self, rdr): # fixup decimals - decimals = ['units'] + decimals = ["units"] for i in decimals: rdr = rdr.convert(i, D) # fixup currencies def remove_non_numeric(x): - return re.sub(r'[^0-9\.]', "", x) # noqa: W605 - currencies = ['unit_price'] + return re.sub(r"[^0-9\.]", "", x) # noqa: W605 + + currencies = ["unit_price"] for i in currencies: rdr = rdr.convert(i, remove_non_numeric) rdr = rdr.convert(i, D) @@ -51,18 +54,18 @@ def get_max_transaction_date(self): def prepare_tables(self): # first row has date - d = self.raw_rdr[0][0].rsplit(' ', 1)[1] + d = self.raw_rdr[0][0].rsplit(" ", 1)[1] self.date = datetime.datetime.strptime(d, self.date_format) for section, table in self.alltables.items(): - if section in self.config['section_headers']: + if section in self.config["section_headers"]: table = table.rename(self.header_map) table = self.convert_columns(table) - table = table.cut('memo', 'security', 'units', 'unit_price') - table = table.selectne('memo', '--') # we don't need total rows - table = table.addfield('date', self.date) + table = table.cut("memo", "security", "units", "unit_price") + table = table.selectne("memo", "--") # we don't need total rows + table = table.addfield("date", self.date) self.alltables[section] = table def get_balance_positions(self): - for section in self.config['section_headers']: + for section in self.config["section_headers"]: yield from self.alltables[section].namedtuples() diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index 01583fc..0f91fa9 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -1,4 +1,4 @@ -""" Schwab Brokerage .csv importer.""" +"""Schwab Brokerage .csv importer.""" import re from beancount_reds_importers.libreader import csvreader @@ -6,71 +6,76 @@ class Importer(csvreader.Importer, investments.Importer): - IMPORTER_NAME = 'Schwab Brokerage CSV' + IMPORTER_NAME = "Schwab Brokerage CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Transactions_' - self.header_identifier = '' + self.filename_pattern_def = ".*_Transactions_" + self.header_identifier = "" self.column_labels_line = '"Date","Action","Symbol","Description","Quantity","Price","Fees & Comm","Amount"' self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" self.get_payee = lambda ot: ot.Action + # fmt: off self.header_map = { - "Date": 'date', - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - "Amount": 'amount', - # "tradeDate": 'tradeDate', - # "total": 'total', - "Fees & Comm": 'fees', - } + "Date": "date", + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + "Amount": "amount", + # "tradeDate": "tradeDate", + # "total": "total", + "Fees & Comm": "fees", + } self.transaction_type_map = { - 'Bank Interest': 'income', - 'Bank Transfer': 'cash', - 'Buy': 'buystock', - 'Journaled Shares': 'buystock', # These are in-kind tranfers - 'Reinvestment Adj': 'buystock', - 'Div Adjustment': 'dividends', - 'Long Term Cap Gain Reinvest': 'capgainsd_lt', - 'Misc Credits': 'cash', - 'MoneyLink Deposit': 'cash', - 'MoneyLink Transfer': 'cash', - 'Pr Yr Div Reinvest': 'dividends', - 'Journal': 'cash', # These are transfers - 'Reinvest Dividend': 'dividends', - 'Qualified Dividend': 'dividends', - 'Cash Dividend': 'dividends', - 'Reinvest Shares': 'buystock', - 'Sell': 'sellstock', - 'Short Term Cap Gain Reinvest': 'capgainsd_st', - 'Wire Funds Received': 'cash', - 'Wire Received': 'cash', - 'Funds Received': 'cash', - 'Stock Split': 'cash', - 'Cash In Lieu': 'cash', - } + "Bank Interest": "income", + "Bank Transfer": "cash", + "Buy": "buystock", + "Journaled Shares": "buystock", # These are in-kind tranfers + "Reinvestment Adj": "buystock", + "Div Adjustment": "dividends", + "Long Term Cap Gain Reinvest": "capgainsd_lt", + "Misc Credits": "cash", + "MoneyLink Deposit": "cash", + "MoneyLink Transfer": "cash", + "Pr Yr Div Reinvest": "dividends", + "Journal": "cash", # These are transfers + "Reinvest Dividend": "dividends", + "Qualified Dividend": "dividends", + "Cash Dividend": "dividends", + "Reinvest Shares": "buystock", + "Sell": "sellstock", + "Short Term Cap Gain Reinvest": "capgainsd_st", + "Wire Funds Received": "cash", + "Wire Received": "cash", + "Funds Received": "cash", + "Stock Split": "cash", + "Cash In Lieu": "cash", + } + # fmt: on def deep_identify(self, file): - last_three = self.config.get('account_number', '')[-3:] - return re.match(self.header_identifier, file.head()) and f'XX{last_three}' in file.name + last_three = self.config.get("account_number", "")[-3:] + return ( + re.match(self.header_identifier, file.head()) + and f"XX{last_three}" in file.name + ) def skip_transaction(self, ot): - return ot.type in ['', 'Journal'] + return ot.type in ["", "Journal"] def prepare_table(self, rdr): - if '' in rdr.fieldnames(): - rdr = rdr.cutout('') # clean up last column + if "" in rdr.fieldnames(): + rdr = rdr.cutout("") # clean up last column def cleanup_date(d): """'11/16/2018 as of 11/15/2018' --> '11/16/2018'""" - return d.split(' ', 1)[0] + return d.split(" ", 1)[0] - rdr = rdr.convert('Date', cleanup_date) - rdr = rdr.addfield('tradeDate', lambda x: x['Date']) - rdr = rdr.addfield('total', lambda x: x['Amount']) - rdr = rdr.addfield('type', lambda x: x['Action']) + rdr = rdr.convert("Date", cleanup_date) + rdr = rdr.addfield("tradeDate", lambda x: x["Date"]) + rdr = rdr.addfield("total", lambda x: x["Amount"]) + rdr = rdr.addfield("type", lambda x: x["Action"]) return rdr diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py index 7a3072c..8284364 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py @@ -1,43 +1,47 @@ -""" Schwab Checking .csv importer.""" +"""Schwab Checking .csv importer.""" from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking class Importer(csvreader.Importer, banking.Importer): - IMPORTER_NAME = 'Schwab Checking account CSV' + IMPORTER_NAME = "Schwab Checking account CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Checking_Transactions_' - self.header_identifier = '' + self.filename_pattern_def = ".*_Checking_Transactions_" + self.header_identifier = "" self.column_labels_line = '"Date","Status","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' - self.date_format = '%m/%d/%Y' - self.skip_comments = '# ' + self.date_format = "%m/%d/%Y" + self.skip_comments = "# " + # fmt: off self.header_map = { - "Date": "date", - "Type": "type", - "CheckNumber": "checknum", - "Description": "payee", - "Withdrawal": "withdrawal", - "Deposit": "deposit", - "RunningBalance": "balance" + "Date": "date", + "Type": "type", + "CheckNumber": "checknum", + "Description": "payee", + "Withdrawal": "withdrawal", + "Deposit": "deposit", + "RunningBalance": "balance", } self.transaction_type_map = { - "INTADJUST": 'income', - "TRANSFER": 'transfer', - "ACH": 'transfer' + "INTADJUST": "income", + "TRANSFER": "transfer", + "ACH": "transfer", } - self.skip_transaction_types = ['Journal'] + # fmt: on + self.skip_transaction_types = ["Journal"] def deep_identify(self, file): - last_three = self.config.get('account_number', '')[-3:] - return self.column_labels_line in file.head() and f'XX{last_three}' in file.name + last_three = self.config.get("account_number", "")[-3:] + return self.column_labels_line in file.head() and f"XX{last_three}" in file.name def prepare_table(self, rdr): - rdr = rdr.addfield('amount', - lambda x: "-" + x['Withdrawal'] if x['Withdrawal'] != '' else x['Deposit']) - rdr = rdr.addfield('memo', lambda x: '') + rdr = rdr.addfield( + "amount", + lambda x: "-" + x["Withdrawal"] if x["Withdrawal"] != "" else x["Deposit"], + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def get_balance_statement(self, file=None): @@ -45,4 +49,6 @@ def get_balance_statement(self, file=None): date = self.get_balance_assertion_date() if date: - yield banking.Balance(date, self.rdr.namedtuples()[0].balance, self.currency) + yield banking.Balance( + date, self.rdr.namedtuples()[0].balance, self.currency + ) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py index 5f452de..6dc55b7 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py @@ -1,11 +1,12 @@ -""" Schwab Credit Line (eg: Pledged Asset Line) .csv importer.""" +"""Schwab Credit Line (eg: Pledged Asset Line) .csv importer.""" from beancount_reds_importers.importers.schwab import schwab_csv_checking + class Importer(schwab_csv_checking.Importer): - IMPORTER_NAME = 'Schwab Line of Credit CSV' + IMPORTER_NAME = "Schwab Line of Credit CSV" def custom_init(self): super().custom_init() - self.filename_pattern_def = '.*_Transactions_' + self.filename_pattern_def = ".*_Transactions_" self.column_labels_line = '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py index 8f3bd1a..4cb0980 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py @@ -1,4 +1,4 @@ -""" Schwab CSV Positions importer. +"""Schwab CSV Positions importer. Note: Schwab "Positions" CSV is not the same as Schwab "Balances" CSV.""" @@ -14,30 +14,33 @@ class Importer(investments.Importer, csvreader.Importer): def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*-Positions-' + self.filename_pattern_def = ".*-Positions-" self.header_identifier = '["]+Positions for account' self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%Y/%m/%d' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%Y/%m/%d" + self.funds_db_txt = "funds_by_ticker" self.column_labels_line = '"Symbol","Description","Quantity","Price","Price Change %","Price Change $","Market Value","Day Change %","Day Change $","Cost Basis","Gain/Loss %","Gain/Loss $","Ratings","Reinvest Dividends?","Capital Gains?","% Of Account","Security Type"' # noqa: #501 + # fmt: off self.header_map = { - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - } + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + } + # fmt: on self.skip_transaction_types = [] def convert_columns(self, rdr): # fixup decimals - decimals = ['units'] + decimals = ["units"] for i in decimals: rdr = rdr.convert(i, D) # fixup currencies def remove_non_numeric(x): - return re.sub(r'[^0-9\.]', "", x) # noqa: W605 - currencies = ['unit_price'] + return re.sub(r"[^0-9\.]", "", x) # noqa: W605 + + currencies = ["unit_price"] for i in currencies: rdr = rdr.convert(i, remove_non_numeric) rdr = rdr.convert(i, D) @@ -53,7 +56,7 @@ def get_max_transaction_date(self): def prepare_raw_file(self, rdr): # first row has date - d = rdr[0][0].rsplit(' ', 1)[1] + d = rdr[0][0].rsplit(" ", 1)[1] self.date = datetime.datetime.strptime(d, self.date_format) return rdr @@ -68,5 +71,5 @@ def prepare_table(self, rdr): def get_balance_positions(self): for pos in self.rdr.namedtuples(): - if pos.memo != '--': + if pos.memo != "--": yield pos diff --git a/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py index 3c50ad1..4ad4504 100644 --- a/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_json_brokerage.py @@ -1,18 +1,18 @@ -""" Schwab Brokerage .csv importer.""" +"""Schwab Brokerage .csv importer.""" from beancount_reds_importers.libreader import jsonreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(jsonreader.Importer, investments.Importer): - IMPORTER_NAME = 'Schwab Brokerage JSON' + IMPORTER_NAME = "Schwab Brokerage JSON" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Transactions_' + self.filename_pattern_def = ".*_Transactions_" self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" def skip_transaction(self, ot): - return ot.type in ['', 'Journal', 'Journaled Shares'] + return ot.type in ["", "Journal", "Journaled Shares"] diff --git a/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py b/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py index fda1141..03df212 100644 --- a/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py +++ b/beancount_reds_importers/importers/schwab/schwab_ofx_bank_ofx.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Schwab Bank' + IMPORTER_NAME = "Schwab Bank" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*Checking_Transations' + self.filename_pattern_def = ".*Checking_Transations" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py index 3e28c73..af7aeb0 100644 --- a/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_ofx_brokerage.py @@ -1,13 +1,13 @@ -""" Schwab Brokerage ofx importer.""" +"""Schwab Brokerage ofx importer.""" from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Schwab Brokerage' + IMPORTER_NAME = "Schwab Brokerage" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*schwab' + self.filename_pattern_def = ".*schwab" self.get_ticker_info = self.get_ticker_info_from_id diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py index 546fca6..fa46fea 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py @@ -6,50 +6,46 @@ fund_data = [ - ('SWVXX', '123', 'SCHWAB VALUE ADVANTAGE MONEY INV'), - ('GIS', '456', 'GENERAL MILLS INC'), - ('BND', '789', 'Vanguard Total Bond Market Index Fund'), + ("SWVXX", "123", "SCHWAB VALUE ADVANTAGE MONEY INV"), + ("GIS", "456", "GENERAL MILLS INC"), + ("BND", "789", "Vanguard Total Bond Market Index Fund"), ] # list of money_market accounts. These will not be held at cost, and instead will use price conversions -money_market = ['VMFXX'] +money_market = ["VMFXX"] fund_info = { - 'fund_data': fund_data, - 'money_market': money_market, - } + "fund_data": fund_data, + "money_market": money_market, +} def build_config(): acct = "Assets:Investments:Schwab" - root = 'Investments' - taxability = 'Taxable' - leaf = 'Schwab' - currency = 'USD' + root = "Investments" + taxability = "Taxable" + leaf = "Schwab" + currency = "USD" config = { - 'account_number' : '9876', - 'main_account' : acct + ':{ticker}', - 'cash_account' : f'{acct}:{{currency}}', - 'transfer' : 'Assets:Zero-Sum-Accounts:Transfers:Bank-Account', - 'dividends' : f'Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}', - 'interest' : f'Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}', - 'cg' : f'Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}', - 'capgainsd_lt' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}', - 'capgainsd_st' : f'Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}', - 'fees' : f'Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}', - 'invexpense' : f'Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}', - 'rounding_error' : 'Equity:Rounding-Errors:Imports', - 'fund_info' : fund_info, - 'currency' : currency, + "account_number": "9876", + "main_account": acct + ":{ticker}", + "cash_account": f"{acct}:{{currency}}", + "transfer": "Assets:Zero-Sum-Accounts:Transfers:Bank-Account", + "dividends": f"Income:{root}:{taxability}:Dividends:{leaf}:{{ticker}}", + "interest": f"Income:{root}:{taxability}:Interest:{leaf}:{{ticker}}", + "cg": f"Income:{root}:{taxability}:Capital-Gains:{leaf}:{{ticker}}", + "capgainsd_lt": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Long:{leaf}:{{ticker}}", + "capgainsd_st": f"Income:{root}:{taxability}:Capital-Gains-Distributions:Short:{leaf}:{{ticker}}", + "fees": f"Expenses:Fees-and-Charges:Brokerage-Fees:{taxability}:{leaf}", + "invexpense": f"Expenses:Expenses:Investment-Expenses:{taxability}:{leaf}", + "rounding_error": "Equity:Rounding-Errors:Imports", + "fund_info": fund_info, + "currency": currency, } return config -@regtest.with_importer( - schwab_csv_brokerage.Importer( - build_config() - ) -) +@regtest.with_importer(schwab_csv_brokerage.Importer(build_config())) @regtest.with_testdir(path.dirname(__file__)) class TestSchwabBrokerage(regtest.ImporterTestBase): pass diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py index b44919e..ff7813c 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py @@ -4,12 +4,13 @@ from beancount.ingest import regression_pytest as regtest from beancount_reds_importers.importers.schwab import schwab_csv_checking + @regtest.with_importer( schwab_csv_checking.Importer( { - 'account_number' : '1234', - 'main_account' : 'Assets:Banks:Schwab', - 'currency' : 'USD', + "account_number": "1234", + "main_account": "Assets:Banks:Schwab", + "currency": "USD", } ) ) diff --git a/beancount_reds_importers/importers/stanchart/scbbank.py b/beancount_reds_importers/importers/stanchart/scbbank.py index 6a73fb7..7c0e864 100644 --- a/beancount_reds_importers/importers/stanchart/scbbank.py +++ b/beancount_reds_importers/importers/stanchart/scbbank.py @@ -7,44 +7,54 @@ class Importer(csvreader.Importer, banking.Importer): - IMPORTER_NAME = 'SCB Banking Account CSV' + IMPORTER_NAME = "SCB Banking Account CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'AccountTransactions[0-9]*' - self.header_identifier = self.config.get('custom_header', 'Account transactions shown:') - self.column_labels_line = 'Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance' - self.balance_column_labels_line = 'Account Name,Account Number,Currency,Current Balance,Available Balance' - self.date_format = '%d/%m/%Y' + self.filename_pattern_def = "AccountTransactions[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "Account transactions shown:" + ) + self.column_labels_line = "Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance" + self.balance_column_labels_line = ( + "Account Name,Account Number,Currency,Current Balance,Available Balance" + ) + self.date_format = "%d/%m/%Y" self.skip_tail_rows = 0 - self.skip_comments = '# ' + self.skip_comments = "# " + # fmt: off self.header_map = { - "Date": "date", - "Transaction": "payee", - "Currency": "currency", - "Withdrawal": "withdrawal", - "Deposit": "deposit", - "Running Balance": "balance_running", - "SGD Equivalent Balance": "balance", + "Date": "date", + "Transaction": "payee", + "Currency": "currency", + "Withdrawal": "withdrawal", + "Deposit": "deposit", + "Running Balance": "balance_running", + "SGD Equivalent Balance": "balance", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): - rdr = rdr.addfield('amount', - lambda x: "-" + x['Withdrawal'] if x['Withdrawal'] != '' else x['Deposit']) - rdr = rdr.addfield('memo', lambda x: '') + rdr = rdr.addfield( + "amount", + lambda x: "-" + x["Withdrawal"] if x["Withdrawal"] != "" else x["Deposit"], + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) return rdr @@ -54,18 +64,18 @@ def get_balance_statement(self, file=None): if date: rdr = self.read_raw(file) rdr = self.prepare_raw_file(rdr) - col_labels = self.balance_column_labels_line.split(',') + col_labels = self.balance_column_labels_line.split(",") rdr = self.extract_table_with_header(rdr, col_labels) - header_map = {k: k.replace(' ', '_') for k in col_labels} + header_map = {k: k.replace(" ", "_") for k in col_labels} rdr = rdr.rename(header_map) - while '' in rdr.header(): - rdr = rdr.cutout('') + while "" in rdr.header(): + rdr = rdr.cutout("") row = rdr.namedtuples()[0] amount = row.Current_Balance units, debitcredit = amount.split() - if debitcredit != 'CR': - units = '-' + units + if debitcredit != "CR": + units = "-" + units yield banking.Balance(date, D(units), row.Currency) diff --git a/beancount_reds_importers/importers/stanchart/scbcard.py b/beancount_reds_importers/importers/stanchart/scbcard.py index b4ab04d..eb0e053 100644 --- a/beancount_reds_importers/importers/stanchart/scbcard.py +++ b/beancount_reds_importers/importers/stanchart/scbcard.py @@ -7,71 +7,87 @@ class Importer(csvreader.Importer, banking.Importer): - IMPORTER_NAME = 'SCB Card CSV' + IMPORTER_NAME = "SCB Card CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'CardTransactions[0-9]*' - self.header_identifier = self.config.get('custom_header', 'PRIORITY BANKING VISA INFINITE CARD') - self.column_labels_line = 'Date,DESCRIPTION,Foreign Currency Amount,SGD Amount' - self.date_format = '%d/%m/%Y' + self.filename_pattern_def = "CardTransactions[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "PRIORITY BANKING VISA INFINITE CARD" + ) + self.column_labels_line = "Date,DESCRIPTION,Foreign Currency Amount,SGD Amount" + self.date_format = "%d/%m/%Y" self.skip_tail_rows = 6 - self.skip_comments = '# ' + self.skip_comments = "# " + # fmt: off self.header_map = { - "Date": "date", - "DESCRIPTION": "payee", + "Date": "date", + "DESCRIPTION": "payee", } + # fmt: on self.transaction_type_map = {} def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) def skip_transaction(self, row): - return '[UNPOSTED]' in row.payee + return "[UNPOSTED]" in row.payee def prepare_table(self, rdr): - rdr = rdr.select(lambda r: 'UNPOSTED' not in r['DESCRIPTION']) + rdr = rdr.select(lambda r: "UNPOSTED" not in r["DESCRIPTION"]) # parse foreign_currency amount: "YEN 74,000" - if self.config.get('convert_currencies', False): + if self.config.get("convert_currencies", False): # Currency conversions won't work as expected since Beancount v2 # doesn't support adding @@ (total price conversions) via code. # See https://groups.google.com/g/beancount/c/nMvuoR4yOmM # This means the '@' generated by this code below needs to be replaced with an '@@' - rdr = rdr.capture('Foreign Currency Amount', '(.*) (.*)', - ['foreign_currency', 'foreign_amount'], - fill=' ', include_original=True) - rdr = rdr.cutout('Foreign Currency Amount') + rdr = rdr.capture( + "Foreign Currency Amount", + "(.*) (.*)", + ["foreign_currency", "foreign_amount"], + fill=" ", + include_original=True, + ) + rdr = rdr.cutout("Foreign Currency Amount") # parse SGD Amount: "SGD 141.02 CR" into a single amount column - rdr = rdr.capture('SGD Amount', '(.*) (.*) (.*)', ['currency', 'amount', 'crdr']) + rdr = rdr.capture( + "SGD Amount", "(.*) (.*) (.*)", ["currency", "amount", "crdr"] + ) # change DR into -ve. TODO: move this into csvreader or csvreader.utils - crdrdict = {'DR': '-', 'CR': ''} - rdr = rdr.convert('amount', lambda i, row: crdrdict[row.crdr] + i, pass_row=True) - - rdr = rdr.addfield('memo', lambda x: '') # TODO: make this non-mandatory in csvreader + crdrdict = {"DR": "-", "CR": ""} + rdr = rdr.convert( + "amount", lambda i, row: crdrdict[row.crdr] + i, pass_row=True + ) + + rdr = rdr.addfield( + "memo", lambda x: "" + ) # TODO: make this non-mandatory in csvreader return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" date = self.get_balance_assertion_date() if date: - balance_row = self.get_row_by_label(file, 'Current Balance') + balance_row = self.get_row_by_label(file, "Current Balance") currency, amount = balance_row[1], balance_row[2] units, debitcredit = amount.split() - if debitcredit != 'CR': - units = '-' + units + if debitcredit != "CR": + units = "-" + units yield banking.Balance(date, D(units), currency) diff --git a/beancount_reds_importers/importers/target/__init__.py b/beancount_reds_importers/importers/target/__init__.py index 1a83926..69b6cc7 100644 --- a/beancount_reds_importers/importers/target/__init__.py +++ b/beancount_reds_importers/importers/target/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Target Credit Card' + IMPORTER_NAME = "Target Credit Card" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = 'Transactions' + self.filename_pattern_def = "Transactions" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/tdameritrade/__init__.py b/beancount_reds_importers/importers/tdameritrade/__init__.py index fd19326..24e4ace 100644 --- a/beancount_reds_importers/importers/tdameritrade/__init__.py +++ b/beancount_reds_importers/importers/tdameritrade/__init__.py @@ -1,17 +1,16 @@ - from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'TDAmeritrade' + IMPORTER_NAME = "TDAmeritrade" def custom_init(self): super(Importer, self).custom_init() self.max_rounding_error = 0.07 - self.filename_pattern_def = '.*tdameritrade' + self.filename_pattern_def = ".*tdameritrade" self.get_ticker_info = self.get_ticker_info_from_id def get_ticker_info(self, security): - ticker = self.config['fund_info']['cusip_map'][security] - return ticker, '' + ticker = self.config["fund_info"]["cusip_map"][security] + return ticker, "" diff --git a/beancount_reds_importers/importers/techcubank/__init__.py b/beancount_reds_importers/importers/techcubank/__init__.py index 725ca1b..a70a185 100644 --- a/beancount_reds_importers/importers/techcubank/__init__.py +++ b/beancount_reds_importers/importers/techcubank/__init__.py @@ -5,10 +5,10 @@ class Importer(banking.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Tech Credit Union' + IMPORTER_NAME = "Tech Credit Union" def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*Accounts' + self.filename_pattern_def = ".*Accounts" self.custom_init_run = True diff --git a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py index f1d115b..396e36e 100644 --- a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py +++ b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py @@ -4,12 +4,14 @@ @regtest.with_importer( - uobbank.Importer({ - 'main_account': 'Assets:Banks:UOB:UNIPLUS', - 'account_number': '1234567890', - 'currency': 'SGD', - 'rounding_error': 'Equity:Rounding-Errors:Imports', - }) + uobbank.Importer( + { + "main_account": "Assets:Banks:UOB:UNIPLUS", + "account_number": "1234567890", + "currency": "SGD", + "rounding_error": "Equity:Rounding-Errors:Imports", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestUOB(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/importers/unitedoverseas/uobbank.py b/beancount_reds_importers/importers/unitedoverseas/uobbank.py index b59fa09..cd9bb1c 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobbank.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobbank.py @@ -11,41 +11,53 @@ class Importer(xlsreader.Importer, banking.Importer): def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'ACC_TXN_History[0-9]*' - self.header_identifier = self.config.get('custom_header', 'United Overseas Bank Limited.*Account Type:Uniplus Account') - self.column_labels_line = 'Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance' - self.date_format = '%d %b %Y' + self.filename_pattern_def = "ACC_TXN_History[0-9]*" + self.header_identifier = self.config.get( + "custom_header", + "United Overseas Bank Limited.*Account Type:Uniplus Account", + ) + self.column_labels_line = "Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance" + self.date_format = "%d %b %Y" + # fmt: off self.header_map = { - 'Transaction Date': 'date', - 'Transaction Description': 'payee', - 'Available Balance': 'balance' + "Transaction Date": "date", + "Transaction Description": "payee", + "Available Balance": "balance", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) # TODO: move these into utils, since this is probably a common operation def prepare_table(self, rdr): # Remove carriage returns in description - rdr = rdr.convert('Transaction Description', lambda x: x.replace('\n', ' ')) + rdr = rdr.convert("Transaction Description", lambda x: x.replace("\n", " ")) def Ds(x): return D(str(x)) - rdr = rdr.addfield('amount', - lambda x: -1 * Ds(x['Withdrawal']) if x['Withdrawal'] != 0 else Ds(x['Deposit'])) - rdr = rdr.addfield('memo', lambda x: '') + + rdr = rdr.addfield( + "amount", + lambda x: -1 * Ds(x["Withdrawal"]) + if x["Withdrawal"] != 0 + else Ds(x["Deposit"]), + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr @@ -55,6 +67,6 @@ def get_balance_statement(self, file=None): if date: row = self.rdr.namedtuples()[0] # Get currency from input file - currency = self.get_row_by_label(file, 'Account Number:')[2] + currency = self.get_row_by_label(file, "Account Number:")[2] yield banking.Balance(date, D(str(row.balance)), currency) diff --git a/beancount_reds_importers/importers/unitedoverseas/uobcard.py b/beancount_reds_importers/importers/unitedoverseas/uobcard.py index 2dfc6b7..a195ed3 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobcard.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobcard.py @@ -7,62 +7,68 @@ class Importer(xlsreader.Importer, banking.Importer): - IMPORTER_NAME = 'SCB Card CSV' + IMPORTER_NAME = "SCB Card CSV" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '^CC_TXN_History[0-9]*' - self.header_identifier = self.config.get('custom_header', 'United Overseas Bank Limited.*Account Type:VISA SIGNATURE') - self.column_labels_line = 'Transaction Date,Posting Date,Description,Foreign Currency Type,Transaction Amount(Foreign),Local Currency Type,Transaction Amount(Local)' # noqa: E501 - self.date_format = '%d %b %Y' + self.filename_pattern_def = "^CC_TXN_History[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "United Overseas Bank Limited.*Account Type:VISA SIGNATURE" + ) + self.column_labels_line = "Transaction Date,Posting Date,Description,Foreign Currency Type,Transaction Amount(Foreign),Local Currency Type,Transaction Amount(Local)" # noqa: E501 + self.date_format = "%d %b %Y" # Remove _DISABLED below to include currency conversions. This won't work as expected since # Beancount v2 doesn't support adding @@ (total price conversions) via code. See # https://groups.google.com/g/beancount/c/nMvuoR4yOmM This means the '@' generated by this # code below needs to be replaced with an '@@' - foreign_currency = 'foreign_currency_DISABLED' - foreign_amount = 'foreign_amount_DISABLED' - if self.config.get('convert_currencies', False): - foreign_currency = 'foreign_currency' - foreign_amount = 'foreign_amount' + foreign_currency = "foreign_currency_DISABLED" + foreign_amount = "foreign_amount_DISABLED" + if self.config.get("convert_currencies", False): + foreign_currency = "foreign_currency" + foreign_amount = "foreign_amount" + # fmt: off self.header_map = { - 'Transaction Date': 'date', - 'Posting Date': 'date_posting', - 'Description': 'payee', - 'Foreign Currency Type': foreign_currency, - 'Transaction Amount(Foreign)': foreign_amount, - 'Local Currency Type': 'currency', - 'Transaction Amount(Local)': 'amount' + "Transaction Date": "date", + "Posting Date": "date_posting", + "Description": "payee", + "Foreign Currency Type": foreign_currency, + "Transaction Amount(Foreign)": foreign_amount, + "Local Currency Type": "currency", + "Transaction Amount(Local)": "amount", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): # Remove carriage returns in description - rdr = rdr.convert('Description', lambda x: x.replace('\n', ' ')) - rdr = rdr.addfield('memo', lambda x: '') + rdr = rdr.convert("Description", lambda x: x.replace("\n", " ")) + rdr = rdr.addfield("memo", lambda x: "") # delete empty rows - rdr = rdr.select(lambda x: x['Transaction Date'] != '') + rdr = rdr.select(lambda x: x["Transaction Date"] != "") return rdr def prepare_processed_table(self, rdr): - return rdr.convert('amount', lambda x: -1 * D(str(x))) + return rdr.convert("amount", lambda x: -1 * D(str(x))) def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr @@ -70,6 +76,6 @@ def get_balance_statement(self, file=None): """Return the balance on the first and last dates""" date = self.get_balance_assertion_date() if date: - balance_row = self.get_row_by_label(file, 'Statement Balance:') + balance_row = self.get_row_by_label(file, "Statement Balance:") units, currency = balance_row[1], balance_row[2] yield banking.Balance(date, -1 * D(str(units)), currency) diff --git a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py index 7a689dc..9793ae4 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py @@ -7,42 +7,55 @@ class Importer(xlsreader.Importer, banking.Importer): - IMPORTER_NAME = 'UOB SRS' + IMPORTER_NAME = "UOB SRS" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'SRS_TXN_History[0-9]*' - self.header_identifier = self.config.get('custom_header', 'United Overseas Bank Limited.*Account Type:SRS Account') - self.column_labels_line = 'Transaction Date,Transaction Description,Withdrawal,Deposit' - self.date_format = '%Y%m%d' + self.filename_pattern_def = "SRS_TXN_History[0-9]*" + self.header_identifier = self.config.get( + "custom_header", "United Overseas Bank Limited.*Account Type:SRS Account" + ) + self.column_labels_line = ( + "Transaction Date,Transaction Description,Withdrawal,Deposit" + ) + self.date_format = "%Y%m%d" + # fmt: off self.header_map = { - 'Transaction Date': 'date', - 'Transaction Description': 'payee', + "Transaction Date": "date", + "Transaction Description": "payee", } + # fmt: on self.transaction_type_map = {} self.skip_transaction_types = [] def deep_identify(self, file): - account_number = self.config.get('account_number', '') - return re.match(self.header_identifier, file.head()) and \ - account_number in file.head() + account_number = self.config.get("account_number", "") + return ( + re.match(self.header_identifier, file.head()) + and account_number in file.head() + ) def prepare_table(self, rdr): # Remove carriage returns in description - rdr = rdr.convert('Transaction Description', lambda x: x.replace('\n', ' ')) + rdr = rdr.convert("Transaction Description", lambda x: x.replace("\n", " ")) def Ds(x): return D(str(x)) - rdr = rdr.addfield('amount', - lambda x: -1 * Ds(x['Withdrawal']) if x['Withdrawal'] != '' else Ds(x['Deposit'])) - rdr = rdr.addfield('memo', lambda x: '') + + rdr = rdr.addfield( + "amount", + lambda x: -1 * Ds(x["Withdrawal"]) + if x["Withdrawal"] != "" + else Ds(x["Deposit"]), + ) + rdr = rdr.addfield("memo", lambda x: "") return rdr def prepare_raw_file(self, rdr): # Strip tabs and spaces around each field in the entire file - rdr = rdr.convertall(lambda x: x.strip(' \t') if isinstance(x, str) else x) + rdr = rdr.convertall(lambda x: x.strip(" \t") if isinstance(x, str) else x) # Delete empty rows - rdr = rdr.select(lambda x: any([i != '' for i in x])) + rdr = rdr.select(lambda x: any([i != "" for i in x])) return rdr diff --git a/beancount_reds_importers/importers/vanguard/__init__.py b/beancount_reds_importers/importers/vanguard/__init__.py index dc2d4f2..170e014 100644 --- a/beancount_reds_importers/importers/vanguard/__init__.py +++ b/beancount_reds_importers/importers/vanguard/__init__.py @@ -1,4 +1,4 @@ -""" Vanguard Brokerage ofx importer.""" +"""Vanguard Brokerage ofx importer.""" import ntpath from beancount_reds_importers.libreader import ofxreader @@ -6,7 +6,7 @@ class Importer(investments.Importer, ofxreader.Importer): - IMPORTER_NAME = 'Vanguard' + IMPORTER_NAME = "Vanguard" # Any memo in the source OFX that's in this set is not carried forward. # Vanguard sets memos that aren't very useful and would create noise in the @@ -17,7 +17,7 @@ class Importer(investments.Importer, ofxreader.Importer): def custom_init(self): self.max_rounding_error = 0.11 - self.filename_pattern_def = '.*OfxDownload' + self.filename_pattern_def = ".*OfxDownload" self.get_ticker_info = self.get_ticker_info_from_id self.get_payee = self.cleanup_memo @@ -31,21 +31,21 @@ def custom_init(self): self.price_cost_both_zero_handler = lambda *args: None def file_name(self, file): - return 'vanguard-all-{}'.format(ntpath.basename(file.name)) + return "vanguard-all-{}".format(ntpath.basename(file.name)) def get_target_acct_custom(self, transaction, ticker=None): - if 'LT CAP GAIN' in transaction.memo: - return self.config['capgainsd_lt'] - elif 'ST CAP GAIN' in transaction.memo: - return self.config['capgainsd_st'] + if "LT CAP GAIN" in transaction.memo: + return self.config["capgainsd_lt"] + elif "ST CAP GAIN" in transaction.memo: + return self.config["capgainsd_st"] return None def cleanup_memo(self, ot): # some vanguard files have memos repeated like this: # 'DIVIDEND REINVESTMENTDIVIDEND REINVESTMENT' retval = ot.memo - if ot.memo[:int(len(ot.memo)/2)] == ot.memo[int(len(ot.memo)/2):]: - retval = ot.memo[:int(len(ot.memo)/2)] + if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2) :]: + retval = ot.memo[: int(len(ot.memo) / 2)] return retval # For users to comment out in their local file if they so prefer diff --git a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py index ecb8b81..02ab167 100644 --- a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py +++ b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py @@ -1,4 +1,4 @@ -""" Vanguard screenscrape importer. Unsettled trades are not available in Vanguard's qfx and need to be +"""Vanguard screenscrape importer. Unsettled trades are not available in Vanguard's qfx and need to be screenscrapped into a tsv""" from beancount_reds_importers.libreader import tsvreader @@ -6,54 +6,76 @@ class Importer(investments.Importer, tsvreader.Importer): - IMPORTER_NAME = 'Vanguard screenscrape tsv' + IMPORTER_NAME = "Vanguard screenscrape tsv" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*vanguardss.*' - self.header_identifier = '' + self.filename_pattern_def = ".*vanguardss.*" + self.header_identifier = "" self.get_ticker_info = self.get_ticker_info_from_id - self.date_format = '%m/%d/%Y' - self.funds_db_txt = 'funds_by_ticker' + self.date_format = "%m/%d/%Y" + self.funds_db_txt = "funds_by_ticker" + # fmt: off self.header_map = { - "date": 'date', - "settledate": 'tradeDate', - "symbol": 'security', - "description": 'memo', - "action": 'type', - "quantity": 'units', - "price": 'unit_price', - "fees": 'fees', - "amount": 'amount', - "total": 'total', - } + "date": "date", + "settledate": "tradeDate", + "symbol": "security", + "description": "memo", + "action": "type", + "quantity": "units", + "price": "unit_price", + "fees": "fees", + "amount": "amount", + "total": "total", + } self.transaction_type_map = { - 'Buy': 'buystock', - 'Sell': 'sellstock', - } - self.skip_transaction_types = [''] + "Buy": "buystock", + "Sell": "sellstock", + } + # fmt: on + self.skip_transaction_types = [""] def prepare_table(self, rdr): def extract_numbers(x): - replacements = {'– ': '-', - '$': '', - ',': '', - 'Free': '0', - } + replacements = { + "– ": "-", + "$": "", + ",": "", + "Free": "0", + } for k, v in replacements.items(): x = x.replace(k, v) return x - header = ('date', 'settledate', 'symbol', 'description', 'quantity', 'price', 'fees', 'amount') + header = ( + "date", + "settledate", + "symbol", + "description", + "quantity", + "price", + "fees", + "amount", + ) rdr = rdr.pushheader(header) - rdr = rdr.addfield('action', lambda x: x['description'].rsplit(' ', 2)[1].strip()) + rdr = rdr.addfield( + "action", lambda x: x["description"].rsplit(" ", 2)[1].strip() + ) - for field in ["date", "settledate", "symbol", "description", "quantity", "price", "fees"]: + for field in [ + "date", + "settledate", + "symbol", + "description", + "quantity", + "price", + "fees", + ]: rdr = rdr.convert(field, lambda x: x.strip()) for field in ["quantity", "amount", "price", "fees"]: rdr = rdr.convert(field, extract_numbers) - rdr = rdr.addfield('total', lambda x: x['amount']) + rdr = rdr.addfield("total", lambda x: x["amount"]) return rdr diff --git a/beancount_reds_importers/importers/workday/__init__.py b/beancount_reds_importers/importers/workday/__init__.py index eb0e55d..a86ce64 100644 --- a/beancount_reds_importers/importers/workday/__init__.py +++ b/beancount_reds_importers/importers/workday/__init__.py @@ -1,4 +1,4 @@ -""" Workday paycheck importer.""" +"""Workday paycheck importer.""" import datetime from beancount_reds_importers.libreader import xlsx_multitable_reader @@ -15,33 +15,35 @@ class Importer(paycheck.Importer, xlsx_multitable_reader.Importer): - IMPORTER_NAME = 'Workday Paycheck' + IMPORTER_NAME = "Workday Paycheck" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*_Complete' - self.header_identifier = '- Complete' + self.config.get('custom_header', '') - self.date_format = '%m/%d/%Y' + self.filename_pattern_def = ".*_Complete" + self.header_identifier = "- Complete" + self.config.get("custom_header", "") + self.date_format = "%m/%d/%Y" self.skip_head_rows = 1 # TODO: need to be smarter about this, and skip only when needed self.skip_tail_rows = 0 - self.funds_db_txt = 'funds_by_ticker' + self.funds_db_txt = "funds_by_ticker" + # fmt: off self.header_map = { - "Description": 'memo', - "Symbol": 'security', - "Quantity": 'units', - "Price": 'unit_price', - } + "Description": "memo", + "Symbol": "security", + "Quantity": "units", + "Price": "unit_price", + } + # fmt: on def paycheck_date(self, input_file): self.read_file(input_file) - d = self.alltables['Payslip Information'].namedtuples()[0].check_date + d = self.alltables["Payslip Information"].namedtuples()[0].check_date self.date = datetime.datetime.strptime(d, self.date_format) return self.date.date() def prepare_tables(self): def valid_header_label(label): - return label.lower().replace(' ', '_') + return label.lower().replace(" ", "_") for section, table in self.alltables.items(): for header in table.header(): @@ -49,4 +51,4 @@ def valid_header_label(label): self.alltables[section] = table def build_metadata(self, file, metatype=None, data={}): - return {'filing_account': self.config['main_account']} + return {"filing_account": self.config["main_account"]} diff --git a/beancount_reds_importers/libreader/csv_multitable_reader.py b/beancount_reds_importers/libreader/csv_multitable_reader.py index 8a8f4ef..94a0433 100644 --- a/beancount_reds_importers/libreader/csv_multitable_reader.py +++ b/beancount_reds_importers/libreader/csv_multitable_reader.py @@ -59,27 +59,35 @@ def read_file(self, file): self.raw_rdr = rdr = self.read_raw(file) - rdr = rdr.skip(getattr(self, 'skip_head_rows', 0)) # chop unwanted file header rows - rdr = rdr.head(len(rdr) - getattr(self, 'skip_tail_rows', 0) - 1) # chop unwanted file footer rows + rdr = rdr.skip( + getattr(self, "skip_head_rows", 0) + ) # chop unwanted file header rows + rdr = rdr.head( + len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 + ) # chop unwanted file footer rows # [0, 2, 10] <-- starts # [-1, 1, 9] <-- ends - table_starts = [i for (i, row) in enumerate(rdr) if self.is_section_title(row)] + [len(rdr)] - table_ends = [r-1 for r in table_starts][1:] + table_starts = [ + i for (i, row) in enumerate(rdr) if self.is_section_title(row) + ] + [len(rdr)] + table_ends = [r - 1 for r in table_starts][1:] table_indexes = zip(table_starts, table_ends) # build the dictionary of tables self.alltables = {} - for (s, e) in table_indexes: + for s, e in table_indexes: if s == e: continue - table = rdr.skip(s+1) # skip past start index and header row - table = table.head(e-s-1) # chop lines after table section data + table = rdr.skip(s + 1) # skip past start index and header row + table = table.head(e - s - 1) # chop lines after table section data self.alltables[rdr[s][0]] = table for section, table in self.alltables.items(): table = table.rowlenselect(0, complement=True) # clean up empty rows - table = table.cut(*[h for h in table.header() if h]) # clean up empty columns + table = table.cut( + *[h for h in table.header() if h] + ) # clean up empty columns self.alltables[section] = table self.prepare_tables() # to be overridden by importer diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 84a3719..69d8cdb 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -59,10 +59,10 @@ class Importer(reader.Reader, importer.ImporterProtocol): - FILE_EXTS = ['csv'] + FILE_EXTS = ["csv"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.reader_ready = self.deep_identify(file) if self.reader_ready: @@ -96,19 +96,26 @@ def prepare_processed_table(self, rdr): def convert_columns(self, rdr): # convert data in transaction types column - if 'type' in rdr.header(): - rdr = rdr.convert('type', self.transaction_type_map) + if "type" in rdr.header(): + rdr = rdr.convert("type", self.transaction_type_map) # fixup decimals - decimals = ['units'] + decimals = ["units"] for i in decimals: if i in rdr.header(): rdr = rdr.convert(i, D) # fixup currencies def remove_non_numeric(x): - return re.sub(r'[^0-9\.-]', "", str(x).strip()) # noqa: W605 - currencies = getattr(self, 'currency_fields', []) + ['unit_price', 'fees', 'total', 'amount', 'balance'] + return re.sub(r"[^0-9\.-]", "", str(x).strip()) # noqa: W605 + + currencies = getattr(self, "currency_fields", []) + [ + "unit_price", + "fees", + "total", + "amount", + "balance", + ] for i in currencies: if i in rdr.header(): rdr = rdr.convert(i, remove_non_numeric) @@ -117,7 +124,8 @@ def remove_non_numeric(x): # fixup dates def convert_date(d): return datetime.datetime.strptime(d, self.date_format) - dates = getattr(self, 'date_fields', []) + ['date', 'tradeDate', 'settleDate'] + + dates = getattr(self, "date_fields", []) + ["date", "tradeDate", "settleDate"] for i in dates: if i in rdr.header(): rdr = rdr.convert(i, convert_date) @@ -131,8 +139,8 @@ def skip_until_main_table(self, rdr, col_labels=None): """Skip csv lines until the header line is found.""" # TODO: convert this into an 'extract_table()' method that handles the tail as well if not col_labels: - if hasattr(self, 'column_labels_line'): - col_labels = self.column_labels_line.replace('"', '').split(',') + if hasattr(self, "column_labels_line"): + col_labels = self.column_labels_line.replace('"', "").split(",") else: return rdr skip = None @@ -151,8 +159,8 @@ def skip_until_main_table(self, rdr, col_labels=None): def extract_table_with_header(self, rdr, col_labels=None): rdr = self.skip_until_main_table(rdr, col_labels) nrows = len(rdr) - for (n, r) in enumerate(rdr): - if not r or all(i == '' for i in r): + for n, r in enumerate(rdr): + if not r or all(i == "" for i in r): # blank line, terminate nrows = n - 1 break @@ -170,18 +178,22 @@ def skip_until_row_contains(self, rdr, value): return rdr.rowslice(start, len(rdr)) def read_file(self, file): - if not getattr(self, 'file_read_done', False): + if not getattr(self, "file_read_done", False): # read file rdr = self.read_raw(file) rdr = self.prepare_raw_file(rdr) # extract main table - rdr = rdr.skip(getattr(self, 'skip_head_rows', 0)) # chop unwanted header rows - rdr = rdr.head(len(rdr) - getattr(self, 'skip_tail_rows', 0) - 1) # chop unwanted footer rows + rdr = rdr.skip( + getattr(self, "skip_head_rows", 0) + ) # chop unwanted header rows + rdr = rdr.head( + len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 + ) # chop unwanted footer rows rdr = self.extract_table_with_header(rdr) - if hasattr(self, 'skip_comments'): + if hasattr(self, "skip_comments"): rdr = rdr.skipcomments(self.skip_comments) - rdr = rdr.rowslice(getattr(self, 'skip_data_rows', 0), None) + rdr = rdr.rowslice(getattr(self, "skip_data_rows", 0), None) rdr = self.prepare_table(rdr) # process table @@ -201,7 +213,7 @@ def get_transactions(self): # TOOD: custom, overridable def skip_transaction(self, row): - return getattr(row, 'type', 'NO_TYPE') in self.skip_transaction_types + return getattr(row, "type", "NO_TYPE") in self.skip_transaction_types def get_balance_assertion_date(self): """ @@ -220,8 +232,10 @@ def get_max_transaction_date(self): # TODO: clean this up. this probably suffices: # return max(ot.date for ot in self.get_transactions()).date() - date = max(ot.tradeDate if hasattr(ot, 'tradeDate') else ot.date - for ot in self.get_transactions()).date() + date = max( + ot.tradeDate if hasattr(ot, "tradeDate") else ot.date + for ot in self.get_transactions() + ).date() except Exception as err: print("ERROR: no end_date. SKIPPING input.") traceback.print_tb(err.__traceback__) diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 5ae1e82..7536b46 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -1,4 +1,3 @@ - """JSON importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers. @@ -19,16 +18,18 @@ from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning import json + # import re import warnings + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) class Importer(reader.Reader, importer.ImporterProtocol): - FILE_EXTS = ['json'] + FILE_EXTS = ["json"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.reader_ready = self.deep_identify(file) if self.reader_ready: @@ -61,8 +62,6 @@ def read_file(self, file): # total = transaction['Amount'] # ) - - # def get_transactions(self): # Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', # 'units', 'fees', 'total']) diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index b40970e..f4590ac 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -8,14 +8,15 @@ from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning import warnings + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) class Importer(reader.Reader, importer.ImporterProtocol): - FILE_EXTS = ['ofx', 'qfx'] + FILE_EXTS = ["ofx", "qfx"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.ofx_account = None self.reader_ready = False @@ -26,9 +27,10 @@ def initialize_reader(self, file): for acc in self.ofx.accounts: # account identifying info fieldname varies across institutions # self.acc_num_field can be overridden in self.custom_init() if needed - acc_num_field = getattr(self, 'account_number_field', 'account_id') - if self.match_account_number(getattr(acc, acc_num_field), - self.config['account_number']): + acc_num_field = getattr(self, "account_number_field", "account_id") + if self.match_account_number( + getattr(acc, acc_num_field), self.config["account_number"] + ): self.ofx_account = acc self.reader_ready = True if self.reader_ready: @@ -41,7 +43,7 @@ def match_account_number(self, file_account, config_account): def file_date(self, file): """Get the ending date of the statement.""" - if not getattr(self, 'ofx_account', None): + if not getattr(self, "ofx_account", None): self.initialize(file) try: return self.ofx_account.statement.end_date @@ -56,27 +58,27 @@ def get_transactions(self): yield from self.ofx_account.statement.transactions def get_balance_statement(self, file=None): - if not hasattr(self.ofx_account.statement, 'balance'): + if not hasattr(self.ofx_account.statement, "balance"): return [] date = self.get_balance_assertion_date() if date: - Balance = namedtuple('Balance', ['date', 'amount']) + Balance = namedtuple("Balance", ["date", "amount"]) yield Balance(date, self.ofx_account.statement.balance) def get_balance_positions(self): - if not hasattr(self.ofx_account.statement, 'positions'): + if not hasattr(self.ofx_account.statement, "positions"): return [] yield from self.ofx_account.statement.positions def get_available_cash(self, settlement_fund_balance=0): - available_cash = getattr(self.ofx_account.statement, 'available_cash', None) + available_cash = getattr(self.ofx_account.statement, "available_cash", None) if available_cash is not None: # Some institutions compute available_cash this way. For others, override this method # in the importer return available_cash - settlement_fund_balance return None - def get_ofx_end_date(self, field='end_date'): + def get_ofx_end_date(self, field="end_date"): end_date = getattr(self.ofx_account.statement, field, None) if end_date: @@ -86,7 +88,7 @@ def get_ofx_end_date(self, field='end_date'): return None def get_smart_date(self): - """ We want the latest date we can assert balance on. Let's consider all the dates we have: + """We want the latest date we can assert balance on. Let's consider all the dates we have: b--------e-------(s-2)----(s)----(d) - b: date of first transaction in this ofx file (end_date) @@ -105,28 +107,41 @@ def get_smart_date(self): have. """ - ofx_max_transation_date = self.get_ofx_end_date('end_date') - ofx_balance_date1 = self.get_ofx_end_date('available_balance_date') - ofx_balance_date2 = self.get_ofx_end_date('balance_date') + ofx_max_transation_date = self.get_ofx_end_date("end_date") + ofx_balance_date1 = self.get_ofx_end_date("available_balance_date") + ofx_balance_date2 = self.get_ofx_end_date("balance_date") max_transaction_date = self.get_max_transaction_date() if ofx_balance_date1: - ofx_balance_date1 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) + ofx_balance_date1 -= datetime.timedelta( + days=self.config.get("balance_assertion_date_fudge", 2) + ) if ofx_balance_date2: - ofx_balance_date2 -= datetime.timedelta(days=self.config.get('balance_assertion_date_fudge', 2)) - - dates = [ofx_max_transation_date, max_transaction_date, ofx_balance_date1, ofx_balance_date2] - if all(v is None for v in dates[:2]): # because ofx_balance_date appears even for closed accounts + ofx_balance_date2 -= datetime.timedelta( + days=self.config.get("balance_assertion_date_fudge", 2) + ) + + dates = [ + ofx_max_transation_date, + max_transaction_date, + ofx_balance_date1, + ofx_balance_date2, + ] + if all( + v is None for v in dates[:2] + ): # because ofx_balance_date appears even for closed accounts return None - def vd(x): return x if x else datetime.date.min + def vd(x): + return x if x else datetime.date.min + return_date = max(*[vd(x) for x in dates]) # print("Smart date computation. Dates were: ", dates) return return_date def get_balance_assertion_date(self): - """ Choices for the date of the generated balance assertion can be specified in + """Choices for the date of the generated balance assertion can be specified in self.config['balance_assertion_date_type'], which can be: - 'smart': smart date (default) - 'ofx_date': date specified in ofx file @@ -139,16 +154,20 @@ def get_balance_assertion_date(self): on the beginning of the assertion date. """ - date_type_map = {'smart': self.get_smart_date, - 'ofx_date': self.get_ofx_end_date, - 'last_transaction': self.get_max_transaction_date, - 'today': datetime.date.today} - date_type = self.config.get('balance_assertion_date_type', 'smart') + date_type_map = { + "smart": self.get_smart_date, + "ofx_date": self.get_ofx_end_date, + "last_transaction": self.get_max_transaction_date, + "today": datetime.date.today, + } + date_type = self.config.get("balance_assertion_date_type", "smart") return_date = date_type_map[date_type]() if not return_date: return None - return return_date + datetime.timedelta(days=1) # Next day, as defined by Beancount + return return_date + datetime.timedelta( + days=1 + ) # Next day, as defined by Beancount def get_max_transaction_date(self): """ @@ -160,9 +179,10 @@ def get_max_transaction_date(self): """ try: - - date = max(ot.tradeDate if hasattr(ot, 'tradeDate') else ot.date - for ot in self.get_transactions()).date() + date = max( + ot.tradeDate if hasattr(ot, "tradeDate") else ot.date + for ot in self.get_transactions() + ).date() except TypeError: return None except ValueError: diff --git a/beancount_reds_importers/libreader/reader.py b/beancount_reds_importers/libreader/reader.py index 9efc9b6..d45fcbb 100644 --- a/beancount_reds_importers/libreader/reader.py +++ b/beancount_reds_importers/libreader/reader.py @@ -5,9 +5,9 @@ import re -class Reader(): - FILE_EXTS = [''] - IMPORTER_NAME = 'NOT SET' +class Reader: + FILE_EXTS = [""] + IMPORTER_NAME = "NOT SET" def identify(self, file): # quick check to filter out files that are not the right format @@ -18,17 +18,19 @@ def identify(self, file): # print("No match on extension") return False self.custom_init() - self.filename_pattern = self.config.get('filename_pattern', self.filename_pattern_def) + self.filename_pattern = self.config.get( + "filename_pattern", self.filename_pattern_def + ) if not re.match(self.filename_pattern, path.basename(file.name)): # print("No match on filename_pattern", self.filename_pattern, path.basename(file.name)) return False - self.currency = self.config.get('currency', 'CURRENCY_NOT_CONFIGURED') + self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED") self.initialize_reader(file) # print("reader_ready:", self.reader_ready) return self.reader_ready def file_name(self, file): - return '{}'.format(ntpath.basename(file.name)) + return "{}".format(ntpath.basename(file.name)) def file_account(self, file): # Ugly hack to handle an interaction with smart_importer. See: @@ -36,17 +38,18 @@ def file_account(self, file): # https://github.com/beancount/smart_importer/issues/122 # https://github.com/beancount/smart_importer/issues/30 import inspect + curframe = inspect.currentframe() calframe = inspect.getouterframes(curframe, 2) - if any('predictor' in i.filename for i in calframe): - if 'smart_importer_hack' in self.config: - return self.config['smart_importer_hack'] + if any("predictor" in i.filename for i in calframe): + if "smart_importer_hack" in self.config: + return self.config["smart_importer_hack"] # Otherwise handle a typical bean-file call self.initialize(file) - if 'filing_account' in self.config: - return self.config['filing_account'] - return self.config['main_account'] + if "filing_account" in self.config: + return self.config["filing_account"] + return self.config["main_account"] def get_balance_statement(self, file=None): return [] diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py index 848e283..cadbc9f 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py @@ -4,11 +4,13 @@ @regtest.with_importer( - ally.Importer({ - "account_number": "23456", - "main_account": "Assets:Banks:Checking", - "balance_assertion_date_type": "last_transaction", - }) + ally.Importer( + { + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + "balance_assertion_date_type": "last_transaction", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestSmart(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py index 0a1a210..fb1b011 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py @@ -4,11 +4,13 @@ @regtest.with_importer( - ally.Importer({ - "account_number": "23456", - "main_account": "Assets:Banks:Checking", - "balance_assertion_date_type": "ofx_date", - }) + ally.Importer( + { + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + "balance_assertion_date_type": "ofx_date", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestSmart(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py index ab7cecd..80ab291 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py @@ -5,10 +5,12 @@ # default balance_assertion_date_type is "smart" @regtest.with_importer( - ally.Importer({ - "account_number": "23456", - "main_account": "Assets:Banks:Checking", - }) + ally.Importer( + { + "account_number": "23456", + "main_account": "Assets:Banks:Checking", + } + ) ) @regtest.with_testdir(path.dirname(__file__)) class TestSmart(regtest.ImporterTestBase): diff --git a/beancount_reds_importers/libreader/tsvreader.py b/beancount_reds_importers/libreader/tsvreader.py index 4ab4560..05ba6a8 100644 --- a/beancount_reds_importers/libreader/tsvreader.py +++ b/beancount_reds_importers/libreader/tsvreader.py @@ -1,14 +1,13 @@ """tsv (tab separated values) importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" - from beancount.ingest import importer import petl as etl from beancount_reds_importers.libreader import csvreader class Importer(csvreader.Importer, importer.ImporterProtocol): - FILE_EXTS = ['tsv'] + FILE_EXTS = ["tsv"] def read_raw(self, file): return etl.fromtsv(file.name) diff --git a/beancount_reds_importers/libreader/xlsreader.py b/beancount_reds_importers/libreader/xlsreader.py index e2077a3..4174c37 100644 --- a/beancount_reds_importers/libreader/xlsreader.py +++ b/beancount_reds_importers/libreader/xlsreader.py @@ -8,19 +8,19 @@ class Importer(csvreader.Importer): - FILE_EXTS = ['xls'] + FILE_EXTS = ["xls"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.file_read_done = False self.reader_ready = False # TODO: this reads the entire file. Chop off after perhaps 2k or n lines rdr = self.read_raw(file) - header = '' + header = "" for r in rdr: - line = ''.join(str(x) for x in r) + line = "".join(str(x) for x in r) header += line # TODO @@ -33,4 +33,4 @@ def initialize_reader(self, file): def read_raw(self, file): # set logfile to ignore WARNING *** file size (92598) not 512 + multiple of sector size (512) - return etl.fromxls(file.name, logfile=open(devnull, 'w')) + return etl.fromxls(file.name, logfile=open(devnull, "w")) diff --git a/beancount_reds_importers/libreader/xlsx_multitable_reader.py b/beancount_reds_importers/libreader/xlsx_multitable_reader.py index 8274988..60455d0 100644 --- a/beancount_reds_importers/libreader/xlsx_multitable_reader.py +++ b/beancount_reds_importers/libreader/xlsx_multitable_reader.py @@ -13,10 +13,10 @@ class Importer(csv_multitable_reader.Importer): - FILE_EXTS = ['xlsx'] + FILE_EXTS = ["xlsx"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: + if getattr(self, "file", None) != file: self.file = file self.file_read_done = False self.reader_ready = True @@ -43,4 +43,4 @@ def read_raw(self, file): def is_section_title(self, row): if len(row) == 1: return True - return all(i == '' or i is None for i in row[1:]) + return all(i == "" or i is None for i in row[1:]) diff --git a/beancount_reds_importers/libreader/xlsxreader.py b/beancount_reds_importers/libreader/xlsxreader.py index 06199d0..a3f374e 100644 --- a/beancount_reds_importers/libreader/xlsxreader.py +++ b/beancount_reds_importers/libreader/xlsxreader.py @@ -6,7 +6,7 @@ class Importer(xlsreader.Importer): - FILE_EXTS = ['xlsx'] + FILE_EXTS = ["xlsx"] def read_raw(self, file): rdr = etl.fromxlsx(file.name) diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index 21b0fa7..f505077 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -8,7 +8,7 @@ from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder -Balance = namedtuple('Balance', ['date', 'amount', 'currency']) +Balance = namedtuple("Balance", ["date", "amount", "currency"]) class Importer(importer.ImporterProtocol, transactionbuilder.TransactionBuilder): @@ -55,7 +55,7 @@ def match_account_number(self, file_account, config_account): def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*bank_specific_filename.*' + self.filename_pattern_def = ".*bank_specific_filename.*" self.custom_init_run = True # def get_target_acct(self, transaction): @@ -68,11 +68,11 @@ def fields_contain_data(ot, fields): def get_main_account(self, ot): """Can be overridden by importer""" - return self.config['main_account'] + return self.config["main_account"] def get_target_account(self, ot): """Can be overridden by importer""" - return self.config.get('target_account') + return self.config.get("target_account") # -------------------------------------------------------------------------------- @@ -82,10 +82,15 @@ def extract_balance(self, file, counter): for bal in self.get_balance_statement(file=file): if bal: metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='balance')) - balance_entry = data.Balance(metadata, bal.date, self.config['main_account'], - amount.Amount(bal.amount, self.get_currency(bal)), - None, None) + metadata.update(self.build_metadata(file, metatype="balance")) + balance_entry = data.Balance( + metadata, + bal.date, + self.config["main_account"], + amount.Amount(bal.amount, self.get_currency(bal)), + None, + None, + ) entries.append(balance_entry) return entries @@ -110,9 +115,11 @@ def extract(self, file, existing_entries=None): continue metadata = data.new_metadata(file.name, next(counter)) # metadata['type'] = ot.type # Optional metadata, useful for debugging #TODO - metadata.update(self.build_metadata(file, - metatype='transaction', - data={'transaction': ot})) + metadata.update( + self.build_metadata( + file, metatype="transaction", data={"transaction": ot} + ) + ) # description fields: With OFX, ot.payee tends to be the "main" description field, # while ot.memo is optional @@ -125,24 +132,32 @@ def extract(self, file, existing_entries=None): # Banking transactions might include foreign currency transactions. TODO: figure out # how ofx handles this and use the same interface for csv and other files entry = data.Transaction( - meta=metadata, - date=ot.date.date(), - flag=self.FLAG, - # payee and narration are switched. See the preceding note - payee=self.get_narration(ot), - narration=self.get_payee(ot), - tags=self.get_tags(ot), - links=data.EMPTY_SET, - postings=[]) + meta=metadata, + date=ot.date.date(), + flag=self.FLAG, + # payee and narration are switched. See the preceding note + payee=self.get_narration(ot), + narration=self.get_payee(ot), + tags=self.get_tags(ot), + links=data.EMPTY_SET, + postings=[], + ) main_account = self.get_main_account(ot) - if self.fields_contain_data(ot, ['foreign_amount', 'foreign_currency']): - common.create_simple_posting_with_price(entry, main_account, - ot.amount, self.get_currency(ot), - ot.foreign_amount, ot.foreign_currency) + if self.fields_contain_data(ot, ["foreign_amount", "foreign_currency"]): + common.create_simple_posting_with_price( + entry, + main_account, + ot.amount, + self.get_currency(ot), + ot.foreign_amount, + ot.foreign_currency, + ) else: - data.create_simple_posting(entry, main_account, ot.amount, self.get_currency(ot)) + data.create_simple_posting( + entry, main_account, ot.amount, self.get_currency(ot) + ) # smart_importer can fill this in if the importer doesn't override self.get_target_acct() target_acct = self.get_target_account(ot) diff --git a/beancount_reds_importers/libtransactionbuilder/common.py b/beancount_reds_importers/libtransactionbuilder/common.py index 7d1fa79..45bade3 100644 --- a/beancount_reds_importers/libtransactionbuilder/common.py +++ b/beancount_reds_importers/libtransactionbuilder/common.py @@ -9,31 +9,55 @@ class PriceCostBothZeroException(Exception): """Raised when the input value is too small""" + pass -def create_simple_posting_with_price(entry, account, - number, currency, - price_number, price_currency): - return create_simple_posting_with_cost_or_price(entry, account, - number, currency, - price_number=price_number, price_currency=price_currency) +def create_simple_posting_with_price( + entry, account, number, currency, price_number, price_currency +): + return create_simple_posting_with_cost_or_price( + entry, + account, + number, + currency, + price_number=price_number, + price_currency=price_currency, + ) -def create_simple_posting_with_cost(entry, account, - number, currency, - cost_number, cost_currency, price_cost_both_zero_handler=None): - return create_simple_posting_with_cost_or_price(entry, account, - number, currency, - cost_number=cost_number, cost_currency=cost_currency, - price_cost_both_zero_handler=price_cost_both_zero_handler) +def create_simple_posting_with_cost( + entry, + account, + number, + currency, + cost_number, + cost_currency, + price_cost_both_zero_handler=None, +): + return create_simple_posting_with_cost_or_price( + entry, + account, + number, + currency, + cost_number=cost_number, + cost_currency=cost_currency, + price_cost_both_zero_handler=price_cost_both_zero_handler, + ) -def create_simple_posting_with_cost_or_price(entry, account, - number, currency, - price_number=None, price_currency=None, - cost_number=None, cost_currency=None, costspec=None, - price_cost_both_zero_handler=None): +def create_simple_posting_with_cost_or_price( + entry, + account, + number, + currency, + price_number=None, + price_currency=None, + cost_number=None, + cost_currency=None, + costspec=None, + price_cost_both_zero_handler=None, +): """Create a simple posting on the entry, with a cost (for purchases) or price (for sell transactions). Args: @@ -59,7 +83,11 @@ def create_simple_posting_with_cost_or_price(entry, account, if price_cost_both_zero_handler: price_cost_both_zero_handler() else: - print("WARNING: Either price ({}) or cost ({}) must be specified ({})".format(price_number, cost_number, entry)) + print( + "WARNING: Either price ({}) or cost ({}) must be specified ({})".format( + price_number, cost_number, entry + ) + ) raise PriceCostBothZeroException # import pdb; pdb.set_trace() diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index f0b9646..22321dd 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -83,51 +83,62 @@ def initialize(self, file): self.initialize_reader(file) if self.reader_ready: - config_subst_vars = {'currency': self.currency, - # Leave the other values as is - 'ticker': '{ticker}', - 'source401k': '{source401k}', - } + config_subst_vars = { + "currency": self.currency, + # Leave the other values as is + "ticker": "{ticker}", + "source401k": "{source401k}", + } self.set_config_variables(config_subst_vars) - self.money_market_funds = self.config['fund_info']['money_market'] - self.fund_data = self.config['fund_info']['fund_data'] # [(ticker, id, long_name), ...] + self.money_market_funds = self.config["fund_info"]["money_market"] + self.fund_data = self.config["fund_info"][ + "fund_data" + ] # [(ticker, id, long_name), ...] self.funds_by_id = {i: (ticker, desc) for ticker, i, desc in self.fund_data} - self.funds_by_ticker = {ticker: (ticker, desc) for ticker, _, desc in self.fund_data} + self.funds_by_ticker = { + ticker: (ticker, desc) for ticker, _, desc in self.fund_data + } # Most ofx/csv files refer to funds by id (cusip/isin etc.) Some use tickers instead - self.funds_db = getattr(self, getattr(self, 'funds_db_txt', 'funds_by_id')) + self.funds_db = getattr(self, getattr(self, "funds_db_txt", "funds_by_id")) self.build_account_map() self.initialized = True def build_account_map(self): + # fmt: off # map transaction types to target posting accounts self.target_account_map = { - "buymf": self.config['cash_account'], - "sellmf": self.config['cash_account'], - "buystock": self.config['cash_account'], - "sellstock": self.config['cash_account'], - "buyother": self.config['cash_account'], - "sellother": self.config['cash_account'], - "buydebt": self.config['cash_account'], - "reinvest": self.config['dividends'], - "dividends": self.config['dividends'], - "capgainsd_lt": self.config['capgainsd_lt'], - "capgainsd_st": self.config['capgainsd_st'], - "income": self.config['interest'], - "fee": self.config['fees'], - "invexpense": self.config.get('invexpense', "ACCOUNT_NOT_CONFIGURED:INVEXPENSE"), + "buymf": self.config["cash_account"], + "sellmf": self.config["cash_account"], + "buystock": self.config["cash_account"], + "sellstock": self.config["cash_account"], + "buyother": self.config["cash_account"], + "sellother": self.config["cash_account"], + "buydebt": self.config["cash_account"], + "reinvest": self.config["dividends"], + "dividends": self.config["dividends"], + "capgainsd_lt": self.config["capgainsd_lt"], + "capgainsd_st": self.config["capgainsd_st"], + "income": self.config["interest"], + "fee": self.config["fees"], + "invexpense": self.config.get("invexpense", "ACCOUNT_NOT_CONFIGURED:INVEXPENSE"), } - - if 'transfer' in self.config: - self.target_account_map.update({ - "other": self.config['transfer'], - "credit": self.config['transfer'], - "debit": self.config['transfer'], - "transfer": self.config['transfer'], - "cash": self.config['transfer'], - "dep": self.config['transfer'], - }) + # fmt: on + + if "transfer" in self.config: + # fmt: off + self.target_account_map.update( + { + "other": self.config["transfer"], + "credit": self.config["transfer"], + "debit": self.config["transfer"], + "transfer": self.config["transfer"], + "cash": self.config["transfer"], + "dep": self.config["transfer"], + } + ) + # fmt: on def build_metadata(self, file, metatype=None, data={}): """This method is for importers to override. The overridden method can @@ -138,20 +149,24 @@ def build_metadata(self, file, metatype=None, data={}): def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = '.*bank_specific_filename.*' + self.filename_pattern_def = ".*bank_specific_filename.*" self.custom_init_run = True def get_ticker_info(self, security_id): - return security_id, 'UNKNOWN' + return security_id, "UNKNOWN" def get_ticker_info_from_id(self, security_id): try: # isin might look like "US293409829" while the ofx use only a substring like "29340982" ticker = None try: # first try a full match, fall back to substring - ticker, ticker_long_name = [v for k, v in self.funds_db.items() if security_id == k][0] + ticker, ticker_long_name = [ + v for k, v in self.funds_db.items() if security_id == k + ][0] except IndexError: - ticker, ticker_long_name = [v for k, v in self.funds_db.items() if security_id in k][0] + ticker, ticker_long_name = [ + v for k, v in self.funds_db.items() if security_id in k + ][0] except IndexError: print(f"Error: fund info not found for {security_id}", file=sys.stderr) securities = self.get_security_list() @@ -189,8 +204,11 @@ def get_target_acct(self, transaction, ticker): target = self.get_target_acct_custom(transaction, ticker) if target: return target - if transaction.type == 'income' and getattr(transaction, 'income_type', None) == 'DIV': - return self.target_account_map.get('dividends', None) + if ( + transaction.type == "income" + and getattr(transaction, "income_type", None) == "DIV" + ): + return self.target_account_map.get("dividends", None) return self.target_account_map.get(transaction.type, None) def security_narration(self, ot): @@ -200,32 +218,35 @@ def security_narration(self, ot): def get_security_list(self): tickers = set() for ot in self.get_transactions(): - if hasattr(ot, 'security'): + if hasattr(ot, "security"): tickers.add(ot.security) return tickers def subst_acct_vars(self, raw_acct, ot, ticker): - """Resolve variables within an account like {ticker}. - """ + """Resolve variables within an account like {ticker}.""" ot = ot if ot else {} # inv401ksource is an ofx field that is 'PRETAX', 'AFTERTAX', etc. - kwargs = {'ticker': ticker, 'source401k': getattr(ot, 'inv401ksource', '').title()} + kwargs = { + "ticker": ticker, + "source401k": getattr(ot, "inv401ksource", "").title(), + } acct = raw_acct.format(**kwargs) return self.remove_empty_subaccounts(acct) # if 'inv401ksource' was unavailable def get_acct(self, acct, ot, ticker): - """Get an account from self.config, resolve variables, and return - """ + """Get an account from self.config, resolve variables, and return""" template = self.config.get(acct) if not template: - raise KeyError(f'{acct} not set in importer configuration. Config: {self.config}') + raise KeyError( + f"{acct} not set in importer configuration. Config: {self.config}" + ) return self.subst_acct_vars(template, ot, ticker) # extract() and supporting methods # -------------------------------------------------------------------------------- def generate_trade_entry(self, ot, file, counter): - """ Involves a commodity. One of: ['buymf', 'sellmf', 'buystock', 'sellstock', 'buyother', + """Involves a commodity. One of: ['buymf', 'sellmf', 'buystock', 'sellstock', 'buyother', 'sellother', 'reinvest']""" config = self.config @@ -234,9 +255,16 @@ def generate_trade_entry(self, ot, file, counter): # Build metadata metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='transaction_trade', data={'transaction': ot})) - if getattr(ot, 'settleDate', None) is not None and ot.settleDate != ot.tradeDate: - metadata['settlement_date'] = str(ot.settleDate.date()) + metadata.update( + self.build_metadata( + file, metatype="transaction_trade", data={"transaction": ot} + ) + ) + if ( + getattr(ot, "settleDate", None) is not None + and ot.settleDate != ot.tradeDate + ): + metadata["settlement_date"] = str(ot.settleDate.date()) narration = self.security_narration(ot) raw_target_acct = self.get_target_acct(ot, ticker) @@ -244,45 +272,75 @@ def generate_trade_entry(self, ot, file, counter): total = ot.total # special cases - if 'sell' in ot.type: + if "sell" in ot.type: units = -1 * abs(ot.units) if not is_money_market: - metadata['todo'] = 'TODO: this entry is incomplete until lots are selected (bean-doctor context )' # noqa: E501 - if ot.type in ['reinvest']: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI + metadata["todo"] = ( + "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 + ) + if ot.type in [ + "reinvest" + ]: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI ticker_val = ticker else: ticker_val = self.currency target_acct = self.subst_acct_vars(raw_target_acct, ot, ticker_val) # Build transaction entry - entry = data.Transaction(metadata, ot.tradeDate.date(), self.FLAG, - self.get_payee(ot), narration, - self.get_tags(ot), data.EMPTY_SET, []) + entry = data.Transaction( + metadata, + ot.tradeDate.date(), + self.FLAG, + self.get_payee(ot), + narration, + self.get_tags(ot), + data.EMPTY_SET, + [], + ) # Main posting(s): - main_acct = self.get_acct('main_account', ot, ticker) + main_acct = self.get_acct("main_account", ot, ticker) if is_money_market: # Use price conversions instead of holding these at cost - common.create_simple_posting_with_price(entry, main_acct, - units, ticker, ot.unit_price, self.currency) - elif 'sell' in ot.type: - common.create_simple_posting_with_cost_or_price(entry, main_acct, - units, ticker, price_number=ot.unit_price, - price_currency=self.currency, - costspec=CostSpec(None, None, None, None, None, None)) - cg_acct = self.get_acct('cg', ot, ticker) + common.create_simple_posting_with_price( + entry, main_acct, units, ticker, ot.unit_price, self.currency + ) + elif "sell" in ot.type: + common.create_simple_posting_with_cost_or_price( + entry, + main_acct, + units, + ticker, + price_number=ot.unit_price, + price_currency=self.currency, + costspec=CostSpec(None, None, None, None, None, None), + ) + cg_acct = self.get_acct("cg", ot, ticker) data.create_simple_posting(entry, cg_acct, None, None) else: # buy stock/fund - unit_price = getattr(ot, 'unit_price', 0) + unit_price = getattr(ot, "unit_price", 0) # annoyingly, vanguard reinvests have ot.unit_price set to zero. so manually compute it - if (hasattr(ot, 'security') and ot.security) and ot.units and not ot.unit_price: + if ( + (hasattr(ot, "security") and ot.security) + and ot.units + and not ot.unit_price + ): unit_price = round(abs(ot.total) / ot.units, 4) - common.create_simple_posting_with_cost(entry, main_acct, units, ticker, unit_price, - self.currency, self.price_cost_both_zero_handler) + common.create_simple_posting_with_cost( + entry, + main_acct, + units, + ticker, + unit_price, + self.currency, + self.price_cost_both_zero_handler, + ) # "Other" account posting reverser = 1 - if units > 0 and total > 0: # (ugly) hack for some brokerages with incorrect signs (TODO: remove) + if ( + units > 0 and total > 0 + ): # (ugly) hack for some brokerages with incorrect signs (TODO: remove) reverser = -1 data.create_simple_posting(entry, target_acct, reverser * total, self.currency) @@ -290,7 +348,8 @@ def generate_trade_entry(self, ot, file, counter): rounding_error = (reverser * total) + (ot.unit_price * units) if 0.0005 <= abs(rounding_error) <= self.max_rounding_error: data.create_simple_posting( - entry, config['rounding_error'], -1 * rounding_error, self.currency) + entry, config["rounding_error"], -1 * rounding_error, self.currency + ) # if abs(rounding_error) > self.max_rounding_error: # print("Transactions legs do not sum up! Difference: {}. Entry: {}, ot: {}".format( # rounding_error, entry, ot)) @@ -298,21 +357,34 @@ def generate_trade_entry(self, ot, file, counter): return entry def generate_transfer_entry(self, ot, file, counter): - """ Cash transactions, or in-kind transfers. One of: - [credit, debit, dep, transfer, income, dividends, capgainsd_lt, capgainsd_st, other]""" + """Cash transactions, or in-kind transfers. One of: + [credit, debit, dep, transfer, income, dividends, capgainsd_lt, capgainsd_st, other]""" config = self.config metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='transaction_transfer', data={'transaction': ot})) + metadata.update( + self.build_metadata( + file, metatype="transaction_transfer", data={"transaction": ot} + ) + ) ticker = None - date = getattr(ot, 'tradeDate', None) + date = getattr(ot, "tradeDate", None) if not date: date = ot.date date = date.date() try: - if ot.type in ['transfer']: + if ot.type in ["transfer"]: units = ot.units - elif ot.type in ['other', 'credit', 'debit', 'dep', 'cash', 'payment', 'check', 'xfer']: + elif ot.type in [ + "other", + "credit", + "debit", + "dep", + "cash", + "payment", + "check", + "xfer", + ]: units = ot.amount else: units = ot.total @@ -321,28 +393,48 @@ def generate_transfer_entry(self, ot, file, counter): # import pdb; pdb.set_trace() main_acct = None - if ot.type in ['income', 'dividends', 'capgainsd_lt', - 'capgainsd_st', 'transfer'] and (hasattr(ot, 'security') and ot.security): + if ot.type in [ + "income", + "dividends", + "capgainsd_lt", + "capgainsd_st", + "transfer", + ] and (hasattr(ot, "security") and ot.security): ticker, ticker_long_name = self.get_ticker_info(ot.security) narration = self.security_narration(ot) - main_acct = self.get_acct('main_account', ot, ticker) + main_acct = self.get_acct("main_account", ot, ticker) else: # cash transaction narration = ot.type ticker = self.currency - main_acct = config['cash_account'] + main_acct = config["cash_account"] # Build transaction entry - entry = data.Transaction(metadata, date, self.FLAG, - self.get_payee(ot), narration, - self.get_tags(ot), data.EMPTY_SET, []) + entry = data.Transaction( + metadata, + date, + self.FLAG, + self.get_payee(ot), + narration, + self.get_tags(ot), + data.EMPTY_SET, + [], + ) target_acct = self.get_target_acct(ot, ticker) if target_acct: target_acct = self.subst_acct_vars(target_acct, ot, ticker) # Build postings - if ot.type in ['income', 'dividends', 'capgainsd_st', 'capgainsd_lt', 'fee']: # cash - amount = ot.total if hasattr(ot, 'total') else ot.amount - data.create_simple_posting(entry, config['cash_account'], amount, self.currency) + if ot.type in [ + "income", + "dividends", + "capgainsd_st", + "capgainsd_lt", + "fee", + ]: # cash + amount = ot.total if hasattr(ot, "total") else ot.amount + data.create_simple_posting( + entry, config["cash_account"], amount, self.currency + ) data.create_simple_posting(entry, target_acct, -1 * amount, self.currency) else: data.create_simple_posting(entry, main_acct, units, ticker) @@ -371,14 +463,38 @@ def extract_transactions(self, file, counter): for ot in self.get_transactions(): if self.skip_transaction(ot): continue - if ot.type in ['buymf', 'sellmf', 'buystock', 'buydebt', 'sellstock', 'buyother', 'sellother', 'reinvest']: + if ot.type in [ + "buymf", + "sellmf", + "buystock", + "buydebt", + "sellstock", + "buyother", + "sellother", + "reinvest", + ]: entry = self.generate_trade_entry(ot, file, counter) - elif ot.type in ['other', 'credit', 'debit', 'transfer', 'xfer', 'dep', 'income', 'fee', - 'dividends', 'capgainsd_st', 'capgainsd_lt', 'cash', 'payment', 'check', 'invexpense']: + elif ot.type in [ + "other", + "credit", + "debit", + "transfer", + "xfer", + "dep", + "income", + "fee", + "dividends", + "capgainsd_st", + "capgainsd_lt", + "cash", + "payment", + "check", + "invexpense", + ]: entry = self.generate_transfer_entry(ot, file, counter) else: print("ERROR: unknown entry type:", ot.type) - raise Exception('Unknown entry type') + raise Exception("Unknown entry type") self.add_fee_postings(entry, ot) self.add_custom_postings(entry, ot) new_entries.append(entry) @@ -392,37 +508,57 @@ def extract_balances_and_prices(self, file, counter): for pos in self.get_balance_positions(): ticker, ticker_long_name = self.get_ticker_info(pos.security) metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='balance', data={'pos': pos})) + metadata.update( + self.build_metadata(file, metatype="balance", data={"pos": pos}) + ) # if there are no transactions, use the date in the source file for the balance. This gives us the # bonus of an updated, recent balance assertion bal_date = date if date else pos.date.date() - main_acct = self.get_acct('main_account', None, ticker) - balance_entry = data.Balance(metadata, bal_date, main_acct, - amount.Amount(pos.units, ticker), - None, None) + main_acct = self.get_acct("main_account", None, ticker) + balance_entry = data.Balance( + metadata, + bal_date, + main_acct, + amount.Amount(pos.units, ticker), + None, + None, + ) new_entries.append(balance_entry) if ticker in self.money_market_funds: settlement_fund_balance = pos.units # extract price info if available - if hasattr(pos, 'unit_price') and hasattr(pos, 'date'): + if hasattr(pos, "unit_price") and hasattr(pos, "date"): metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='price', data={'pos': pos})) - price_entry = data.Price(metadata, pos.date.date(), ticker, - amount.Amount(pos.unit_price, self.currency)) + metadata.update( + self.build_metadata(file, metatype="price", data={"pos": pos}) + ) + price_entry = data.Price( + metadata, + pos.date.date(), + ticker, + amount.Amount(pos.unit_price, self.currency), + ) new_entries.append(price_entry) # ----------------- available cash available_cash = self.get_available_cash(settlement_fund_balance) if available_cash is not None: metadata = data.new_metadata(file.name, next(counter)) - metadata.update(self.build_metadata(file, metatype='balance_cash')) + metadata.update(self.build_metadata(file, metatype="balance_cash")) try: - bal_date = date if date else self.file_date(file).date() # unavailable file_date raises AttributeError - balance_entry = data.Balance(metadata, bal_date, self.config['cash_account'], - amount.Amount(available_cash, self.currency), - None, None) + bal_date = ( + date if date else self.file_date(file).date() + ) # unavailable file_date raises AttributeError + balance_entry = data.Balance( + metadata, + bal_date, + self.config["cash_account"], + amount.Amount(available_cash, self.currency), + None, + None, + ) new_entries.append(balance_entry) except AttributeError: pass @@ -431,11 +567,15 @@ def extract_balances_and_prices(self, file, counter): def add_fee_postings(self, entry, ot): config = self.config - if hasattr(ot, 'fees') or hasattr(ot, 'commission'): - if getattr(ot, 'fees', 0) != 0: - data.create_simple_posting(entry, config['fees'], ot.fees, self.currency) - if getattr(ot, 'commission', 0) != 0: - data.create_simple_posting(entry, config['fees'], ot.commission, self.currency) + if hasattr(ot, "fees") or hasattr(ot, "commission"): + if getattr(ot, "fees", 0) != 0: + data.create_simple_posting( + entry, config["fees"], ot.fees, self.currency + ) + if getattr(ot, "commission", 0) != 0: + data.create_simple_posting( + entry, config["fees"], ot.commission, self.currency + ) def add_custom_postings(self, entry, ot): pass diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index 24efeb7..63c4232 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -50,10 +50,15 @@ # }, # } + def flip_if_needed(amount, account): - if amount >= 0 and any(account.startswith(prefix) for prefix in ['Income:', 'Equity:', 'Liabilities:']): + if amount >= 0 and any( + account.startswith(prefix) for prefix in ["Income:", "Equity:", "Liabilities:"] + ): amount *= -1 - if amount < 0 and any(account.startswith(prefix) for prefix in ['Expenses:', 'Assets:']): + if amount < 0 and any( + account.startswith(prefix) for prefix in ["Expenses:", "Assets:"] + ): amount *= -1 return amount @@ -63,8 +68,8 @@ def file_date(self, input_file): return self.paycheck_date(input_file) def build_postings(self, entry): - template = self.config['paycheck_template'] - currency = self.config['currency'] + template = self.config["paycheck_template"] + currency = self.config["currency"] total = 0 template_missing = defaultdict(set) @@ -74,16 +79,29 @@ def build_postings(self, entry): continue for row in table.namedtuples(): # TODO: 'bank' is workday specific; move it there - row_description = getattr(row, 'description', getattr(row, 'bank', None)) - row_pattern = next(filter(lambda ts: row_description.startswith(ts), template[section]), None) + row_description = getattr( + row, "description", getattr(row, "bank", None) + ) + row_pattern = next( + filter( + lambda ts: row_description.startswith(ts), template[section] + ), + None, + ) if not row_pattern: template_missing[section].add(row_description) else: accounts = template[section][row_pattern] - accounts = [accounts] if not isinstance(accounts, list) else accounts + accounts = ( + [accounts] if not isinstance(accounts, list) else accounts + ) for account in accounts: # TODO: 'amount_in_pay_group_currency' is workday specific; move it there - amount = getattr(row, 'amount', getattr(row, 'amount_in_pay_group_currency', None)) + amount = getattr( + row, + "amount", + getattr(row, "amount_in_pay_group_currency", None), + ) # import pdb; pdb.set_trace() if not amount: @@ -94,17 +112,17 @@ def build_postings(self, entry): if amount: data.create_simple_posting(entry, account, amount, currency) - if self.config.get('show_unconfigured', False): + if self.config.get("show_unconfigured", False): for section in template_missing: print(section) if template_missing[section]: - print(' ' + '\n '.join(i for i in template_missing[section])) + print(" " + "\n ".join(i for i in template_missing[section])) print() if total != 0: data.create_simple_posting(entry, "TOTAL:NONZERO", total, currency) - if self.config.get('sort_postings', True): + if self.config.get("sort_postings", True): postings = sorted(entry.postings) else: postings = entry.postings @@ -123,9 +141,17 @@ def extract(self, file, existing_entries=None): self.read_file(file) metadata = data.new_metadata(file.name, 0) - metadata.update(self.build_metadata(file, metatype='transaction')) - entry = data.Transaction(metadata, self.paycheck_date(file), self.FLAG, - None, config['desc'], self.get_tags(), data.EMPTY_SET, []) + metadata.update(self.build_metadata(file, metatype="transaction")) + entry = data.Transaction( + metadata, + self.paycheck_date(file), + self.FLAG, + None, + config["desc"], + self.get_tags(), + data.EMPTY_SET, + [], + ) entry = self.build_postings(entry) return [entry] diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index 56ea30b..f789e9b 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -4,7 +4,7 @@ from beancount.core import data -class TransactionBuilder(): +class TransactionBuilder: def skip_transaction(self, ot): """For custom importers to override""" return False @@ -16,7 +16,7 @@ def get_tags(self, ot=None): @staticmethod def remove_empty_subaccounts(acct): """Translates 'Assets:Foo::Bar' to 'Assets:Foo:Bar'.""" - return ':'.join(x for x in acct.split(':') if x) + return ":".join(x for x in acct.split(":") if x) def set_config_variables(self, substs): """ @@ -31,11 +31,16 @@ def set_config_variables(self, substs): 'source401k': '{source401k}', } """ - self.config = {k: v.format(**substs) if isinstance(v, str) else v for k, v in self.config.items()} + self.config = { + k: v.format(**substs) if isinstance(v, str) else v + for k, v in self.config.items() + } # Prevent the replacement fields from appearing in the output of # the file_account method - if 'filing_account' not in self.config: - kwargs = {k: '' for k in substs} - filing_account = self.config['main_account'].format(**kwargs) - self.config['filing_account'] = self.remove_empty_subaccounts(filing_account) + if "filing_account" not in self.config: + kwargs = {k: "" for k in substs} + filing_account = self.config["main_account"].format(**kwargs) + self.config["filing_account"] = self.remove_empty_subaccounts( + filing_account + ) diff --git a/beancount_reds_importers/util/bean_download.py b/beancount_reds_importers/util/bean_download.py index 9bf37ec..fd4d8fc 100755 --- a/beancount_reds_importers/util/bean_download.py +++ b/beancount_reds_importers/util/bean_download.py @@ -30,26 +30,31 @@ def readConfigFile(configfile): def get_sites(sites, t, config): - return [s for s in sites if config[s]['type'] == t] - - -@cli.command(aliases=['list']) -@click.option('-c', '--config-file', envvar='BEAN_DOWNLOAD_CONFIG', required=True, - help='Config file. The environment variable BEAN_DOWNLOAD_CONFIG can also be used to specify this', - type=click.Path(exists=True)) -@click.option('-s', '--sort', is_flag=True, help='Sort output') + return [s for s in sites if config[s]["type"] == t] + + +@cli.command(aliases=["list"]) +@click.option( + "-c", + "--config-file", + envvar="BEAN_DOWNLOAD_CONFIG", + required=True, + help="Config file. The environment variable BEAN_DOWNLOAD_CONFIG can also be used to specify this", + type=click.Path(exists=True), +) +@click.option("-s", "--sort", is_flag=True, help="Sort output") def list_institutions(config_file, sort): """List institutions (sites) currently configured.""" config = readConfigFile(config_file) all_sites = config.sections() - types = set([config[s]['type'] for s in all_sites]) + types = set([config[s]["type"] for s in all_sites]) for t in sorted(types): sites = get_sites(all_sites, t, config) if sort: sites = sorted(sites) name = f"{t} ({len(sites)})".ljust(14) - print(f"{name}:", end='') - print(*sites, sep=', ') + print(f"{name}:", end="") + print(*sites, sep=", ") print() @@ -57,31 +62,48 @@ def get_sites_and_sections(config_file): if config_file and os.path.exists(config_file): config = readConfigFile(config_file) all_sites = config.sections() - types = set([config[s]['type'] for s in all_sites]) + types = set([config[s]["type"] for s in all_sites]) return all_sites, types def complete_sites(ctx, param, incomplete): - config_file = ctx.params['config_file'] + config_file = ctx.params["config_file"] all_sites, _ = get_sites_and_sections(config_file) return [s for s in all_sites if s.startswith(incomplete)] def complete_site_types(ctx, param, incomplete): - config_file = ctx.params['config_file'] + config_file = ctx.params["config_file"] _, types = get_sites_and_sections(config_file) return [s for s in types if s.startswith(incomplete)] @cli.command() -@click.option('-c', '--config-file', envvar='BEAN_DOWNLOAD_CONFIG', required=True, help='Config file') -@click.option('-i', '--sites', '--institutions', help="Institutions to download (comma separated); unspecified means all", - default='', shell_complete=complete_sites) -@click.option('-t', '--site-types', '--institution-types', - help="Download all institutions of specified types (comma separated)", - default='', shell_complete=complete_site_types) -@click.option('--dry-run', is_flag=True, help="Do not actually download", default=False) -@click.option('--verbose', is_flag=True, help="Verbose", default=False) +@click.option( + "-c", + "--config-file", + envvar="BEAN_DOWNLOAD_CONFIG", + required=True, + help="Config file", +) +@click.option( + "-i", + "--sites", + "--institutions", + help="Institutions to download (comma separated); unspecified means all", + default="", + shell_complete=complete_sites, +) +@click.option( + "-t", + "--site-types", + "--institution-types", + help="Download all institutions of specified types (comma separated)", + default="", + shell_complete=complete_site_types, +) +@click.option("--dry-run", is_flag=True, help="Do not actually download", default=False) +@click.option("--verbose", is_flag=True, help="Verbose", default=False) def download(config_file, sites, site_types, dry_run, verbose): # noqa: C901 """Download statements for the specified institutions (sites).""" @@ -91,12 +113,14 @@ def pverbose(*args, **kwargs): config = readConfigFile(config_file) if sites: - sites = sites.split(',') + sites = sites.split(",") else: sites = config.sections() if site_types: - site_types = site_types.split(',') - sites_lists = [get_sites(sites, site_type, config) for site_type in site_types] + site_types = site_types.split(",") + sites_lists = [ + get_sites(sites, site_type, config) for site_type in site_types + ] sites = [j for i in sites_lists for j in i] errors = [] @@ -106,8 +130,8 @@ def pverbose(*args, **kwargs): print(f"Processing {numsites} institutions.") async def download_site(i, site): - tid = f'[{i+1}/{numsites} {site}]' - pverbose(f'{tid}: Begin') + tid = f"[{i+1}/{numsites} {site}]" + pverbose(f"{tid}: Begin") try: options = config[site] except KeyError: @@ -116,11 +140,11 @@ async def download_site(i, site): return # We support cmd and display, and type to filter - if 'display' in options: + if "display" in options: displays.append([site, f"{options['display']}"]) success.append(site) - if 'cmd' in options: - cmd = os.path.expandvars(options['cmd']) + if "cmd" in options: + cmd = os.path.expandvars(options["cmd"]) pverbose(f"{tid}: Executing: {cmd}") if dry_run: await asyncio.sleep(2) @@ -129,9 +153,8 @@ async def download_site(i, site): else: # https://docs.python.org/3.8/library/asyncio-subprocess.html#asyncio.create_subprocess_exec proc = await asyncio.create_subprocess_shell( - cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) + cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) stdout, stderr = await proc.communicate() if proc.returncode: @@ -150,26 +173,30 @@ async def perform_downloads(sites): if displays: print() displays = [[i + 1, *row] for i, row in enumerate(displays)] - click.secho(tabulate.tabulate(displays, - headers=["#", "Institution", "Instructions"], tablefmt="plain"), fg='blue') + click.secho( + tabulate.tabulate( + displays, headers=["#", "Institution", "Instructions"], tablefmt="plain" + ), + fg="blue", + ) print() s = len(sites) if success: print(f"{len(success)}/{s} sites succeeded: {', '.join(success)}") if errors: - click.secho(f"{len(errors)}/{s} sites failed: {', '.join(errors)}", fg='red') + click.secho(f"{len(errors)}/{s} sites failed: {', '.join(errors)}", fg="red") -@cli.command(aliases=['init']) +@cli.command(aliases=["init"]) def config_template(): """Output a template for download.cfg that you can then use to build your own.""" path = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(*[path, 'template.cfg'])) as f: + with open(os.path.join(*[path, "template.cfg"])) as f: for line in f: - print(line, end='') + print(line, end="") -if __name__ == '__main__': +if __name__ == "__main__": cli() diff --git a/beancount_reds_importers/util/needs_update.py b/beancount_reds_importers/util/needs_update.py index e93c1fc..0d1c9e5 100755 --- a/beancount_reds_importers/util/needs_update.py +++ b/beancount_reds_importers/util/needs_update.py @@ -11,33 +11,45 @@ import ast -tbl_options = {'tablefmt': 'simple'} +tbl_options = {"tablefmt": "simple"} def get_config(entries, args): """Get beancount config for the given plugin that can then be used on the command line""" global excluded_re, included_re - _extension_entries = [e for e in entries - if isinstance(e, Custom) and e.type == 'reds-importers'] - config_meta = {entry.values[0].value: - (entry.values[1].value if (len(entry.values) == 2) else None) - for entry in _extension_entries} - - config = {k: ast.literal_eval(v) for k, v in config_meta.items() if 'needs-updates' in k} - config = config.get('needs-updates', {}) - if args['all_accounts']: - config['included_account_pats'] = [] - config['excluded_account_pats'] = ['$-^'] - included_account_pats = config.get('included_account_pats', ['^Assets:', '^Liabilities:']) - excluded_account_pats = config.get('excluded_account_pats', ['$-^']) # exclude nothing by default - excluded_re = re.compile('|'.join(excluded_account_pats)) - included_re = re.compile('|'.join(included_account_pats)) + _extension_entries = [ + e for e in entries if isinstance(e, Custom) and e.type == "reds-importers" + ] + config_meta = { + entry.values[0].value: ( + entry.values[1].value if (len(entry.values) == 2) else None + ) + for entry in _extension_entries + } + + config = { + k: ast.literal_eval(v) for k, v in config_meta.items() if "needs-updates" in k + } + config = config.get("needs-updates", {}) + if args["all_accounts"]: + config["included_account_pats"] = [] + config["excluded_account_pats"] = ["$-^"] + included_account_pats = config.get( + "included_account_pats", ["^Assets:", "^Liabilities:"] + ) + excluded_account_pats = config.get( + "excluded_account_pats", ["$-^"] + ) # exclude nothing by default + excluded_re = re.compile("|".join(excluded_account_pats)) + included_re = re.compile("|".join(included_account_pats)) def is_interesting_account(account, closes): - return account not in closes and \ - included_re.match(account) and \ - not excluded_re.match(account) + return ( + account not in closes + and included_re.match(account) + and not excluded_re.match(account) + ) def handle_commodity_leaf_accounts(last_balance): @@ -49,9 +61,9 @@ def handle_commodity_leaf_accounts(last_balance): considered to be the latest date of a balance assertion on any child. """ d = {} - pat_ticker = re.compile(r'^[A-Z0-9]+$') + pat_ticker = re.compile(r"^[A-Z0-9]+$") for acc in last_balance: - parent, leaf = acc.rsplit(':', 1) + parent, leaf = acc.rsplit(":", 1) if pat_ticker.match(leaf): if parent in d: if d[parent].date < last_balance[acc].date: @@ -72,33 +84,48 @@ def accounts_with_no_balance_entries(entries, closes, last_balance): # Handle commodity leaf accounts accs_no_bal = [] - pat_ticker = re.compile(r'^[A-Z0-9]+$') + pat_ticker = re.compile(r"^[A-Z0-9]+$") def acc_or_parent(acc): - parent, leaf = acc.rsplit(':', 1) + parent, leaf = acc.rsplit(":", 1) if pat_ticker.match(leaf): return parent return acc + accs_no_bal = [acc_or_parent(i) for i in accs_no_bal_raw] # Remove accounts where one or more children do have a balance entry. Needed because of # commodity leaf accounts - accs_no_bal = [(i,) for i in set(accs_no_bal) if not any(j.startswith(i) for j in last_balance)] + accs_no_bal = [ + (i,) for i in set(accs_no_bal) if not any(j.startswith(i) for j in last_balance) + ] return accs_no_bal def pretty_print_table(not_updated_accounts, sort_by_date): field = 0 if sort_by_date else 1 - output = sorted([(v.date, k) for k, v in not_updated_accounts.items()], key=lambda x: x[field]) - headers = ['Last Updated', 'Account'] + output = sorted( + [(v.date, k) for k, v in not_updated_accounts.items()], key=lambda x: x[field] + ) + headers = ["Last Updated", "Account"] print(click.style(tabulate.tabulate(output, headers=headers, **tbl_options))) -@click.command("needs-update", context_settings={'show_default': True}) -@click.argument('beancount-file', type=click.Path(exists=True), envvar='BEANCOUNT_FILE') -@click.option('--recency', help='How many days ago should the last balance assertion be to be considered old', default=15) -@click.option('--sort-by-date', help='Sort output by date (instead of account name)', is_flag=True) -@click.option('--all-accounts', help='Show all account (ignore include/exclude in config)', is_flag=True) +@click.command("needs-update", context_settings={"show_default": True}) +@click.argument("beancount-file", type=click.Path(exists=True), envvar="BEANCOUNT_FILE") +@click.option( + "--recency", + help="How many days ago should the last balance assertion be to be considered old", + default=15, +) +@click.option( + "--sort-by-date", help="Sort output by date (instead of account name)", is_flag=True +) +@click.option( + "--all-accounts", + help="Show all account (ignore include/exclude in config)", + is_flag=True, +) def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts): """ Show a list of accounts needing updates, and the date of the last update (which is defined as @@ -141,21 +168,33 @@ def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts entries, _, _ = loader.load_file(beancount_file) get_config(entries, locals()) closes = [a.account for a in entries if isinstance(a, Close)] - balance_entries = [a for a in entries if isinstance(a, Balance) and - is_interesting_account(a.account, closes)] + balance_entries = [ + a + for a in entries + if isinstance(a, Balance) and is_interesting_account(a.account, closes) + ] last_balance = {v.account: v for v in balance_entries} d = handle_commodity_leaf_accounts(last_balance) # find accounts with balance assertions older than N days - need_updates = {acc: bal for acc, bal in d.items() if ((datetime.now().date() - d[acc].date).days > recency)} + need_updates = { + acc: bal + for acc, bal in d.items() + if ((datetime.now().date() - d[acc].date).days > recency) + } pretty_print_table(need_updates, sort_by_date) # If there are accounts with zero balance entries, print them accs_no_bal = accounts_with_no_balance_entries(entries, closes, last_balance) if accs_no_bal: - headers = ['Accounts without balance entries:'] - print(click.style('\n' + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options))) + headers = ["Accounts without balance entries:"] + print( + click.style( + "\n" + + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options) + ) + ) -if __name__ == '__main__': +if __name__ == "__main__": accounts_needing_updates() diff --git a/beancount_reds_importers/util/ofx_summarize.py b/beancount_reds_importers/util/ofx_summarize.py index e5ab2f6..541465b 100755 --- a/beancount_reds_importers/util/ofx_summarize.py +++ b/beancount_reds_importers/util/ofx_summarize.py @@ -8,10 +8,11 @@ from ofxparse import OfxParser from bs4.builder import XMLParsedAsHTMLWarning import warnings + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) -def analyze(filename, ttype='dividends', pdb_explore=False): +def analyze(filename, ttype="dividends", pdb_explore=False): ts = defaultdict(list) ofx = OfxParser.parse(open(filename)) for acc in ofx.accounts: @@ -19,14 +20,21 @@ def analyze(filename, ttype='dividends', pdb_explore=False): ts[t.type].append(t) if pdb_explore: import pdb + pdb.set_trace() @click.command() -@click.argument('filename', type=click.Path(exists=True)) -@click.option('-n', '--num-transactions', default=5, help='Number of transactions to show') -@click.option('-e', '--pdb-explore', is_flag=True, help='Open a pdb shell to explore') -@click.option('--stats-only', is_flag=True, help='Print total number of transactions contained in the file, and quit') +@click.argument("filename", type=click.Path(exists=True)) +@click.option( + "-n", "--num-transactions", default=5, help="Number of transactions to show" +) +@click.option("-e", "--pdb-explore", is_flag=True, help="Open a pdb shell to explore") +@click.option( + "--stats-only", + is_flag=True, + help="Print total number of transactions contained in the file, and quit", +) def summarize(filename, pdb_explore, num_transactions, stats_only): # noqa: C901 """Quick and dirty way to summarize a .ofx file and peek inside it.""" if os.stat(filename).st_size == 0: @@ -45,45 +53,64 @@ def summarize(filename, pdb_explore, num_transactions, stats_only): # noqa: C90 sys.exit(0) print("Total number of accounts:", len(ofx.accounts)) for acc in ofx.accounts: - print('----------------') + print("----------------") try: - print("Account info: ", acc.account_type, acc.account_id, acc.institution.organization) + print( + "Account info: ", + acc.account_type, + acc.account_id, + acc.institution.organization, + ) except AttributeError: print("Account info: ", acc.account_type, acc.account_id) pass try: - print("Statement info: {} -- {}. Bal: {}".format(acc.statement.start_date, - acc.statement.end_date, acc.statement.balance)) + print( + "Statement info: {} -- {}. Bal: {}".format( + acc.statement.start_date, + acc.statement.end_date, + acc.statement.balance, + ) + ) except AttributeError: try: positions = [(p.units, p.security) for p in acc.statement.positions] - print("Statement info: {} -- {}. Bal: {}".format(acc.statement.start_date, - acc.statement.end_date, positions)) + print( + "Statement info: {} -- {}. Bal: {}".format( + acc.statement.start_date, acc.statement.end_date, positions + ) + ) except AttributeError: print("Statement info: UNABLE to get start_date and end_date") print("Types: ", set([t.type for t in acc.statement.transactions])) print() - txns = sorted(acc.statement.transactions, reverse=True, - key=lambda t: t.date if hasattr(t, 'date') else t.tradeDate) + txns = sorted( + acc.statement.transactions, + reverse=True, + key=lambda t: t.date if hasattr(t, "date") else t.tradeDate, + ) for t in txns[:num_transactions]: - date = t.date if hasattr(t, 'date') else t.tradeDate - description = t.payee + ' ' + t.memo if hasattr(t, 'payee') else t.memo - amount = t.amount if hasattr(t, 'amount') else t.total + date = t.date if hasattr(t, "date") else t.tradeDate + description = t.payee + " " + t.memo if hasattr(t, "payee") else t.memo + amount = t.amount if hasattr(t, "amount") else t.total print(date, t.type, description, amount) if pdb_explore: print("Hints:") print("- try dir(acc), dir(acc.statement.transactions)") - print("- try the 'interact' command to start an interactive python interpreter") + print( + "- try the 'interact' command to start an interactive python interpreter" + ) if len(ofx.accounts) > 1: print("- type 'c' to explore the next account in this file") import pdb + pdb.set_trace() print() print() -if __name__ == '__main__': +if __name__ == "__main__": summarize() diff --git a/setup.py b/setup.py index fd3debf..0926e9f 100644 --- a/setup.py +++ b/setup.py @@ -1,54 +1,54 @@ from os import path from setuptools import find_packages, setup -with open(path.join(path.dirname(__file__), 'README.md')) as readme: +with open(path.join(path.dirname(__file__), "README.md")) as readme: LONG_DESCRIPTION = readme.read() setup( - name='beancount_reds_importers', + name="beancount_reds_importers", use_scm_version=True, - setup_requires=['setuptools_scm'], - description='Importers for various institutions for Beancount', + setup_requires=["setuptools_scm"], + description="Importers for various institutions for Beancount", long_description=LONG_DESCRIPTION, - long_description_content_type='text/markdown', - url='https://github.com/redstreet/beancount_reds_importers', - author='Red Street', - author_email='redstreet@users.noreply.github.com', - keywords='importer ingestor beancount accounting', - license='GPL-3.0', + long_description_content_type="text/markdown", + url="https://github.com/redstreet/beancount_reds_importers", + author="Red Street", + author_email="redstreet@users.noreply.github.com", + keywords="importer ingestor beancount accounting", + license="GPL-3.0", packages=find_packages(), include_package_data=True, extras_require={ - 'dev': [ - 'ruff', + "dev": [ + "ruff", ] }, install_requires=[ - 'Click >= 7.0', - 'beancount >= 2.3.5', - 'click_aliases >= 1.0.1', - 'ofxparse >= 0.21', - 'openpyxl >= 3.0.9', - 'packaging >= 20.3', - 'petl >= 1.7.4', - 'tabulate >= 0.8.9', - 'tqdm >= 4.64.0', + "Click >= 7.0", + "beancount >= 2.3.5", + "click_aliases >= 1.0.1", + "ofxparse >= 0.21", + "openpyxl >= 3.0.9", + "packaging >= 20.3", + "petl >= 1.7.4", + "tabulate >= 0.8.9", + "tqdm >= 4.64.0", ], entry_points={ - 'console_scripts': [ - 'ofx-summarize = beancount_reds_importers.util.ofx_summarize:summarize', - 'bean-download = beancount_reds_importers.util.bean_download:cli', + "console_scripts": [ + "ofx-summarize = beancount_reds_importers.util.ofx_summarize:summarize", + "bean-download = beancount_reds_importers.util.bean_download:cli", ] }, zip_safe=False, classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Financial and Insurance Industry', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Natural Language :: English', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.7', - 'Topic :: Office/Business :: Financial :: Accounting', - 'Topic :: Office/Business :: Financial :: Investment', + "Development Status :: 4 - Beta", + "Intended Audience :: Financial and Insurance Industry", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Topic :: Office/Business :: Financial :: Accounting", + "Topic :: Office/Business :: Financial :: Investment", ], ) From 3d4a6e98bde62cd2726622ae8065f34963090cd3 Mon Sep 17 00:00:00 2001 From: Rane Brown Date: Sat, 23 Mar 2024 14:55:24 -0600 Subject: [PATCH 17/59] feat: format with isort, use pyproject.toml --- .github/workflows/pythonpackage.yml | 3 ++- beancount_reds_importers/example/my-smart.import | 3 ++- beancount_reds_importers/example/my.import | 4 +++- .../importers/ally/tests/ally_test.py | 2 ++ .../importers/amazongc/__init__.py | 3 ++- .../importers/capitalonebank/tests/capitalone_test.py | 2 ++ .../importers/dcu/tests/dcu_csv_test.py | 2 ++ .../etrade/tests/etrade_qfx_brokerage_test.py | 3 ++- .../importers/fidelity/__init__.py | 1 + .../importers/fidelity/fidelity_cma_csv.py | 3 ++- .../importers/schwab/schwab_csv_balances.py | 2 ++ .../importers/schwab/schwab_csv_brokerage.py | 1 + .../importers/schwab/schwab_csv_positions.py | 2 ++ .../schwab_csv_brokerage/schwab_csv_brokerage_test.py | 3 ++- .../schwab_csv_checking/schwab_csv_checking_test.py | 2 ++ .../importers/stanchart/scbbank.py | 6 ++++-- .../importers/stanchart/scbcard.py | 6 ++++-- .../importers/unitedoverseas/tests/uobbank_test.py | 2 ++ .../importers/unitedoverseas/uobbank.py | 6 ++++-- .../importers/unitedoverseas/uobcard.py | 6 ++++-- .../importers/unitedoverseas/uobsrs.py | 4 +++- .../importers/vanguard/__init__.py | 1 + .../importers/vanguard/tests/vanguard_test.py | 2 ++ beancount_reds_importers/importers/workday/__init__.py | 1 + beancount_reds_importers/libreader/csvreader.py | 8 +++++--- beancount_reds_importers/libreader/jsonreader.py | 10 ++++++---- beancount_reds_importers/libreader/ofxreader.py | 8 +++++--- beancount_reds_importers/libreader/reader.py | 2 +- .../last_transaction/last_transaction_date_test.py | 2 ++ .../balance_assertion_date/ofx_date/ofx_date_test.py | 2 ++ .../balance_assertion_date/smart/smart_date_test.py | 2 ++ beancount_reds_importers/libreader/tsvreader.py | 3 ++- beancount_reds_importers/libreader/xlsreader.py | 6 ++++-- .../libreader/xlsx_multitable_reader.py | 8 +++++--- beancount_reds_importers/libreader/xlsxreader.py | 1 + .../libtransactionbuilder/banking.py | 6 +++--- .../libtransactionbuilder/common.py | 3 +-- .../libtransactionbuilder/investments.py | 7 ++++--- .../libtransactionbuilder/paycheck.py | 4 +++- beancount_reds_importers/util/bean_download.py | 6 ++++-- beancount_reds_importers/util/needs_update.py | 8 ++++---- beancount_reds_importers/util/ofx_summarize.py | 7 ++++--- .ruff.toml => pyproject.toml | 6 +++++- setup.py | 2 ++ 44 files changed, 119 insertions(+), 52 deletions(-) rename .ruff.toml => pyproject.toml (64%) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index af768a6..0312cb2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -34,6 +34,7 @@ jobs: - name: Test with pytest run: | pytest - - name: Check format with ruff + - name: Check formatting is applied run: | ruff format --check + isort --profile black --check . diff --git a/beancount_reds_importers/example/my-smart.import b/beancount_reds_importers/example/my-smart.import index 66b62da..2ba0b7b 100644 --- a/beancount_reds_importers/example/my-smart.import +++ b/beancount_reds_importers/example/my-smart.import @@ -4,7 +4,8 @@ import sys from os import path -from smart_importer import apply_hooks, PredictPayees, PredictPostings +from smart_importer import PredictPayees, PredictPostings, apply_hooks + sys.path.insert(0, path.join(path.dirname(__file__))) from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/example/my.import b/beancount_reds_importers/example/my.import index 8cc410e..6eec804 100644 --- a/beancount_reds_importers/example/my.import +++ b/beancount_reds_importers/example/my.import @@ -6,9 +6,11 @@ from os import path sys.path.insert(0, path.join(path.dirname(__file__))) +from fund_info import * + from beancount_reds_importers.importers import vanguard from beancount_reds_importers.importers.schwab import schwab_csv_brokerage -from fund_info import * + # For a better solution for fund_info, see: https://reds-rants.netlify.app/personal-finance/tickers-and-identifiers/ # Setting this variable provides a list of importer instances. diff --git a/beancount_reds_importers/importers/ally/tests/ally_test.py b/beancount_reds_importers/importers/ally/tests/ally_test.py index 84b2bad..8f7e87f 100644 --- a/beancount_reds_importers/importers/ally/tests/ally_test.py +++ b/beancount_reds_importers/importers/ally/tests/ally_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/importers/amazongc/__init__.py b/beancount_reds_importers/importers/amazongc/__init__.py index e1f5d3b..e05e0b6 100644 --- a/beancount_reds_importers/importers/amazongc/__init__.py +++ b/beancount_reds_importers/importers/amazongc/__init__.py @@ -23,9 +23,10 @@ import datetime import itertools import ntpath + from beancount.core import data -from beancount.ingest import importer from beancount.core.number import D +from beancount.ingest import importer # account flow ingest source # ---------------------------------------------------- diff --git a/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py b/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py index 6ca6ac9..a7215c0 100644 --- a/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py +++ b/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import capitalonebank diff --git a/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py b/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py index dbadc22..c05e35c 100644 --- a/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py +++ b/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import dcu diff --git a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py index 1ad8980..656fa32 100644 --- a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py +++ b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py @@ -1,9 +1,10 @@ # flake8: noqa from os import path + from beancount.ingest import regression_pytest as regtest -from beancount_reds_importers.importers import etrade +from beancount_reds_importers.importers import etrade fund_data = [ ("TSM", "874039100", "Taiwan Semiconductor Mfg LTD"), diff --git a/beancount_reds_importers/importers/fidelity/__init__.py b/beancount_reds_importers/importers/fidelity/__init__.py index 3330376..d71ff9c 100644 --- a/beancount_reds_importers/importers/fidelity/__init__.py +++ b/beancount_reds_importers/importers/fidelity/__init__.py @@ -1,6 +1,7 @@ """Fidelity Net Benefits and Fidelity Investments OFX importer.""" import ntpath + from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py index e6abf7b..7380253 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py @@ -1,8 +1,9 @@ """Fidelity CMA/checking csv importer for beancount.""" +import re + from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import banking -import re class Importer(banking.Importer, csvreader.Importer): diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py index 1ea5361..98fdc32 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_balances.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_balances.py @@ -2,7 +2,9 @@ import datetime import re + from beancount.core.number import D + from beancount_reds_importers.libreader import csv_multitable_reader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index 0f91fa9..1c08ec8 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -1,6 +1,7 @@ """Schwab Brokerage .csv importer.""" import re + from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py index 4cb0980..3e7de52 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_positions.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_positions.py @@ -4,7 +4,9 @@ import datetime import re + from beancount.core.number import D + from beancount_reds_importers.libreader import csvreader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py index fa46fea..ea91049 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py @@ -1,9 +1,10 @@ # flake8: noqa from os import path + from beancount.ingest import regression_pytest as regtest -from beancount_reds_importers.importers.schwab import schwab_csv_brokerage +from beancount_reds_importers.importers.schwab import schwab_csv_brokerage fund_data = [ ("SWVXX", "123", "SCHWAB VALUE ADVANTAGE MONEY INV"), diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py index ff7813c..04e3bb7 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py @@ -1,7 +1,9 @@ # flake8: noqa from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers.schwab import schwab_csv_checking diff --git a/beancount_reds_importers/importers/stanchart/scbbank.py b/beancount_reds_importers/importers/stanchart/scbbank.py index 7c0e864..9d344bb 100644 --- a/beancount_reds_importers/importers/stanchart/scbbank.py +++ b/beancount_reds_importers/importers/stanchart/scbbank.py @@ -1,10 +1,12 @@ """SCB Banking .csv importer.""" -from beancount_reds_importers.libreader import csvreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import csvreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(csvreader.Importer, banking.Importer): IMPORTER_NAME = "SCB Banking Account CSV" diff --git a/beancount_reds_importers/importers/stanchart/scbcard.py b/beancount_reds_importers/importers/stanchart/scbcard.py index eb0e053..4dfc4da 100644 --- a/beancount_reds_importers/importers/stanchart/scbcard.py +++ b/beancount_reds_importers/importers/stanchart/scbcard.py @@ -1,10 +1,12 @@ """SCB Credit .csv importer.""" -from beancount_reds_importers.libreader import csvreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import csvreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(csvreader.Importer, banking.Importer): IMPORTER_NAME = "SCB Card CSV" diff --git a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py index 396e36e..f989eb4 100644 --- a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py +++ b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers.unitedoverseas import uobbank diff --git a/beancount_reds_importers/importers/unitedoverseas/uobbank.py b/beancount_reds_importers/importers/unitedoverseas/uobbank.py index cd9bb1c..44fb7e0 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobbank.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobbank.py @@ -1,10 +1,12 @@ """United Overseas Bank, Bank account .csv importer.""" -from beancount_reds_importers.libreader import xlsreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import xlsreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(xlsreader.Importer, banking.Importer): IMPORTER_NAME = __doc__ diff --git a/beancount_reds_importers/importers/unitedoverseas/uobcard.py b/beancount_reds_importers/importers/unitedoverseas/uobcard.py index a195ed3..e1f840a 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobcard.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobcard.py @@ -1,10 +1,12 @@ """SCB Credit .csv importer.""" -from beancount_reds_importers.libreader import xlsreader -from beancount_reds_importers.libtransactionbuilder import banking import re + from beancount.core.number import D +from beancount_reds_importers.libreader import xlsreader +from beancount_reds_importers.libtransactionbuilder import banking + class Importer(xlsreader.Importer, banking.Importer): IMPORTER_NAME = "SCB Card CSV" diff --git a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py index 9793ae4..3e459ef 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py @@ -1,9 +1,11 @@ """UOB SRS importer.""" import re + +from beancount.core.number import D + from beancount_reds_importers.libreader import xlsreader from beancount_reds_importers.libtransactionbuilder import banking -from beancount.core.number import D class Importer(xlsreader.Importer, banking.Importer): diff --git a/beancount_reds_importers/importers/vanguard/__init__.py b/beancount_reds_importers/importers/vanguard/__init__.py index 170e014..76f48e9 100644 --- a/beancount_reds_importers/importers/vanguard/__init__.py +++ b/beancount_reds_importers/importers/vanguard/__init__.py @@ -1,6 +1,7 @@ """Vanguard Brokerage ofx importer.""" import ntpath + from beancount_reds_importers.libreader import ofxreader from beancount_reds_importers.libtransactionbuilder import investments diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py b/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py index 379661b..8473b32 100644 --- a/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import vanguard diff --git a/beancount_reds_importers/importers/workday/__init__.py b/beancount_reds_importers/importers/workday/__init__.py index a86ce64..ff06c3f 100644 --- a/beancount_reds_importers/importers/workday/__init__.py +++ b/beancount_reds_importers/importers/workday/__init__.py @@ -1,6 +1,7 @@ """Workday paycheck importer.""" import datetime + from beancount_reds_importers.libreader import xlsx_multitable_reader from beancount_reds_importers.libtransactionbuilder import paycheck diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 69d8cdb..c6610ad 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -3,12 +3,14 @@ import datetime import re +import sys import traceback -from beancount.ingest import importer -from beancount.core.number import D + import petl as etl +from beancount.core.number import D +from beancount.ingest import importer + from beancount_reds_importers.libreader import reader -import sys # This csv reader uses petl to read a .csv into a table for maniupulation. The output of this reader is a list # of namedtuples corresponding roughly to ofx transactions. The following steps achieve this. When writing diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 7536b46..38ffc8f 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -11,16 +11,18 @@ Until that happens, perhaps this file should be renamed to schwabjsonreader.py. """ +import json + +# import re +import warnings + # import datetime # import ofxparse # from collections import namedtuple from beancount.ingest import importer -from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning -import json -# import re -import warnings +from beancount_reds_importers.libreader import reader warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index f4590ac..45fd204 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -2,12 +2,14 @@ beancount_reds_importers.""" import datetime -import ofxparse +import warnings from collections import namedtuple + +import ofxparse from beancount.ingest import importer -from beancount_reds_importers.libreader import reader from bs4.builder import XMLParsedAsHTMLWarning -import warnings + +from beancount_reds_importers.libreader import reader warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) diff --git a/beancount_reds_importers/libreader/reader.py b/beancount_reds_importers/libreader/reader.py index d45fcbb..7309978 100644 --- a/beancount_reds_importers/libreader/reader.py +++ b/beancount_reds_importers/libreader/reader.py @@ -1,8 +1,8 @@ """Reader module base class for beancount_reds_importers. ofx, csv, etc. readers inherit this.""" import ntpath -from os import path import re +from os import path class Reader: diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py index cadbc9f..fb4b451 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py index fb1b011..b13ac82 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py index 80ab291..a194f50 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py @@ -1,5 +1,7 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import ally diff --git a/beancount_reds_importers/libreader/tsvreader.py b/beancount_reds_importers/libreader/tsvreader.py index 05ba6a8..4f8fad8 100644 --- a/beancount_reds_importers/libreader/tsvreader.py +++ b/beancount_reds_importers/libreader/tsvreader.py @@ -1,8 +1,9 @@ """tsv (tab separated values) importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" -from beancount.ingest import importer import petl as etl +from beancount.ingest import importer + from beancount_reds_importers.libreader import csvreader diff --git a/beancount_reds_importers/libreader/xlsreader.py b/beancount_reds_importers/libreader/xlsreader.py index 4174c37..765357e 100644 --- a/beancount_reds_importers/libreader/xlsreader.py +++ b/beancount_reds_importers/libreader/xlsreader.py @@ -1,11 +1,13 @@ """xlsx importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" -import petl as etl import re -from beancount_reds_importers.libreader import csvreader from os import devnull +import petl as etl + +from beancount_reds_importers.libreader import csvreader + class Importer(csvreader.Importer): FILE_EXTS = ["xls"] diff --git a/beancount_reds_importers/libreader/xlsx_multitable_reader.py b/beancount_reds_importers/libreader/xlsx_multitable_reader.py index 60455d0..d9d12c0 100644 --- a/beancount_reds_importers/libreader/xlsx_multitable_reader.py +++ b/beancount_reds_importers/libreader/xlsx_multitable_reader.py @@ -1,11 +1,13 @@ """xlsx importer module for beancount to be used along with investment/banking/other importer modules in beancount_reds_importers.""" -import petl as etl -from io import StringIO import csv -import openpyxl import warnings +from io import StringIO + +import openpyxl +import petl as etl + from beancount_reds_importers.libreader import csv_multitable_reader # This xlsx reader uses petl to read a .csv with multiple tables into a dictionary of petl tables. The section diff --git a/beancount_reds_importers/libreader/xlsxreader.py b/beancount_reds_importers/libreader/xlsxreader.py index a3f374e..7235766 100644 --- a/beancount_reds_importers/libreader/xlsxreader.py +++ b/beancount_reds_importers/libreader/xlsxreader.py @@ -2,6 +2,7 @@ beancount_reds_importers.""" import petl as etl + from beancount_reds_importers.libreader import xlsreader diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index f505077..abddbca 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -2,11 +2,11 @@ import itertools from collections import namedtuple -from beancount.core import data -from beancount.core import amount + +from beancount.core import amount, data from beancount.ingest import importer -from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder +from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder Balance = namedtuple("Balance", ["date", "amount", "currency"]) diff --git a/beancount_reds_importers/libtransactionbuilder/common.py b/beancount_reds_importers/libtransactionbuilder/common.py index 45bade3..c348bb2 100644 --- a/beancount_reds_importers/libtransactionbuilder/common.py +++ b/beancount_reds_importers/libtransactionbuilder/common.py @@ -2,9 +2,8 @@ from beancount.core import data from beancount.core.amount import Amount +from beancount.core.number import D, Decimal from beancount.core.position import Cost -from beancount.core.number import Decimal -from beancount.core.number import D class PriceCostBothZeroException(Exception): diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 22321dd..7dca20d 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -3,10 +3,11 @@ import itertools import sys -from beancount.core import data -from beancount.core import amount -from beancount.ingest import importer + +from beancount.core import amount, data from beancount.core.position import CostSpec +from beancount.ingest import importer + from beancount_reds_importers.libtransactionbuilder import common, transactionbuilder diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index 63c4232..b487b07 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -1,9 +1,11 @@ """Generic banking ofx importer for beancount.""" +from collections import defaultdict + from beancount.core import data from beancount.core.number import D + from beancount_reds_importers.libtransactionbuilder import banking -from collections import defaultdict # paychecks are typically transaction with many (10-40) postings including several each of income, taxes, # pre-tax and post-tax deductions, transfers, reimbursements, etc. This importer enables importing a single diff --git a/beancount_reds_importers/util/bean_download.py b/beancount_reds_importers/util/bean_download.py index fd4d8fc..fb05eb8 100755 --- a/beancount_reds_importers/util/bean_download.py +++ b/beancount_reds_importers/util/bean_download.py @@ -2,13 +2,15 @@ """Download account statements automatically when possible, or display a reminder of how to download them. Multi-threaded.""" -from click_aliases import ClickAliasedGroup import asyncio -import click import configparser import os + +import click import tabulate import tqdm +from click_aliases import ClickAliasedGroup + import beancount_reds_importers.util.needs_update as needs_update diff --git a/beancount_reds_importers/util/needs_update.py b/beancount_reds_importers/util/needs_update.py index 0d1c9e5..f602dac 100755 --- a/beancount_reds_importers/util/needs_update.py +++ b/beancount_reds_importers/util/needs_update.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 """Determine the list of accounts needing updates based on the last balance entry.""" -import click +import ast import re +from datetime import datetime + +import click import tabulate from beancount import loader from beancount.core import getters from beancount.core.data import Balance, Close, Custom -from datetime import datetime -import ast - tbl_options = {"tablefmt": "simple"} diff --git a/beancount_reds_importers/util/ofx_summarize.py b/beancount_reds_importers/util/ofx_summarize.py index 541465b..1bde0ad 100755 --- a/beancount_reds_importers/util/ofx_summarize.py +++ b/beancount_reds_importers/util/ofx_summarize.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 """Quick and dirty way to summarize a .ofx file and peek inside it.""" -import click import os import sys +import warnings from collections import defaultdict -from ofxparse import OfxParser + +import click from bs4.builder import XMLParsedAsHTMLWarning -import warnings +from ofxparse import OfxParser warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) diff --git a/.ruff.toml b/pyproject.toml similarity index 64% rename from .ruff.toml rename to pyproject.toml index 03403de..d2d873b 100644 --- a/.ruff.toml +++ b/pyproject.toml @@ -1,7 +1,11 @@ +[tool.ruff] line-length = 88 -[format] +[tool.ruff.format] docstring-code-format = true indent-style = "space" line-ending = "lf" quote-style = "double" + +[tool.isort] +profile = "black" diff --git a/setup.py b/setup.py index 0926e9f..217e140 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from os import path + from setuptools import find_packages, setup with open(path.join(path.dirname(__file__), "README.md")) as readme: @@ -21,6 +22,7 @@ extras_require={ "dev": [ "ruff", + "isort", ] }, install_requires=[ From b5f2855d49438d7b6da67df77b5336d19a1689ae Mon Sep 17 00:00:00 2001 From: Rane Brown Date: Sat, 23 Mar 2024 15:08:03 -0600 Subject: [PATCH 18/59] docs: include setup and formatting steps --- CONTRIBUTING.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a33af04..2cdb81e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,5 @@ +# Contributing + Contributions welcome. Preferably: - include a test file. I realize this is sometimes a pain to create, but there is no way for me to test external contributions without test files @@ -18,3 +20,30 @@ Contributions welcome. Preferably:          ├── History_for_Account_X8YYYYYYY.csv          └── run_test.bash ``` + +## Setup + +Development setup would typically look something like this: + +```bash +# clone repo, cd to repo + +# create virtual environment +python3 -m venv venv + +# activate virtual environment +source venv/bin/activate + +# install dependencies +pip install -e .[dev] +``` + +## Formatting + +Prior to finalizing a pull request make sure to run the formatting tools and +commit any resulting changes. + +```bash +ruff format +isort --profile black . +``` From e94b3f1e3b37cb52ce83d37bd421005be1a40470 Mon Sep 17 00:00:00 2001 From: William Davies <36384318+william-davies@users.noreply.github.com> Date: Mon, 25 Mar 2024 20:37:47 +0000 Subject: [PATCH 19/59] docs: fix typo --- beancount_reds_importers/libreader/csvreader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index c6610ad..414d54c 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -12,7 +12,7 @@ from beancount_reds_importers.libreader import reader -# This csv reader uses petl to read a .csv into a table for maniupulation. The output of this reader is a list +# This csv reader uses petl to read a .csv into a table for manipulation. The output of this reader is a list # of namedtuples corresponding roughly to ofx transactions. The following steps achieve this. When writing # your own importer, you only should need to: # - override prepare_table() From b93a854435538e0a3bc298edee68f07987a2c9cc Mon Sep 17 00:00:00 2001 From: Ammon Sarver Date: Sat, 30 Mar 2024 17:32:02 -0600 Subject: [PATCH 20/59] feat: add genericpdf paycheck importer --- .gitignore | 1 + .../importers/bamboohr/__init__.py | 32 ++-- .../importers/genericpdf/__init__.py | 71 +++++++++ .../genericpdf/tests/genericpdf_test.py | 33 ++++ .../genericpdf/tests/paystub.sample.pdf | Bin 0 -> 49181 bytes .../tests/paystub.sample.pdf.extract | 11 ++ .../tests/paystub.sample.pdf.file_account | 1 + .../tests/paystub.sample.pdf.file_date | 1 + .../tests/paystub.sample.pdf.file_name | 1 + .../libreader/pdfreader.py | 144 +++++++++--------- requirements.txt | 21 +-- 11 files changed, 218 insertions(+), 98 deletions(-) create mode 100644 beancount_reds_importers/importers/genericpdf/__init__.py create mode 100644 beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py create mode 100644 beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf create mode 100644 beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.extract create mode 100644 beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_account create mode 100644 beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_date create mode 100644 beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_name diff --git a/.gitignore b/.gitignore index ce0145f..f72a6a7 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.debug-* # Translations *.mo diff --git a/beancount_reds_importers/importers/bamboohr/__init__.py b/beancount_reds_importers/importers/bamboohr/__init__.py index 2818190..6d05acc 100644 --- a/beancount_reds_importers/importers/bamboohr/__init__.py +++ b/beancount_reds_importers/importers/bamboohr/__init__.py @@ -7,7 +7,7 @@ # BambooHR exports paycheck stubs to pdf, with multiple tables across multiple pages. # Call this importer with a config that looks like: -# +# # bamboohr.Importer({"desc":"Paycheck (My Company)", # "main_account":"Income:Employment", # "paycheck_template": {}, # See beancount_reds_importers/libtransactionbuilder/paycheck.py for sample template @@ -15,25 +15,25 @@ # }), # + class Importer(paycheck.Importer, pdfreader.Importer): - IMPORTER_NAME = 'BambooHR Paycheck' + IMPORTER_NAME = "BambooHR Paycheck" def custom_init(self): self.max_rounding_error = 0.04 - self.filename_pattern_def = 'PayStub.*\.pdf' - self.pdf_table_extraction_settings = {"join_tolerance":4, "snap_tolerance": 4} + self.filename_pattern_def = r"PayStub.*\.pdf" + self.pdf_table_extraction_settings = {"join_tolerance": 4, "snap_tolerance": 4} self.pdf_table_extraction_crop = (0, 40, 0, 0) self.debug = False - self.funds_db_txt = 'funds_by_ticker' self.header_map = { - 'Deduction Type': 'description', - 'Pay Type': 'description', - 'Paycheck Total': 'amount', - 'Tax Type': 'description' + "Deduction Type": "description", + "Pay Type": "description", + "Paycheck Total": "amount", + "Tax Type": "description", } - self.currency_fields = ['ytd_total', 'amount'] + self.currency_fields = ["ytd_total", "amount"] def paycheck_date(self, input_file): if not self.file_read_done: @@ -45,18 +45,18 @@ def prepare_tables(self): def valid_header(label): if label in self.header_map: return self.header_map[header] - - label = label.lower().replace(' ', '_') - return re.sub(r'20\d{2}', 'ytd', label) - for section, table in self.alltables.items(): + label = label.lower().replace(" ", "_") + return re.sub(r"20\d{2}", "ytd", label) + + for section, table in self.alltables.items(): # rename columns for header in table.header(): - table = table.rename(header,valid_header(header)) + table = table.rename(header, valid_header(header)) # convert columns table = self.convert_columns(table) self.alltables[section] = table def build_metadata(self, file, metatype=None, data={}): - return {'filing_account': self.config['main_account']} \ No newline at end of file + return {"filing_account": self.config["main_account"]} diff --git a/beancount_reds_importers/importers/genericpdf/__init__.py b/beancount_reds_importers/importers/genericpdf/__init__.py new file mode 100644 index 0000000..b5f7595 --- /dev/null +++ b/beancount_reds_importers/importers/genericpdf/__init__.py @@ -0,0 +1,71 @@ +"""Generic pdf paycheck importer""" + +import datetime +from beancount_reds_importers.libreader import pdfreader +from beancount_reds_importers.libtransactionbuilder import paycheck + +# Generic pdf paystub importer. Use this to build your own pdf paystub importer. +# Call this importer with a config that looks like: +# +# genericpdf.Importer({"desc":"Paycheck (My Company)", +# "main_account":"Income:Employment", +# "paycheck_template": {}, # See beancount_reds_importers/libtransactionbuilder/paycheck.py for sample template +# "currency": "PENNIES", +# }), +# + + +class Importer(paycheck.Importer, pdfreader.Importer): + IMPORTER_NAME = "Generic PDF Paycheck" + + def custom_init(self): + self.max_rounding_error = 0.04 + self.filename_pattern_def = r"paystub.*\.pdf" + self.pdf_table_extraction_settings = {"join_tolerance": 4, "snap_tolerance": 4} + self.pdf_table_extraction_crop = (0, 0, 0, 0) + self.pdf_table_title_height = 0 + # Set this true as you play with the extraction settings and crop to view images of what the pdf parser detects + self.debug = True + + self.header_map = { + "CURRENT": "amount", + "CURRENT PAY": "amount", + "PAY DESCRIPTION": "description", + "DEDUCTIONS": "description", + "TAX TYPE": "description", + "TOTAL NET PAY": "description", + "YTD": "ytd", + "YTD PAY": "ytd", + } + + self.currency_fields = ["ytd", "amount"] + self.date_format = "%m/%d/%Y" + + def paycheck_date(self, input_file): + if not self.file_read_done: + self.read_file(input_file) + *_, d = self.alltables["table_1"].header() + self.date = datetime.datetime.strptime(d, self.date_format) + return self.date.date() + + def prepare_tables(self): + def valid_header(label): + if label in self.header_map: + return self.header_map[header] + + return label.lower().replace(" ", "_") + + for section, table in self.alltables.items(): + # rename columns + for header in table.header(): + if section == "table_6" and header == "": + table = table.rename(header, "amount") + else: + table = table.rename(header, valid_header(header)) + # convert columns + table = self.convert_columns(table) + + self.alltables[section] = table + + def build_metadata(self, file, metatype=None, data={}): + return {"filing_account": self.config["main_account"]} diff --git a/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py b/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py new file mode 100644 index 0000000..9c64531 --- /dev/null +++ b/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py @@ -0,0 +1,33 @@ +from os import path +from beancount.ingest import regression_pytest as regtest +from beancount_reds_importers.importers import genericpdf + + +@regtest.with_importer( + genericpdf.Importer( + { + "desc": "Paycheck", + "main_account": "Income:Salary:FakeCompany", + "paycheck_template": { + "table_4": { + "Bonus": "Income:Bonus:FakeCompany", + "Overtime": "Income:Overtime:FakeCompany", + "Regular": "Income:Salary:FakeCompany", + }, + "table_5": { + "Federal MED/EE": "Expenses:Taxes:Medicare", + "Federal OASDI/EE": "Expenses:Taxes:SocialSecurity", + "Federal Withholding": "Expenses:Taxes:FederalIncome", + "State Withholding": "Expenses:Taxes:StateIncome", + }, + "table_6": { + "CURRENT": "Assets:Checking:ABCBank" + } + }, + "currency": "USD", + } + ) +) +@regtest.with_testdir(path.dirname(__file__)) +class TestGenericPDF(regtest.ImporterTestBase): + pass diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf b/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5890f3a0010f802e2c69729192e18a526872e847 GIT binary patch literal 49181 zcmdqJ1yCi+(k_boU^BQ5?(Xgk?(VX1cXtMNcNpAZaA$BC+#LpYce}7>?{oI~&-wrN z;=Q;LH||<1x~r?QvNE%~YjtLQ*~GF!BGmLWj4;GK+ZWr1g_qgmJ$*2Y_;mQz`era( zT==virWOtWySKfCo&!J#U|?+sz^9c2SQ$H*;4`wbv*Gjbz}P$30rV_kT!7m&G!(F# zP`t8qa<~9R-b+i0~%5L_cU^21>*T*3GWS&kh5~qhC!IGMCe6^k# znlc-iPG652pK8JXN{;DS)>jE7+DffEUz40lpq+^GOv3pyf~b~Xh~XYxon^k5TU<0u zaxhoWpf(Ksy#&3?XOi*~j-A0cm7PItH?RU(w`)EEZ!j65%*}TS*NadtQ#;>eQrh6; zLOwSS@}4c)1bRn`nK)9~zE5X)$N8eVFnOY@aeHqZ`V46Ulet_0nTC&FxC19^k_03p zxhx0L+69_ew2S3pQfmVQOUD^iZ#{U>8zMDp_hwYYjkv4=03=3S2a3Z0_4@xWr7`VQO_H{uJ#8!Pcb})3{HhoBoC#c~>jUpuEym6z6o7n4N;o6rb zSH{dURt}|FBj1$!qPR9><BgNCD}GNb7r ziy=tdN#@3_xVa}!&A=|nO=)J-d1{TOo4qf|Y{+Y>#q7U_72sU`OqOZcApfP>lQgl= zSd?^W=H*;Hy+S+e^0>EszCtGhx3d0_yZYI&dNs8xl)R*+Bqg>(1UOF4uu8!AQDb6? zRzb{&2S3j*4fnjl8kHU?z~<7;lSEp%5>7oP8+>IFx(kv?WrbSOk(Dwy0BnX|O;UgW zl0=U>68;u-Nv9(ONJD!Rxhfbij_=dWSB8iT6FE9YJNqjd9Pi`!hvwMTPoJPOHGhif zLIT$y??v;i1Ihvd>jIv`E_36q@0%Lg*cW3@6p$73o4+A*K||%s*`_T3591g3mY=V0 zKdf8-GE~^EpZiP~g)pzpTngF~(_FEw@6JpMhaPB_PM5CHiJ_ftNh2MBZ|2()BZkcE zXc6lz{aG3l38@U}Vqi+Jy2H!Xfgu-hsjgAHbv-;RbrL#WH;^y9hs1Q{yP%1WbDx## zgPpzpN(>lcke_sS6rLV@XUh_HnT-0AnCiSvR#_IF<(PzYby`4w5O|@#8ip!PnT$~v zh_hh}6Q7>)cZf_k6e2l(=MT&ZI0ukmKUE=@W_WvUuJ`l4_Q0F(c{bpDCF&`$#^+Og zp<10eKi?^w8T|3Ug%G3_$s`5(S$?>I!;zFjZ7Dc-q?=knU_=5 zGVM3z>fOV2n`qsh z5Y*E5F)Tuu>LH?9>jRC=1s&GRRNuY^QX(wY7xMddtDe0g=Y}H%_9q){ULIS(|I+&* z?Gel(T@_bX2plArn=BafQRDG^$49N4TvPNjYYNxTb1d}?Tb6`=PNiEWWYhefTH=ZG z;gQ}2*U!Hi>P!u=AMIgA#|f(oHk2b`7v;iuHePER9Qo*b@sXxV%l@9iNAMN@u zhWOFl(zdX~lT2eD$$+#sM$-2A;dZ!mn3lC6OTftzVfU*EAe;QMOk z#U2dyWgY786@&T~gNt8#ekunA8CRX8t68N$L|LJg+J~M%sF{AUicw@$|BBaxgo6#F=q%_*oN(uGd@Ay5LW^G4SXBLa(di#ag(#EGjhuvYI->-_U%{xm)+_r@I zqNx;mlVVuT7>?wrXM6gkfQh7KFIv>YOTB~UrRR!AFu%{I9B{%LEw53A%l0#+<`W|> zQi==+p1KAYEhiMS(B5Ly1<0^+Y@h>lel)x0AepfsUHG%BGoxP?%5B|WFCB8KUdgcP ztafxfpWWXpBj^x~9l=wP^It27a>pqM`3r z!@sLwKUjeB@($uQT%1%REm^Hs3 z#svzLFeH3f3l_9sQ?}zWM3_Xgi-$H8auov0;p@LP_g#%~N$KbJntMphIwXOk@QW}T zxj`GBS!HDL??}}hyC@8>5ZOK<6v3G8sA8n6n>HSQ@A@)#5WEHxP)`I~C<4oZj*+_M zQ%wZx|4dQ~yZDQ~g$k}ye@ib;Y3SV+o z>PQ(1A7V04c-KcCdRJXI0~~jMQq|AtD4V`%cbNBjb5& z>JX~bnh~O!ZiqB43Xuj=3?+8`Ckx9@)nRE<$53qjQp`Av08~~DEia26VL{>4ZD|X7 z3-=+~?SiRj(-RE;mWg7^J-;bL23DOPrVexUB7q9HY~Y2T4RPV$w_f5(9D!>&-ll23 zxej3H3kV3rh24lWLEqsjN}nu~x9+jk@&k_|b0dkE)oJqmT>$hg!}X;Dy7rDnSx50S zEzJ`*0^~+C$9}^(d_)E5=xioru;$Z8Gz5I45{0vYjGdY=aAZ%M4qtMxE@Cpu`b4t= zg@&*+yh=Vp;?J6V#rotZzJVgPCf?e@2nMH+r>~`A zFcf50f^=v3lj96ENxZG_jz$m9#bp}Qf2qZUQ}4(3(|cUQ?4Ao`4W7#l)^Ma_(jTIb zCy)&%-TmH6GGY6e5Scfq5RgXv15$<|JSC>?wRQ7TV0mV6hr420;^f`@8qg<9Y;c|W ze9ZkX@+HP~UIk|P64}>2AH>xPa6~#MKgg2g_{mekZSw&6qEVyEG(8PCBFC z&Ffr>cgl&S6E6lYv3Dg}#z>Tb&B)fKR}`jzvrc?WN(NKcZ_|3}f8J_)T)zi0e_E2c ziSE&}9c|@8XWAB476kEi=yQOZy9|ik<~qiz_7GXCQ*C~A9vl_(AMe8>2@G(tF-0oDYS z9_U#WMs|Bn5J#-{0dNy;2!XNty|&|4RgJ-R`U;{4gZlNAJkBL6LRSf57m%akLbK6X46at0;uaRN zZH0ypo%CqllWdrMcoL}!zK{x$yo5IiEC~MX3vhyMX_#kZt^C`+jOnQ+w|#sVxDR3F z=LmreLYUxPzHq{#=WcKWL;xAo@U*MVTh!DDnRChPecGX8tU{Bx&+AsC1;JCsLhvuN z14|=^IK;Q;u0k8kH) zEl};b&MY)sV-!zkw3fO+pq}9vBBi<6|otchPSG|P?+D3 z)Umnq8ZV!RkYok>Dy+IVJIvdoD|#gbFg_PPL2w57oKyL|?y5G$9oqSG9ZGs@>AP%`39255+#r1;8YSBz59u(YD>8cp>qL=k5&WRs%b!7bJU?QG zQVYX$)5)lnTHP}toDJ;$!|}HK@SDenl!PkYyWA`->PNKp2xJDU9kk)2heforj^;z$ zY->&%4|FRS)(WaC=}|jXVg^O=2DP*6{FG)?+aST6x0a}z{>DX zNBynz?%vb?&COSIwR!XFW%Z2z*xCWC9Pk<5n&t6nZ>ReYmnvZ3Ft>rlJ2s7JRM0G=1;;m!`iB{3myOS_Maa zhd)gIhpzavDyD{S0Tt#qv1z{mOpQ$(@LAuAVsDnVv$1{;z2H08IRf6){HFna4@RpX zrDt!BPyeT40;Uf3vH&|lYfBqztM{|W_RnYy%bT?SDV_uK_Ylr;Dt8I91j^S_6Ko$r zI(rC+Q9kUh3}1LYIu<%+dL({C{*T~Da5TZb*$@yE5D*VtenGST{kVKOoxL8nUum9iyDPn}dOjJRd!k#g&sJ5;sm?&LO1(r3c)} zZY zKDuG1xMgA{I&FU;s~rj0r@Jv;S0k&aV8_#T4~z4;k31T@ zo>!hd5S9-@)HNkpby{M!cXCkFK0Y^YXEN|@SwGAku0>Q7UNb#NeBFh2_)|)AH!Tir zt)ic?={p|1PdZ-LT4-qrUQW;Q5$ZhB??hp@x_Pgl*j1luG+jI@2e^U@Gq(hv5WM|E z*9PnEY_|R7+lG(Z9xn)fdGqU@E>6c)&0B@Cv~Xd+)Oab^nY@UoNPW_AgZGt+ z39(0kR2|!R>M1e6h{9-fJDfZy*0|Iw8dI1d7Z2|Ee*D4p1;gt;hqg=8>kdaO>8sJ8 z$|leE60+7ouGhU!#b)vu@-^Hx@%?~TSqzu@FJtseN!DbB|p`Jzady%&UHTzg&m^TdHCJc6n4vfhTJH@ z$V_dTm?>`5eu4tI((*JRuIT*v;Ts&LMcukd8`LdW2_%dA7*>q36Unu11-Y7~y5J z^lY|F-)Nq8yNN>c5Zqo>rm_23f7`)FeO4-^Vk~ZcRv|u#K9ZC&fze`HjhC&=A4^MV zH`A2?_@RPVcLL-930}1pJ+-(tW{)dnIOowJ;NodRbu1dG0{2L8N!90pS)Sz6$joAE z&=4N`u%+}xG!du3ApHQ&3Yj+*u2bBFd$-%1)ig86pk4({pK-a9c!eC(6xfhH&Gsp* z3bjDI6qIwLPFXsHhkk*Uujk6$M2mM$%h$ z@v)co3LrZ87a;_shF}Hp(0&?-AQbqLUxSXz@$;S%7);I5kE#b#tGA{^p&CgpOHl4^ z8k1Rgl>fz+bAV9Uu91LTWu)C^YOr`nXdgiTQzOowq5>u_kqWb0D-GL_U`$QkFn=#~ z&U-N|=O%e>w_Qoc?nbA-goz;3xUD25T$uU08pF?|uNg2Z7+N(tQ;n%Y)GAHoa5d$z z7Kx^5cFxm$TjP(W0yg_1DtA}w3P{;M9{u@YO~&(vKF2TVVZ<_<^j4MKhuYT?LkwxE z$JbSGeD)TbtwP?1vR3=mg;0P0Bbg?d;wxN!#fB`BJ&((w7|UbVVMxhnrU4Zu600>Z zzhg0K@Jpq4%Qw$ZQ$9QzxI(%!}fNbm!cH<-T zyF@Jc_hm1)mbRrhWsmN&G9v-{)((L(2s|jj(k}&fGD|^Q=Oj5s3=m%$M#pSSUrXKM zNX_86Kr^0{+-lDpfO!fL3+$TX82K;tNeWY{kYkyNdpZRAJkSvx+;+Elh+_xXEVVy& zvWv+NGnd)~H;@qITmrdfg?8A5ePn3E^qT(_`pIEBSr9K0xQkrt!zYuodTT5m$MDjN z;axRe=8woN6DT~~cZy&voRO$SQ6lX2VnLtSA_*A2f&XOPrFh)$H*<0%_`vB9CoqkQ z|LP|8=ErUrs^c7dR`%4lRXV(m!V2XwresLz(+AYhYQ`E1%ayGrc;G?+eCOqbyNPB^ zGz>nZLYI={o1v^09crKN77I2BXFn7xAO*sm@~A_n6nr~~PVuZ$#4ppYvMzOZvuC?~ z1`9}q4q4fQ$-?!Z07=6Ug~f+24Vc*SzzkvVywGe~Oi79@Duy1~$GIT#F0Zsg^3FJR zL*XArsV?jID9l4O>m&IU?h-TNGmDZydcfaCiLTsQO&k3e{YDPXgOR%gi%%psQCpv@m+k}KfZ5#X_SyN9|f zS(`b+x51-Gsupb1Yr9d$tHXJInaUEMUCpqrV}&VEw>rZ;3%pDrCiQ0N=tf8G<3ke1 zWg2$Sb_Hi)%5B4?tF^+YqjlyzR5CI|VOFXITI$#tv4txs=H=~m6ML5D&oGaCm%1#L zk@}W6wXCd6A9ZY#yjcR4)gVFKw78JoJuQ>fZBr*>s(Ms;n%-&DpJ4|4i4FJ56-FC5 zX;i_F4eZsmLGM(cSmMiO-RrF}97{yO^0|((u?5l;rj?Rw&c|-Pw-( zn@xSOuG20f6c*0nqeK`T1$U($DAJYtB2{iLh2&H(=@AaEs4~+PAI?l@wx<8 za(Wefb-WN!nQ1c4lzbCHy3VJWN}XJKQv-~jiEZ_%YMD@Au;xvIY1EGy$IU=m5$kx&dGeV55r%=M$d?f5`t@wRuj|}J(WFh1<+t?80uxB+CHc&V^m1=GIh6^{p#?H-Q zHhSP`!|k*Oh3eC?injfGbWHbV5swST_64EXnOI|Lwp&+bjD%?2vETmS*q1==&pD-c z3LNIdYeJ>RoLDCGwF54>?svud(0cIVW-f_zm3Vxa@Z*MmWi1$ug~xE7CdFA^14a~O^mKOYpbA1il=Vl?ZLlTW;889yv+PR$ZYf^>nXAKy zih>vJd&#$(pnv~>IPB*>+hU39NtYRO=itEtgdmga9@>R509QA^CS%t?lG>zrr?k^Z z(2dB-qX};?k2EAG2T^!qU)>)&p8j(qJ$(%fhgPShL!I+fxBZCtAS;S#_H zFdJe{PQVbVSPbj+#CWfIc1o)el<3Qp4O)-R#y1+z9m8oHg28EhVrpn z`0}mT^h4ad%4Bm%Y6im>x;+Q}3OTDw^CXaMhNfY%U(BVND%2?g1)wo-qvCeW_=Tw< zgLFEn?dj5klDU$k3uQ`H3Z=_TzmR1J1!5WmRb=k>L09%l5VlHX@XzqiRd$M^ zr8jHKrIiXV)9biXJ+BwII;8w#OGcfvd{O}olM&Azc8pe`(0(Ej_aTu%-W9X_vj^ST zrQmAvX2HpWx9!=^BPFy=C$Nm@6=2HFGG3G#N;xa>cpeO@F?#&Gd~g3e%scZZ!5y)? zeDKF#vy_gf?oDZz_e#i@-wd>F`zy-mz(mQjQ8puzm4chQB2}?e8iAUcDfqdK)VJVq%XlmsRNLqr<+qIGgu1F8alo;OEsyUDteYsR6B4kb3 zz2{1NVeQFFEIl?EFnleg56T-n9(NLb+79riz;x9a&};TD9efwUOsQ@+OXN+6N~cZD za*jiR6t~dXAGnGH(vR6L3+&YCq!^o)ZtMHS{7-<2=0?okp#-B}L6{uGhb$@B3^8$3 zIo+G---cfiwakO19Wk*Xj?uvEajBGwXDMZMy1+}jW9{X_6Cln~(&2@u2on+<>eyNA zCgV~>2IJCwNrO9n9RPuaJ`o7Q+nQb~C^to;ukDp^-2M#uoV^-l;IuLCjSfZA118xC zrX2wkrF!jqKO@_y{gDp%qYDybU{eY zs-gD~jJwxjujcW*AD-p|TR8rp1b{pRCTVnDd84}kgeq3v`Saicb8lUGr|ui$QNH#s z>8lumymRBqnR${r&G`yV0*?_THsFAw)`{w%E!z{%7AAZn-Q3iEHCFFx)6;Mp=T z(EUfp_Fu3G$5YiPK6FrgKOz_EVEtoJuDDt0h&A0py`u-*T2nwc4MwO`o}o2gUbZBv zo(GraqM!}(og49L!!CfOZ5;{jAqjZ2f^}(83Qzrd~^YR|s z`0Kn#zXfV&`QK3-R_}3yKg}#J`W9gM6UV{u9<=#Sk%50PU`C>|qQU}d&{!L5+c+`- zKJgW%8#MkT=NA+(xezfQS&***Bu+x$BA<}Rl?@d;fklG|41_eewhAa!>{KVaiecbaMDZ6ZzwC&n*Z# zA~3uvaOZ}R5t!(m5XkgX8+S@d(U-oOn-dxz!XnGJ7r7E(wIuMG1~CXPBlo`LMLjbe}k*4#jXgrG!L zylx<#hZ>0VFazzaCgm}e_E8LOFS`$Tu$rM(85Kh*8(hlfFMUh zI%LBz`O?sVQ0aca_F=REUBw3q@cDuWng~ks1;msO>^nctJP-^&{XDRfznKltw6EJZ zn6w{@4TL6`O6La~SU4Y1KA5jRBVy5tQY=B6PrPhSKtX$c7xpXM%VJ_6$Hu2AR_PFb`Dkvrxr} z4JFz&ye{N|k`9U1xwIkPh{^-s(1o^vbp}KgsM8e(#Ww&2Iw7J8g>^0@5UolCjP%(& zh9D25SU5ErJ`bxY;!BL}7AQnej$Sf4Y9GdCkXrbLzOvq|zBt`XvO%)4B-##(5hA1S zy>Zembgdg^2`W~O#VZ5=uao3{o`Z`FXq1D06{SZDU;uBdADI#J* z?Ls{VAayax`4-7I5PU^+?a|aHua2DOZxzEPk&gQ$_LT@to`^i9)>l!?j6jQoo22b? z+#rXcs2#c*pSDZ|Q5L=qF+wb_Y&;psCmC^UVSxo{cJU@YS3Xz4eEAPWR@15_$;}z+ zGHpVh;-0jkag0fqNt=V^aiuZl35QAU#4_=7NzO?+G+CrNpS{Ij3Z5E4Cfml|#@wzMl85M`)4gMq$H)(l3vUUJx@6jE z8rvLOVyQIMer+XBxTmn9@JJv?FiHSUh$@j(vYz=glTl(>A~Q{AMq;LCwqW)+LsJ?t zRygLIIhLWr^t$yFbWqhC;@0?JcHMozeGrG4irJ2dfvJR<#MH-(k!qDXn7W<1%hX?M zq7GXJroOGdTc=HzkaF@<(TKPvj6J8k;<%7{f+ugKl)K!Zs7BQ|SGs2OqgE`91}lGq zW`lyWfQ$sKDy>eX+-VMvTvmBjUWaIhhWAkCIR2t;7=PN2qMXp24!umh+zkW5l9^Bofsrc&mX{%=*wYHDR7 zv)Sz3Tmv4tu4(s_u-sv)VHv0^s5(@~R0`CGRCsFSYT&AtwFgxX-HSb`=2`}x{a?*C z4QfUUR(9&ji>cR~I*vU#Q?;sAP+O#1#%|zm2p&lu=^?p;T!Px*Z|5DH$XWtCXOIr} zHX29I!#5#G-ASvb#0u9k)JDliKaA$W!l9j^O;e-WP1tTUV%x>g7}A96xY*=OD=j5# zsvkc5l$wj!b?UL~S()cvcy4^8M36v;5e@5)FO1v89D~&kPO4OiQK@W*z12T8J1ykl z;0fW0;Q83@)L!aId#Cpd^6d1mayfUl`!M$?1I7kc2E7N92POih49Nq_0F4ap2-f`r z_(zK$9_^$Kk>)fih`^JeQy>>KGn^L6gFQM86;m0>k!z!piCm#Bfs z0nxAsR0&jPkwMX3;TcgWVRzvqVTmMFDz%y|k#O~J`~q%NJS-GcId6l_#kB-I#_sk} z?nazz-2t9rz>)&K2TTYh^=O3{h*gN8h;4$%@Yk(7e%HD1-71ccj?X1$ES@MHF%77_ ztDO=c)(fqo)MDf&ZKLgG%B2%*9x|OdQTq6eZz4vL%wEl?B{}j%^7>0SQ5vzo8f=Z( z_f=z2K-Q4lPo$lzt<){6VVuFs!ACDjuMSvVUO8IAy8tyzVxPRnJ>RkiDy)ST}02{Z}FZM0F3YQS6cTA)}QuNc*o zn=K^~Xir@05C01NB}2R%d$^|NsdB&a^CsZ9E&nF}(Rtko%6WNZzxh%%MPE&OVZ35p zt9<+C#kY%w7L}uRWjzgBT?Er8!>C$@qVH|um1d$dg$H>DzL-&|J8cK<4aXUa{Y!Hl zYnI+qz!9LmkUY4buv@(cZe`8O`7L|(mN&|}#}2<8Gv#(te>smC6Acv2@9&Co zoZ+0FV=ZEhXX%QF?W^9j?YDkjzZP`=Sh_FEnA>y*#`c;0{8#@S-f1Qr*Me)PXURV4 zbJk7$xO%91&g}ARNPQlQsk@~6+P=?J2F@rBHE-Ih$5r@EN@~rJvBX%+C`x)7dzQ1; zg+O26c~}Tq5QmBbPjl_oWJX0*#p(o8L%bWGFIAc*~z>?&ge&+pdnX=XA$vf{mlTHVV?dCfrxb zbc2r-k5P)$iX4cRi;jighp)I*U-h01hbK*K26m9Y)ZZpdlxBH3Keid`5B8jyPaLN3 zimw*6vp>wHC`1`*Z2rdY_n}pn#q|z!0DQ_d?u1R?X=DzHBCq|F#VFZx65UKEgkq zUg_!on{};!d3p_0(lEgWeBw!K+9HPMG0{*r334-4i=``8j6}$gJRfPidx*l zr$MUMBi3Es*CvnMbc}B=hs~#)o)mJ!5q|p2thYzYa&@ne2ZVtTP)Sw+UKR2w`bTaR zgRgCK<-_rMgZZE9XIdII;uKj0LDhGwTir}*gBdVmxXkbPg{EIIw{l4dvgi+&=gO2W zJ9&XAS89A5yeAw+I$5UmvuTSGI?d;$Ic6hn7W%%b44c}Qc^r3QLZwZ`-Hj3Ks2sTY zp8I#=VLse3o#CHF;kef?_c}lA=`v584%eqKHVLh&n1clwHD4r^-+*oEeQU1?Q$EI6 zclY%+b&K2Z|z{lS;usCqIt(#3Yhv)*u2qzL-e zbEBn-am22#0j->?A6iHMrqQbrq!rpFYyZ;%?NKlaZ%NgN-1>my&=G7?7+$ho`l;3W zaEwyg-oaXkA7<8y7}2rCe2{o&uo{Kx53pgRsDQ{2_d~h$8vd4{CSEv?5eMUB!TERq zH#)7;Io*hn1H)jhK_k}IB<}z#Q5pyR1$n17=omao&W1j?DtzsZuUFw`@k7O5&q6x+ z9GS@6SS%RTfhg%U87J*raPO)IP4M0 zt?Vsc^^ANY;qj}}Nk_&*6YZs!?QY0bEsUI5M{dxHKd`X`L14 z5t$*}nasm1g4cuoeAcCqi)`pI!)I!Qxbdj@b`vrGFy zlPy6%{@zr8bc4wc(ue~P7nw)-X|AljU0~ADR9L<5In;sqSmr^gIdmrtRwON(EE?i~WmI$6Ck6$1;{`?vQ&38&Vqz8|;0 zgdRj5zJ)UQTF9g2hk1E~DlL$KQH!Zte+` zsV6T=Iz#+`vly$d!*LqW$9g50O0* zezNTl%_cbOIm0=(`Vw9ewOb3hj_E)hxl+pCnk*Wa!PAN=P&f6RWtVV9Vp<29>3%gD zr$6lOSu^7iy}F$`XJxC||26^75faXjVjc1Vj*%2o{_j)VI|$+5rnuiIf`3kNe-XWd z82(>Oag2;?%>U00dcF#zrh>{6FYB#QRaYiMmq;QoU^w_E(D$Hq0utWPEKuJciA`9UN4tH z1r{}25Q!!p3ZTzJso6)V7kGJb;X>uC;;fub#jBA1h@c!7_QKoIs>yE3dSByH zpdB5D4sNYd)gjOc`?;ESUC=|@gh1E2^I7;;JbEWTl$=FgpaL?GC1y2|bf!ViUHGvai?)nY3Q{B!pf1^dxH;F`hEir{?&>FT4r$;Qk}XiZBLwmN0A3 zf__qx0Xw2XA78*Ua^3_Hj}8;yQTvQF@i1sV+0Y)PzUx<>+lGfz9YdD&)di(%v}rxZ zkc|gicZ9Qw)7a&@(1e1if+2?UOFlE@ucn%VAKGWFnk*VWJ6i3s+m4cv^j9>!4nXve z=ginE&w|3?O5;p2ZME9kjyepl6<9!$Y50866Gx`8v+x-reJcmeg?iejvV#Ax7MI zCg}ro{?JoQ(?^;JR}BB5j0MPr=OeA7=OeVEpinZhi~jcBmxz_YRbWDy^SV2vC$?3; zhF>_r;BCrJOsn7c9?uQVJRWUV=0E0ypU;2fzlh`Y0QqtX8^q4`P!?VRW5_e$Ibace zDXO2$?eSgIJ|XP zc|uYUJwkVpQ80Ovr_X@Mpb7soh7iN)N3;Y-+7D&edYJoJP~T}46xmm+I$rk)&BK!n z!9PXu7$-xd8|IS$K446+BdZ91?N=N(948XF?1abYNhpIyF5)huw;x!PQl{4BV$q7} z_Fg9&Nivdw&0pwN2h{0L2$&$%()E06im_V+Qzz2JvqROBZbZ^7a;pv&wt+$*FhP$? zuwmdb#8K=U2!sll03rxDBQf7HvHpP!uoyJEAIps73AC9b`-Ix<37)g=VHzTtPE28F zyKY2eXiMy&Vw}ji^r1a?i#sgce>}{k_ZgaTZ+#dbERMn7Ui1a!MAvNqJ%!NxuB64M zTzdeay(Y)TI#^hY?86Aip-{90Mo&E)HGe`8X1T!Ogi$TdvA~BSBpd<2XCInx zltmv`{Apzo+(IG7C71&A&~c_Z9q0{yg4Fb;(>eddg6$NjHlYHR=w_&*lJpy_W;kLj z!_@G%>~qrnFyEjG>x9|jqt$z&yP}GB19_9^r!Y3jy+nXcCkYhAFry1WOi6Zc1KC#7Y)QBotXoSV}S#+A1Feq@+f~M?|T$ zrON76Epr$7>n_b!%&ey!r*UTdXN-%6XH<$#m3i~MM4tn{e*Kyu5-rj!lG6th0TIE7 ziNLJQ=$KL=Nu$uI*;58!p|7ORGA%J#G_zweHcd2}HNl;2EnBF}{{|>TDeIk5FH$O< z`Q|8LE~}xkB*m%BDdXOBqkRm!*v8?2!KosaUF9Dvy6$&&}M4AJSXLI zzpP2)&#H;D)D6m9TM>Q{fn17QP>~D~>j<^JoIdux(ugQbUb@ec*iyfwc2X=D*^MW8 zrLEJ~*j^e-Y=in%nDdi4Qc03WSyxOK^*Bt7tWd2;n3oKl`pVXpJGln9mOL`p|6Pa~Q%`o)H8IURlqckRO8h zf|}qR;aU5p`rM<&FghHcP((v>gK`I|XV1TX>>ZpEJF1=8FJYWn=vVutx$7}3yb`!6 zzx91Ae`%(Zd9iN!y6CV5n&smpX<6nt&j$CBe+KMYgS!=nRgBfPRk@XSBWLsFiusDQ zkt$d}C+ z-x?puLFg{)`rdUD_#wbOU<`p5g+C%VR29cI%)D>g9;*>FMmq+UU`(V~lvu2{hpQj^ zIa6~orzsosxpsXcDL0H^#@fjM21@7I^M%wJ*BS(4Sw?xr8>80E-irelfBQ!J`kxr< zke94ySulQJhhUxp-W!-3o06|TnN!--UewHM9)x_qVaj!|skW*;ad8s3yWf7bwQl@LfZ=Zwvq_Fte zIog}%lm>>ZgxZap7O=XZf2=QND;i=j_&v5iUoT&0jCAL6w;@3-k(&HayHl;o;B&pr zXji@PY#0UVH5I&0t8vIdY&tP-Le218{Oi5ifpIyC5lR>(t|#uxeY?0(e%ZyWd`JFL z2Yx%Ex2@?y4GAPG%05?OGZGX6vIOCq|AoIy z`^eqRPEL`aWl$RetC#WPN+MyiXh~{SL2Q9)5qkMb0-O*RXo9+}R>Bqs# zm)$6>v$pYz+}kKoS~qQrgR?!hi@1InQ5simv*shmru)#bgxS)#%FISom*cDJ=tJJ+ z_0`6Cj49V^ql=NH2BYPJX6nWjchcL@%NAkhpcCq^VV*Of<&ew>Qh2k^=1(2)4z5l( zr||nHckJi8?ooG9nJ@BZ!!4PiI?P_>t`_HwIdg5dsyeD`38Yh5S6m4^I0uzaQjZPP zW7wIVx9;PoIG|GyZkfZG-ggG~u$yQtujyCDOS;SJ-AS&rLE1`fEM9H95w~eK4aL@Y z2qW&I-VNuK=iYVgQEuL^f~!FYuMp4(cl!VC!uB@ z|2sVVJ9gdi4Uhd6&XO=Sw0}dq|3;{5y+^73M8>}@WB*s4f_e^m7S_hUq51ZIVe*v$ zcJ`*$R`|5^G)#Z%{|Em3FPZr98GesID`0JB_?E)2@y7i9`v#-`E$<%y{l7ucB~7i& z-|_wiZ+LtrCT1E224;L#W_lXtH*h&KD?1H6D?2_bD;*6z1H0BcX8bRy|2>a@-@xGy z@f7fBmE^^KZy5e-9>5x@~xY9DRxbg!e#mWB~;16{D-_-gu z@8JKNT1+%_boBVFOmCKBc~^^#hUpKr=xJD(>HkHo_hJ4T^-qKSC$)Y<|Nmx=zw#UY zp_Z|=HI1qItPwG_vv&|Q(X+#6 zd_y%z>iv1Z#{OrP#rwU-z(xo2`)>HXeMgY|{rlGbFUSAg5i1iNKJyzd);D@qy0`Kh zAEvi5+gm-$+x|BnmiOAX8umAe_a5)9%q;Ib-}?UU^G3n)R!ax-PV?ULz0beQ4S$Dp z{5Igf+UB>dr1dP{ARhmdK`r!*-%@wp@g8rc6?n_rfzhC*f15d|8R=Nxl6Bt1qNit| zf14!4-ew+C1AZ%G3&5MCwEXr4@A)M3baX857?R&d)PF#41odowC#}G+u`|6*C=LKi z<@YYCf2Ojqu)ZBg>ACz)a-n}W${(Y@y;}sp@Mfbo+P^0V{ZaAPo%b7}LXXeR#{4$! zpMVusI>t9l%YS+`$=enCkGGLTsKa_F&ZNAy+h<*VVQ{5OLDVIIgYF>;C`1f40ujRx zfeH{civJ=21ss71_3chQlz_>nr(8xIWezgCewnDqzse6qSPr&4Y$dT7(1KJa{v*^R zW_jVY?W1Z{eMg(~D0k&)=4IRHt>^BtTKiEViQN=MEZMY+fgt6A$^p3{7N!anLeHVJ z;RP*P9m^VeCFzN%r?>s^knp+NqV%_|;;=MK)V@Op`1|d!3B<}v^esWuwZ^A<7j8$K zi(YD0)|>0Xm5_&RR+i;iR^Dge>rSZ;tSr47_C|zb)@@CeuC1At5qMTwv(u1cdIZl4 zE^ERvB2By>z%6Hq2L$<>W4|<4Bg`J4#f;!Roy3gokfL}?r8IxIvv54KDjlDBI096A zwPB^3KCA@~27c;@KD4@oA=pK!_M)$3JQk3_1G=WY9*2^X`J9LG1!9sXulq}Xv~)ee3%v@$YcHqHx&OiqWvJ;ZKV9E zUZQb2XzQ{rd)P^OBIwQDCF6N!B}3iXO_xl5WCx%zgyTtNE}d!#^K*|L8s<&cQmvBS zxP7;c+@%;WY9!D-79##Zf;jOQF#1fa<5hBf#O$=QCkU}|YWNCmlYw{??Fygkq$4Il;l9ZoZ>VP7+>DZ3t(m}Xv;Y6)i~P*BEw;|;lM z)5gASt*bOIg1k&Bp6p;&r#vkSBLUzpq|G4IK3`mfP5;Cl64*~U*-^r^j<4J|leiRR zM;_a8ssrS*w3Jeh=^SjOc!Da)*D!@2oU5xjUm6szdhZIsUC~SNNeDRG|BUk0{A_&L zY&Uz333vZt<<=F?EV5)uFtTkGG7ay`xs_R@cz5$*Tj>(r1?3A?89CP3PT2|h z>cb8htK+U~_eje%;Sq>B16 zDaCfqm8SEt4k&(peJSx&M^I$}>f`m`O5Lf^H+J?dzQ_GU#aA7(T(0urSK(utUENQu zSvi^5;^@Dgd;B~)Zunm>>*SE&Tnk*X%%ShR7m?4ct(wNXc`>n?3*G1H=AYD-kP~=8 z2A!uxwuz!Wu$ozeke^V#`BC;@XLaxH1%Iby5VzJAzD2{=>9WwuQ$#T5@b~!C74y;K|LnEE_>WX$Nh@>*B z#vV4&@u!(9N=Nxi*Zeyqb6J-=iaW=77ED?3o@52#<{qXgmnJu*B^NK9eq=2~jbEJk z@pgH4`!x;EkjdDX^Nou#3i#D*%w5YZMwu3`HpeZT%ig}=T?`MVKef=r4xaU3#yzGJ${@`>Kf{;vg+V|q*^DBAms-l94H|ZS z{`PM(ONFj0-?r}l)c{|sXKwLx&L33w8M3rN&-?|_A|rxd`IAlz*Dv#tZV|ArKq(uS zqynSQmU8)Fq1;wQSXubxm4Qz9wZD{kxuLSLDExMy=k~SM2WAoL<;%B2VywHDN30d^ zxr_38tKr>CaIWBTaA~oe%m%=|=a!_v-QXj9fX86LpD)K@!a`ZJJJ$Xbm5HYDHkf3u zL02KQhC~(R*s^}>P5TL=tc_lEQJiwrB!pAt+ghO&Lon)~C%9J3SLgRx z+5s=!lD$Q>MR|+*#ZxEil9sYd8S|3XqNaX@Z|^3~V0Rcu$-7s+E+)R8ck}Jybf=s?-&PXLgqZ39oIAvR-g`X@` zVKMroNjeG?uU`Z*3ZpN+L!$;?+XMnYcd}K_5s@?H_pp^0nqnp^10zf!5N1RTGh_8J zodw#Tk&U-vkMh{-rXU|cF0iBnpT3`zX58G}C+86r(N>XGxtVDA;5>JU_u}Wk<9Ckd zPWE*%B6Dc<)&@xl221^$#Te=mXUB{vHD~bPWQkHX_z1QNJXSxXKw#rh2eN{#utZg+ zJpt{d66`q527q9V@Rl&2c-@+IHfi@d{5()0%=h`K}ji-W+_irwih>`fFEs~rCQ%2dEcro#!W9f;wW%y3{ufgu}XWTB?Ewi4q z$ygeI5OC$wc(;2vA}Mo7{m~Al_@<^s#EyXV*6hul5RR5PA$V2Axe9Wt0dFaeXquc7 zxHI&xHm`v^gm%8a#n*c{1i|6z;vDjw zi@eYuk7>{+!Hkx8Vxpd@3`GBtTcwHa*2b%Zm8QyK%Qby(rq36R!jfZa$K$=KUDp$I zs0{%EH35DP0D&1k&)$)IdX$uu7JnonCTZ4^38^dzrvfi}ztt7?>K-m(-Ot#bF9BBO zUiJ+Rj=<36pHZ|UkA$z@wBTDJj>YGrdBrg#gF$coF_OhckBqtxIvVhE@1s`t2mRML z-6-amf1IyfFO6*7n0>UCI3F=3qz%xhSHu_q$P(yiR5)_P8ayo0zC-cYw?$veh+3>{ zrVMI0O=4uYt_7)z=7{spK_7u)Jno-N7)m|0_W2B!HdgVnj0|!<Vf*lW`#-c|cwVM@3D;3v$iYs_m&jsOFh6Z?uI{#=t*THTdy6Fd%ps$A&v?P{Cu* zRSZZauxEyNTgR)-1mF*2qNOa|^d~bvygzAYbA4fMmGH!)r?hf$Aq}eHR>&a$#Kjjxx#U)$!Sp(%JDuc=Q*B94d9$Y^vgOq;(B^vb`d+u zQLhe=m;6d2zewqfyGr@e7*R^5EM7QUFqyBa&wK^?KyL@Fi!w!GmC0jDn<*KVo4g`W zTAO&`%X+CxAqvYVML1|rAr@C;XX4wxFf5(VXKTmT*<8q)8<^{)>0EUva!orr)2ASR zyhriTirgKBR~u0qS#hQF_jCAZw$&0VuqDU0X>wzobq`D{phz_XXCazdqY&!|D;#u& z+$pL!8iCU^l;zyo6e1xLiPghT9s%jY5}oOKrP1ESvDoL~ipQ4N0hze(SVBWi5ge>3 z9(@9{^T%;%LqFyh>xGb~ubi!>xvlNwGJQ&2m*Zx;JQ0p-MN(3-Hl)3%KWY_$_fq#$ zKyEFBXraQe3|)6FUZlKIa%O7*o9+EJ-U9hX86v1^N&!ia89@2GQiFIt6Lf6=3MceghzyAc*G(_-`(P7gU^L+cG3YcDseGV#RJmSQh3CY`=dpc z{yQFEzpGjX@s2UMB$WuBeanlS##Q6mcuc?FN4xThM`I%7gwB|75r~>)pPM5Xm)M}2 zO>CNua}jW+`VUZ25i$dJjN(GMFDELHNOV!-iyZ^S&{IFG5%*w#HwVF~p+Y<5(F0U>jf8jYRF zH$0gCY?|`c{j^)iX5H|%J+P2X|9U8G*(m4fw!-ZkXFT9;>n;bl;k?X*CIY$*g@6ceool zn)q5_RJ!C^g5!BV&i1Lt!ukO->%jt3C-5Qzs^12rypJNCo?Bbm8aNNI;CxIilQO9U z9&c@M(1}#GH?0`j4&-h=vcdqStHYt1Vy+h0>CuLC{vko6da^D$s(nX$h<-7@ACW2(DHA&M@=5&1G z@82X9Sa|HpMeGRq@7}`)C8P3do=JWYNXPA=zQE5tszXkOC0vfj{wQ&ULb{UfX|VwT zA)z%{cI0aLvmK+UtS(;{E{AeBH$$K>ntGGS+`CqHDY)L2JdbBfyoc#eRvTov?t9dC z%T>|8zuex;7b`tCJTDIb*WCj{-bwEg^`~o*&t7tk#Ose?LdJVbPHGnmN@aU1 zm57l@=s59*VY*dXF?Be*%<)+*Ru)%AK~j5YWQa0_7jTrnloUk6;3dLnpix>{;bxX=rhUhQgyZ4#gk=KZ0DvMMVob?H6ZvGgI@6uQh+K(+ET{EkS~aCFzQjL;Dk_5E**T7Sa0CM*DJas#uo!>wy+fpFNLvJy7a9s+O-AH5D>}2|%=GHX~BWmh4 zUG~f8Qx4g%1A}A)Ca~6kV{&xkb!uxP+;fGtmy+72b=MQCoY%vauxpHmYGplDN>+lN z^@oWTVzYU3Q!{f@J_ncIcko_vWLWqb$X5}`p3HCjH!%D{{S?;t_HO#u&ow>ZrVO!Ii=t&*#KyjoymU!uGt{vo~u;}s%Vu~^(j(28Vtqqqd2r0PZN zF5W(M;24bUo>q8C^TZJ|v;$}cE|U>(TnS`ijyXPl{i`;=)JMrobu$R73zHyU`!s8;2rO$ShT*p}d1j+|8qo~x4Mr#HpCq)N1^+>LUATSJd$b-ZZ zfk}tYRd+`K5^=7+?l7;*wE0knA2y&1)as7QaljDUAsdeKSi1&MueXP=0NDa*1s1`J zauM6{Q>vA?Qr~IJc&HWt?tM$-XvVV}1q?J|8GTKvo0-a-!#r(Ck3J4&vx2q z7`d*tbnP((M7IXT+}T+~0W$tTmNjQ~aIL@#g7>0K_?LBFAstrXoUrG_7t#tFvzwcU z-;=GHIpW;K{L^WL^AN;N}j;DjkQzm;>?D8RK`Z^9FZtv(Pu`!#!xQ11s=$X z>M*P?t)m%01+Kzn4$PS5Mj!1-ZR`tvgWDM;wNL-tN;flpL_5JcfjWVGi1|*uC`R+u z=;B!=NY=3xW|I@e#u=yclh)Cl#8DV$M+c2Ld>EdYV=)NqEU}1CzTnOeOMq1^#Q9;S z)b9_v%z5LExgE+(wSru_BXrQO*RE|^ijbQLzDGV;5k3VfZn+VHMf_JSyhq2HswTF4 zP>oZ={#_nRgmPPe(j3^SIsmqPo;KY$3sC!<>Dzu{P2cIBz7}I>uAR+Igr9O$UgH- zKE#q`oKU-61MFEm+d3I{i3$t(&HbYE5B`r(SnxkX;eTSof9Jvf)AH{>=Ds{L1uwp2FX9R=V%Wt(SIrTKRGkw-B>2vj012?(q0anj4;0-?)y(Js46`4F!zjN2CM*h~NbH2-x`$GW)MlNa(}B z;lzN_et_RjLlETsB%%geBvtQe$|M2<)bCR&nBC1+wdq(@UMn(JQZF?XPEBolOa=09 zdfL2t^6J`hk(u0lJMS>&UDWod_|kqWFi8>k^A=V(wnQ!7zE%Y|+}=^xVrI9CO{3413(SacWym$CMor1FNU~|Q zM!(tAZZ)IpHp}|VV-61wPX)f(8fzqSankmH-*jkNnT_@=iO7gz(>S^#% zwCRjw^z23nHZn^KZ7g_s;YTZW%}33U6C!OA(P+&AnAXCY_IcQJlGnpP)|TwTOd;G4 zSS##EjpNx(-(og_HJHxpv|`soMN%yg*4??_DXZu6{v&u#@8B4r6zI=$l1Rzwv_dav zm>7uZ!?++7D4TtlE>m1NwZj2$Mfz!QK1ZC@xr zC?(I;696Y16FW8eb{d=TQ+Wj;>}I^6QQZ*3h>iI1Gu80X!e0`{>DMG-f*IGWy-|jw z=S1zO?S>@dM9chmN8cYMDm=xm$~>$}sJuWBY{LMxARb40H~eu9>mIU!NcTqC;Z0Vd zvAsMCH?#%|IUG580A-EDW2ch)koKG_P$I6`mY2-_Gy<0=R&@*68-CkDNt3Fe%R;$}J*dpI%q7B51a!U_4IO z-0iVh>#^Z42^Tt|Ur?i~K|HXRnmYdARyC|ItkGe#IjY$7F{4>sSef>dRW2fQsmNBI(x3sP2EY83Gm2Sone1Lb0Dqm_U1F7e2p-ynCQO=(RL--sx@R4?)HOC&ZK z)n?a{Nl-9KIgh=lm2Mm+S;3>MrHWvn&A%Vkecbnu;I_*0p4#F*;+nl7+=s*?%IH^` zDQ%e(Q+j?T_@jOd?Zdr z?|mhhpWAQ@(qrcUcB3>bc8|n|hI6wT)>dtPQLtmTx#*@(ho+$`PDxtBC7Q%I6;@41 z;inOxkJcViv`L_wl9R5zEfzqePlP_-|r z9A2p9eO!7Bq^sQ^=%~S3(qyBau9UN+JkZFcB)+AMn!mD?o2}amYq;o`m)T8pnOZ=X zmB%$S@V{+5t*-WlvsO1Y{^QB=2eZCW%#BjRwoZVXL8>e@+P`O)r&PuEgQ=C#yojCZ zLZN?#Qu2;l!nFEvC>0VJeFf9u?m2Qnob^mo|AL>c&c3(FR zxUQo4cZeOPvWugmBTKo=dYQ_eg$ZeT-eh|_5c)^}oysOcF7yGlo|A^b@);$hB-P$v zPo$I%x;lW~kJ&))>$(sXd}zE+9v(od=!$UIYI_uimK%rOHB$s1+^=+75X&2%!7mYB zP)udJlkhKA)U+X7ZGzX7uzuJb94f=9uFgw;Ygb&B9-&*Tb?~(v5>Fjy>T2}MVB9(9 zWnk`WfRjFqSn+bvMIERA0wI1wLfUby#lGUV>jWbPlJ1Q(7^0Yw$R?l; z)0z@mrTq8J|9Kv8$Cw-^!YW!3+0A_sNdO;|Kl=YR*nbNp;WjkFa%B zcN;W^F8AXV&i<11{<|PsmbNQ=bkEr}%R9&?6mH*8zo6&xt`GyXM#W&((oQIA0<&~Hv`;U+CJ`UfE{Qfnd?bOeA;hf@&#e!( zn(QUX+6WBR*xo++E#xf=I^Gvw=mT|_Jh-S~E_!`3%XA@I-a~!FWs%1TZi^HR=9QTy z@KQIk17*v{y284{`q(<^x(gdkR(9`Hj|WYc|JHcO3o8&@h5#QK;vv}gFJMTm-o9S( z8B%T1O}t7}mx!s|57rVXcPV#_+(22HWRo${gFCj)4KijGZCtqwb_Q@06P9&Zn7*@p zi+F*-ZLAWqs%&^fH#j4PNKFmUsoo!>TbwsF*KF%h^!saC)8JKjz2c+{*}$NCW(6@+ z_^`23$D{z0k3(P|#Q_kcC!)`91Q{pFnsKtfezJT1WMAfkx>o6pyBOXxij8D+#3DXa zVv9N}Kr1)I;*ve=7QY^pZ1LCz-wj+TINjY-4C3uQf{VHcu`>KYYz?cwP?ei9-pA8= zlECIb+oxg6>v;3?(;#CHnXC}{8i}e0bJ%P)M}{WVbD-4OwM2u;biCTGbJ_Fg#JRQ?!JHY(X-yf0x=XekQ;DfXC+ffgzh!z}zH+ zt|Nb3We+-gVRk`14c3%a#B*f0TgP=wi`r7?^yV)R&U zI{%oyWJs~OcWG0Qq6m(fpK`S7G#3xkE^{b4l2ScBK=fAuM+%> z3|8Z=cPS^R)%e5L;{Fd9I~^_3qV&@C3H+Z(n{#k$D#7!BirkBJj*RsQr=&LZj`Jlg zEsc&HuR6g#3I~&F3Jde<*_r&(!$5}WrOw+6(~FPL@p^dO!v3TfY!f%j-(f#Vx5K(_ zNNiM((hn_8T1((bDH|xE92Sl4i=co#Q8*NC70Pc>h;*%xV3cxAOUTyOjV7WVT{9nBv<(&GuX8d+3VFq z2kfFwjzJzBGKj|&?ak{?6`5WM=ZjW=tGujIZ`vXgaQsH@#a+-$ilLes)~#!&&g&KM z1h>h+mT33T%WXrQ_w47ck4wVBr4aATO0vYFXt}==!41FTpYV#sN(M~%97hqAcNJ3y*w?wcB5>?XiFowiGuXfj;Jat zE-fwVz)~^L7mItovAV2o?I;SuQ|Jo7w3Snb4iVq_{J)~1&UQ%IhW9c^z09X0_ zw5MZRQ{`z#89qH2#Jc0Tqci7BVuh}iY4Y&ag82%a23$k>vp_Hxr6s4dW>itjB`uEHJ;0y zV4kbR1zYxFFg?}S9p{Qf07KIS*?QxUC|4oijO1!xdFu(6jc5Di;zp~ zGDJNq1e9)*!pa|1HOrQHw!T|_5*qOsU$y%S$H^KKNpEe%)O1?P{BY1W^!VuaZQh3T zzUhiOUuycnY>@7gPX|yrG`4DJjw2f7v6((x^;2 zfZxLt(=?I3j@Pzv1BLc+LYT#XDGe3E)7LOeve(?~`d0~tdOZ8a_6m+r0BIO~Nhi{ztn6&K~MXK#|9jIkckaR5rJdvxu9+is&zO^K>RN%b(;( zB@6lvuK-j8U!Tjp%*>3JRZtC~wiK`8=TkK{`qB9v`?5lKEt4CiUiMmaL|&^z?tsEpI&UM_IfOr{U(#1wbRjUJW8RY9>m7scT+}|3pOM zF}&QH+E@ymw=I*-OjS=<>>;)=tT))^l(N8pdT}3|C)y*;4PSw+%z`EX^gIJ&eE&HZkO9jXbz-)$!2341n#gR7v$ZEqg4Wx82Kiqn5$mFM@|Pun`}4+a#M= zt0;;TEw45N=^vSe;WtUDOUkCqU)1XB-u^g9t|P1kLkz(56qWn6bm&Jib>|A zPiykjD~Zs$X&m5tMR0g##^B^jId%K6a#wR_!dtsavLlqDPI{0%uut>|<*eMfU=P4~ zF=_xVUY(d|*^t2}hxfn+Kof8mM+60Z{}RF*z4o>L?YDeRq^&g z_{`W6SVx3RsAPn7gcKs@{+4mt8gg<2O%CzK2zd_a#=Fi)B8#17@&Z%O?jh*HTENae zqC1lzWv0cd&O$B}FWSa-k4@Ruql@OR6Tz6v7Ci$GWSOsZ6Ae@z4w|-yO<-;v@Rypn z1Qtc+s_0EjZY}qt$RJ+ly~f|WpFiPppYoIKdq5Ucs4(xF(S>rES$xmU@qRP-R%n$< zF*6C$9Z1m7vzWf4??wzpX-?)ss+Jp5A_K>v#$k$&h$}LYHxBgo=;CVx<+-6AYBe0oIB?Wu8N5yPI~%$<9#)-8Jrx9=8_()WWT;+ zCD5u-Q4jq5RyuLpJ$q5L+i2(lh`%gw8jkmCY6^hLDekWRQC|Q@m?G!spn^b<1wZ{h zLvhK048cA?8^uv12vCidi$Ci}5LuE)iyAHy)ys;J-k&NSUOJ{ue7$A?IFJK8azb=w z^Vjv4g&r`-sFuNeKkw)_uXeH74_iHpIk6Wz|VF-etsi6QsGz6El9R-;TY ziU<2TnmQ>H@|u&a&>57h|ZDWf>*;)W&eqTG9b-?tAIyq?xI zv;1P#6VPC6w*C!6)-#zTMEjs}mer^mZ(CUrq>LNfnYD3$x6zjGY$)zXYxUtw!HwP# zesiJ+HK@JwQ9tV zmr_I*QE311bByRp`Me2%Uz+xhvqFp;LD;kv(J;z^4l7ORD8dmnjjUHaJUcIm&*Y27 zcn_zi04;p76YKKmDxnHF#rp~Fr3x}&_bU4-_e4|J#G=QL08dPA;g_h*pvT(YP2E=T zm+`oG>l~}(j_{R9H1LavmBEv!>fo&WWQ$7(G$B$131mYpU^`9>__qn}0v)oW?b}iP zK6x_yYLC0hvcG91oz;?EGiqP3Y>wEnN}gZb2P&5+g>2}?LG?v(uK!F!L#!IZ?>oGL zO|XFh` zps*%IUsPW~n-p^E#q8WACcy|O3%@h_lj*UN4lW&qGkic{*pA_Vb-!q-Ej!w48S#y! z@#%+kfw#!IY~9VnZYgx$ekvT|W<@jxtp=1x$AYdnzA+t>DisN%ISgV}mnu~IOyW9? za%>CJ=VB z<#~%+%eV|XVz9*cwVIc>a73`%dd9^q!q3?e6oh%zP!6K1}llHG0gA@U`E?`{<;U2~qZbmW-)U z$H#PenxsPOx{Dg|NQ5S}En&z~(57b(C@bNfzarh4?nyQygSFoq@C8KwxO*LdF{;1R zQ=;4RdH8j|-mcY_*XBH);q>-BkE~Dgr&YlRB90iI zi1pDJ^kbp@#nziYA?r6!Ee&KBEku$=CdYwOUZb|%q`wm;VCoz)XA9SIL&NjXNXQ$4 zW9iSqO(cvujR<9r>=P2jJ5@Z_t1^nUs@$7V9b{vdpx5>6=U?(7JQ8_MJSkEri#?9B zq^u*`wV&$aP9tA)K1tn6r!?cm+fy_0H^ejG(@IVimPqZc8E0k5!lsd!6Ew2(9T05a z)y==Uzg&19RH8?5l{mtqO`p;r$4gTgMTa?*vSAC^c6!>25l(?BEIw9V10T@%903bw z?jH8)e=v)6G{Aj9oHZn!LL0WtQL$*(g8+4{1;d3;C3A?U0mCPuZ7LMi)|O5|7eRCl*JF)!;%3%U{a83{5)kIH-`<89Q-z_N?t6l^>Ov0%xEG5jDb7=qnY6)yxz@}S~Y zE4brb`-P_u@jaoH6guQ3lt!OTTqevcXSqbUq_Cm)VN{QQHFOZDVqV(IU&gq`otgQ# zqq!mW2r^F#PAIec?hZLqOU^MPZ)r;Q+#MYd8gQ2haK|++#Wn&y*N?}!7c19$ zfr8$&KPO`|XN5w%PKm$`47`r7CqzFjHlvRMgYtu4$o4o9LeP8yYHVQs^lQMJKEvK^ z;S1G**|hD~RTQ`1=hd6ue``g!ZJ<9fB{6Q^;;GZm8} z^A@)sK^GNIch}fbZuDLE)I!b4^Gc8gv#RLmA-rj`pK3XNfDlr|Zv=z~-5yQ9#kkSP zePylbwdO+vnI39geHH(-3^lb$L-LlSi-OCo~6DATq$It0QwKe zskRS%Gp@RlC1S%&JHf=*OG%BgJ6jhJ9X@YEHG~DX37lUBGUbhThDT{SWapys`!R zT=D2|y6JEqPR^ySL-;5qX6O_16q`kzGR5%S7tVhRsl^fk z(x1XrjF{fqD++_F-E%Mkh$&97nS0;4u%jHq zwK(g%j#v(Iva#WD(OHXJ4k9(ES+0wL8>$U!B4sIeaVH z=r_~@j(+IiS7R%P?zA7MRuVvNG@b_dtSUR`iVOIRYBZT4KZRnycCogO$9vT#|C z;6bL1>)2#MHr_ekMxg!yJ8Rv%hqTq@1(Y8ma2l z{T@!Qo#7SUyqaI_F8X>oUdXFe2b?IZnLzZoqhb4Im55WMwO&1up~w&Iu+wt;Or=s& zJcFb(OydUVP{`Jp2uW@h?Q|?rD2!IsVZn=62lI{NF?|hZfKYJAkNJKV?Q_oLi02-V z*-EY1Z0pp`t-xut^nzdhtBn=cWADO)_xUA7g*pE6k@JYEVl`mVg(Tbn`xPv+TCzeW zt^y@{}#@6LSYmh zi5=$5C7pGwq2{+B3gp#wkp6hhfn)`Iu*t6tF1hZ_tWFMy*Eqy*iQBS!x!U zefDpAkA>{rr#Dd`Dt&os^pnxbpmIkj%ZwHjdwB;7{Mf^}@})v86`DTRG?2z9@@wOL z&_Tmz^+15DW0d$fw-G5~MaRuQhC_e$*HL(;{02rI@hL;bDeG5>=M_^8#GLg6j4MT6 z_)u~Dg^F6SWPAj)P@~g-F?mDnjJfP14NX4SeCfXJzZGRX&rGTuO<*lIlRip6;pD9~ zd|m|A2(xBivMqhtWK;8wl%e9jj!Apqq_-ID-6k8wU}rSDu5_S2T@v?tO<+4Hf0;{f z6=%2C?`%yM@?CSnemm}GIrKP~pBP3PmijYm6~Pp5mNfsk$iyS30j`Pjx1H|c)Ov08 zK9D{D3{s#_$aK}(Q$ne9jj=fe}6^__vMV}C%P!|%U zrXR%+z-z9Pv|ZGb6zLMMmuEmuUk&#lCToq8Me2sfrZ?xt*~Ktr(sO-+rx=}tdpP6+1p)g zfEwf{5FZ9Ax5k)IUD9HAynC4?O+~aykLy zefLoWlKH%5SXs3Zraj+ZM7F1!?%ny-)cC#CQXPh1;OiB=`PHE4q0@&zZE>jCVj!?o z1g|87LiljBDCd@@Id3?_0q=Q%esW(I$RP~_Ic`*I{UGSV1j5OsZv{BOM8HG;xQmxj zv^DtQpOdo%V#^q^p2HBcnFcm-b>Kl9shFT@c8w8QDFc4WiiHluYe_p0O@{W_v!Mvi zi85lMP{kb-q%pJ)K8`$-Ts!pB|D8El6yO;eA(jY>5fsXRED{==Rtwwxq!BuAaYwr~Fvj}K87a)> z##C}I?V0&BJI+?5Lpf3St~L^;MQ};4Q?OIeB}6SHH=IWkdWD_+gfQN9O-yCj^d27$ zXof|;X4Og@hd2r|F8Y1s&_U$eL}&Rlo}fO*MM#T7B*fuV1jGSUWw5#~9cWWl*E>pj z!9t|p?vlBf4c3+) z?$WZ-I-*o!8j!ZdW%dTC%Ve^nd1Q~?E$_-3s=UHOjPi9h?7WUf=4u26rKMBwShZ}) z1UMJWN=7*k#~=o%@+A1RdKDo;IRgB%d%ibMBPo(uo{5ua?=F7TiQqpFsRmu6*v+nLR5A7a~zE0(wrr(bV(m01xIf3)1ZBum%RW==iTdc4 zAxO0$WSB;KO$_GoPM^p^{OZErhEuS0@O6j(@cYexNq>N_+$j;lPhnC@jKU296>s`e z(vo0rIfXs+75)oF*4B6zU8c7&M=+M>cH@(~Muo-`)5B=$Gh)i8?erizIz`26XbGF+ z`p9pYasSni7n{w~h057=;-s;0&H@H8Pz|UH4xklK)F9}<{6|!Pzyyj~uY)mg@l5G< z4|cBcrhXDd0+rKZPQ}f5VH6xkgIquS`ko#?d8-PDkw#r4Oh=fYO>R%131h}A!Li$& z(Ty-LFB_r^$hok%6nIblD3_P@u`fqS3&M(7m3+J>+7Jhg_{~pu zEDsD{oB80OgP|x;Cs1foM^Y&LESL7&d5;ji^ryIcPzD?oZ(%16njpoG1(~dePKz%z zHc~s{Il}vby(Y7Dz%0-XSq1BjP^Zo8Uo@gJqGKkqi{$Q;W`k3Aw>SCaVAy38n!~g^ zT~tJKhL5KX!R6ma>e*VDmz6M(W(9;YhYahV-AKi+Siljq(@I@o@tr zxksr%*%Pzyc2cQ?uk=ECCVzVFeKJp(c z3t)x2^tu`{!Xgob;)5bl0Ilk*b(WWtlbY-WAV$Pon7SJ4o35<7S=82DZ7M0=&lhp8 zm%Ji35DNnv6i-yPUJi1`Otv`Rv>x=tztbZxDJgHdj_pa&pyX&;B6tTR*QsF_gNpB5 zHz%v1$%T+=neyKwWxmyhs;ezkgvJxp9nfs6Rlzq9Wm#XHBcB!IKg(is)HY64UpYBP zd+@ktp$~RJUuLH+vyl?Rk?j^rf+esyzwf|uzP<%*^(T{F6P?MSR<@b7hqJjwrKyo& zhJhFSir)kxAbeaNCP1kjXaKh((|||tDc~wCBnx%7xj{GX z;nGKUl}Sc1tc30sUd%%{a}4ScL_6()D&#URb{HLB4oheg(w5KkwE_2|XCd`^S1D#j zW{4I$fsu7Hr!MR{{E~P~{4DD?IdJX)X$GYqd#yp)bIq~_`0Kodbgkz9Wg5tfkg(r3Rt1{D&}FdfUfkDh3~p^x+Bsm`@;l$n4_ zd#JsT*KKPUf95doRWWDe3A|IF6jqqIZQSo*IXF}*+KMXj3!UJ*eE@oRr<(q&-t-^G zIsZ%8_aE}b{};ij5T%%rrL&QvnStIvwS)gAHf8*uep>$@h)vnPos9k|_53SJ{I=X- z{}yI`d)oXBf75?U|Hl7b{|4r7^PB%efcW?Pf426oR~D9Uk?40|{kGcr8~&00)-$qx zxA-^xS19_gy5DJtzxjXb{44MObp2mFe!u%)@xOKd#u>ib{SFX+TmBno`<}(W>ioTa z%T@p8{4M?0yuRcAF`9puq5k8B|JQZ=k0tr%68)R!(SKgP{~|%f)1YQ&rpKdZVxz~S z|F$^$YjyPZsPg~b>ge0+?BAV^zU88{|I6vM}Kb(&KPV?UtsQ?uac^wUSx_BxHKR%CNR6V7s+}%oWh>H>WaS31$ zEBXOJ6Y>2a;7$Z)A!8;$0V2@t5kN-7>jeZ-1O7?5JWzWHL3L)W?9z2iac%8watZm3 z7B{ke&X3;OXiZK#9HuhZ9S$=X9T;sU-`xuYqC!Okhvp~J0t+ZTQSeEa^h7qwRX36R zo-e?m@3?BXARq2q#p_6>LJy2w@C#fGZ#yD5vRQrpM`2$X6xSARiMzYg1b1m%8h3Ys zy9IX-?gWB{;4XpS?%ELCU4jz`9$cpHoA=(#y?3VGsZ-Upzgqh{-`A&i|Ji%3m8fQJ zs}Q&`s3+2n>^h0Z5WVlJ8jV{z{-uQ@ozFP*e!$;G<*16UCiuAv_=~`+n=%mEcPh{m z_{-uhDldxPY;nBpTJTyYdTHXT=9aJMcbMmMNg{;?b>}0}HiqJeEz9uxZ?Uq)I2xF` z8~xs;33GU_!5*Qhl)4YKcpv|Gu7X7uw$PX;@Pi^@GUq8OLJnJ{II*c{BOkqnii~(> z-LWiX)s(qFv3db;35vrtI~L`bzhZDc;~p2ZgC4Y>QJBtj+{pg47r}MKViqa%iV{Y& zQ^e)#b4xMBoH!gIF-7@1IX~LAl+LIJz;N1{yTMUjrLyJNv&Q6Ok7CbL<|6?}0Rn9` z_&hu0N7%tOP`2FoqR|A&t}uo4Jk z>Re^7`flBIq-)K$#y^ZXw1vz?^dR2U=U_&UdkZ;{!fi@!eJ}S5)dJ=!UcoJe%c0~; zd2N4A%Au!?5PREP7&f#dXhC2kC-w4^5Hv;jn(%bFXqCHAHtBj-{*QeY^jjGI$h+-o z24rWEZOUyBiu}=8-)oF-*1LN?L}m%%S0rPy$qFP#g({>+T*S7Q7^^-&v24!1?uGYt z;DZt<6Y~;!a}0bbBhGFM-;|HH7Yew*e%=Xr$LEc9E-fvV-h~LSVoH5a%V7EGXQ0N< z?g1ETH5i=u&!1}6kOoMoeVnt>ivh0DDH?`w=R!oxtAt)anjtxS{n%ovU@UdL{AsB9 zSjt!z+mg0|5&h!xqA_;n2XFCeEq7+6*SidSAp<^iGWCUa@cWIg@;Ky^zYJOooM~Sj zg`GiK3xxFam}K2YrUO-WDGDcnB~9J4e!;7Yh@_=@2Bctr;is~RclfzFf@(rR{dhP_ z3oVmZ3mJ_n*J(btd~#Es-*TlYNdst~W{PJ!DsNR$mNLj}k@!*{e<@UAeqztnbO(1V zdUbtJx$N=L<+O9a@LlpbZgV=Xq*~Y=?XVyD>mH5*E1tj0Vb2jP^f?pWR#82k56NAe zj$V8n7}jd?`&Fs<&Ib`MeOxwLB&PAy70;aXbB?^MQ@9`qwpGsD)e?+@BCD5{r?JIHK`Ga+GF`mh1qKOIWA_1*6LP_PP<93KHv3OA2JZ++%YibtMQ1vgOMk z`L=$f@I!Z$diso$n;D{{))I`@)RaCnL?3|CpQn&TZMAWi@!f1eIJ z_CJ!oK^!+cfa>_{QQVcal>UMDC>A3pXLYQG5*R8Q^0`*U+G2%KR$pn18DQ@dlgKJWja5y*WRo`~qSb za?N$(LQUAeS zkOR-i-$41HZNT(2+<6`ww|}1i?d1@IX^vi;g zXPJ4N$%+KW1wA+htA9l3Lb^6*pGGdQbS>sS?>t4uL!+mojSPkU7|OnR62jAmZPW6e z)jDM+f5}c;A;OUEf^2LWPlmCiyIx=QLw<7_@;wuihWw^3BD`k{{(QgTU1R!&uj;ob zVO>OR?{(rxA$rdaAXx`+-IIRN4@KR~KDKQB&@QlS@GxeZbD$FNq83xLI=9MJ!d!Ag zt)UNZS3^i~k*Z&o>Z9*kUDnuCWu~BM=O}NF0>VMS8PU{?Hbtg}w`;}!^UVMShv*Nr zcWU!$AZs>(*1$&>0xE(Q`~_;Wes%vBVhLc}!OKwQfB@R@u4 zhJ6SWSmxO9k;JB88Ci)ZxpEc{zL1nIDl<*G1+iK za0H#~XNYx0+Jbhqj}<)*9M~?HQJk;Y_WnbeufA2c`BrMFU6Yt0cjvEQ5r>RocV-fO z&t8IN`ONuL>P0Jf-9wM7Te`<#rMj)5zcL(KfPC?r%Pn1l!ntWIj1JSz6`aAt2FIFdtb^VHN;1 ztdwY1wvwFHIL_XNehz(V0;$en)@Cm~uD0bN)KF1nz>e za`6PVHpuy8w6wI9j%&;O0ZC>9wLb?QTgr)wCUJYX;DSX8u6k=e>YNaRAre0!7Ruf0R*_2{WOS3q zNOsmwnskb~97&5p_?E%2e!isEx(9(b{s$a^KLZCPI)V=%fK^Kl zQ1_Ys7}KBD6WJg35kDfz!^{scA_`_w5-B3;E&n=nS(BE z>)yN^Y_}Tu@=D~|8NEC;#L6{pKGtF6Q*`3YjBgZfTeQi~H&@@@@A0yr+Sk#2S!g^s zRAFp~GRxXy%>owmk}#e$XdQimy=Z!(fPsrFID$LhT8=&^*tH-;t71QO=V|w&lyRf+ z76;W^l$LOzA9p{u@codk!^!LW>b2_|+%f{Po+)E>Q=#|S($xQ?2`UFB`lUq9g>z4Z zb$A!t8W>@A>6m;IwI{O2N=Sqaj5RBs3abf6_<9p3AQ5qcWDW(#%HKW(nC1|iD;FTNuT_N3WEncq2_xVq3GFU@7LOd0+X!IU<}}ec-M(N z<(lQw0mD_3NAU^ba%^l*50jL?e>pO0gS0KP)r4RT&r?~BtH6G)A{A>6Nz!@?A5xciY-iMO z1HFD>jl3xjd7J)G2Gjuu3dk#7PZ_tjm}^3$h=!a)?t;}r4i_aqO#wtBiw6fp<#np1 z8JJ`1wMTXJgTf7krQ1YLpYJ!R^Wr{Y^*p}1Ual#QodpIg9Z~V$l%X$Uau1ZlUDQE= z5K?h5qMJ4WK}QZTo3`4a2xa}; zqOm*`W@~&i!Z;39 zjREUD%`tiuH&)`t+^ZP5T06RVgA~LMHQEF+&Rs1yNh3#QqoWg}XT(U4^$kigpSCoA zO`F;egPYI^Q>ismeBeJ`PQW`_UP*|`?$nfr%v*0us_K23wR5n+jyHfKq`aX>i#$ud zMNyzHvcCA*Nqw1En7yhL;7t@jCfA=R!*RwRlb+%V#vQ%J?Sd2NS8hr4XLC^-i$Epp zqms*wiu+1;sqD7gZ-KgrhCv_ec4SAQ-k2SYy_)Xyf`yMyfM5S@E;++q^(RSoMBx34 zHD`6Ei@?uw7qCrym?MJSj#$G$`T`3ZuF?w0;^&EPH@-JYA9DM(&*piv>w}gl9A%|{ zOHXG7#r2D*XEO!A+y;+NA&lF9Kx$Q`?01DJZX_#aQP4ypWk7OplSMivLW&0BrHHks z7m-+Uo-_*Y|AB*T&u#Oo0m&||S5_Bua>X_7 zVVPSNj8A?NOA8&T_RSRWume7SMSximo57Po8WS~}P{@7*P-{`S561|uIgk6QvoSMO zXr>_>L*xB^W{Y~si+=oNFvC)zZsTu#2}%jl7#e|N5eH}*3s{wp0l|_UNSuw+up>V& z+9XHLS|T~Zd;z+GaBR+G(=1Tpgg|>#WIi?%4DJCQw6q*6gv~J;iC)5~%N4flG{%^4 zB4U3Wu26a`2Bgr@=z!&b2YjpY(~Cz!lg;mUq-4XliP;Q0vkrmb942*}fwEE+&kiH9!*#B6- zf&bX5Lj?bwDYRTIgIygRDFYKGFuREh_QeibYFC;y?*)I5z(!=4_C}y`rc+TOyQcGB_!Nq{w@>y z(MTw5*2r$u!82<~A}4Xm9g8j!--SP@Hu7tji%G0QOgcnF2J<9XfT}?vctZ)v)fEe= zd3haBC~F`WnHBGeUX|RE;GQIBOF}2FLubLIv3VbEs@20Cumm)~;9IOx+TsmsCkbr# zw*EC;QC9aFxxD%AHIZOyS!3JJf3)K$Yv;NtSxrxS+3JG6{ zNZLAK8f?0+J(k-S5pi?X06M`#Nui`@EC#xy7w9Ni3mG!f3tK}fG7j4CqyZ>KE_x2h zj;2kl+pI9Ey?IHogK!#V7YCRm;##t$U?twHfZE$Iyw?b_ZlB^gg)0GGRwWs%seLe} zZD63T0|;;q^qOzvt-^bAQ+X~t*ATXSkR0v=uxqAR**wdGA@-7k z`I|{8Cll7QitM@xDxRe|jBV{+*Aw3yc2iykn^VBeO7}3%eOa%~=T`g_&;hT=Bh8Na zt_EUDfy{;=l*IIe87)$}9lsTi39SK`o~oW(zGj3ivbXi^_K4&{On^jr{?8xK%mut8*~(Si+dB*o60>trcS120#&h+PE@TSJ5-hsmFc<`Qr0XBRk1>7uy-w$l0V=m*&{hM zF2S|ThkCcXhc!X$@B~>CK(}og+N*UjL@)N9?ay=NFydKWi{0#3!rLmt*Y#n?oiTaI z??E?bEwmv83-5OazYz{@vATa&#tBn;t66Vwf}$}FK;G#CUYy37fQ!s^YEraMjCP_ zAJKUFSoNj@>zE}tr>@ecGRfPjH~oI3Rw4ptH=Ta3!butL3;8N#;e(<@gLE>jw+fzj zueB}_NA;a%h!6TWP9laivLh@uj+)qFS!B!(uUPPJ0O(oa3#(-)|#{E#FC)>P-m&p+1RZk%-- zj2f^LSH7G-!rf8o##FQzgQ+fmEQ-d7_Rx0U3}KM@l>e!yexfwFgDFTu^@{Nio0JW~ z*OQnm>s1eyEYUVyDF%~^bAuKT$u2dZ6;bp83crcTpjZ}f6Ukjro5C!F{0zfih9aT* z4WDKHa=xbf8p36?P9$=Bvo=rk?%^1D>FH;*p7=q(wGSWcd3u;9vBX`}^>F#{QK|Zz z)|^LN1-x;0Po;#c0DMcND=E}e{&u|@imsX|*G{QDSHLYz<|rlC8cxbPpLKEP0UG%} z)~a{{jKxqtJbLw}9x=YYY}lG*>1i8$J@-ME@}1`w+woC|toVmLDy&e|AYqz1H?Pg`UuiqKGw3&J zCT9bQmV$%NM_=q-hDGMa$h-PTn%BFw`+nV+x~cp=`{Ql`_apJRb!5_-o!VUS9FuUc zF6)vm-D|+!UDSBuJ5Mb&oL`z|8PK&WyRkk){-Ds1jWkCb-xlcMmB=z+?e5>~Up9oK zwz9brcqAalqV}DtAi-|AD`{Yd4{rXoHMpN;Op2+97x%=Ig!Md_Ebd4to-s1rAT3j_ zC`zo>Jv0B87{K*|Lq%Lfv8g~gA53$w9lgA}i24V&1~CSORX#mGW6Irr5ziW5Rmyv% z=bz_Wft)QZCp=FUtUL7NMz4S~MqDmMYx+lEH6-weiXQ7_-;iv{oTfRNpXHC4;wV3^8Wsa(aVu~e@ zE=gkvo)GL|?7d`}y5Ls5WT9U|nKl5zado}Jy_QJ=&xR|#d$~%<#WK=WgyxfD$p>EH zp}jmjc3&glE%W5P7zI4C@W7Ovuj@5w8{#{zSfET+1-VEGEdFdHov8YmvNVXgML~!u z7YMZ$Zx;o^QXSiNfjKuPg8vPZqCnTWhaxV|n1;7gssYc{@s;3tq*Oz;b|>F^w;DrU zDeyN=m87LsjEo7xrPnydd`PJ{R13#3AK&ySpJ-vmx=(^z19)w4N2IVF-WWZ@lA6mO zKrQ)T#@r{d7z58ozJ@=WwI7Q_f$scTn36i171Isjr%f+OY_&e{dQyGGpg{6bSM+%I znGshIM16=#lQ?Um@&n8>temBlJ?qhdoCh$wi zQ11MXVTwOtls=lBmp_DTRJXa=pC+c;c9z!oqR(Rq2Tyi>ikkh3$oeP*0{ZrzVP296 zKkfJ`kTm@6eLKcE$BXC+9hcU`3u6>u=jhC{_|3#}$DE{Gp(V2G2;iQ<=w4)mm#2Au z@^CDMKI*!+fbEaRu^IHK-S^BeRRB>X@ArJ`$xC_q5yx?0_ot~N;tLQg69RKcgKG+Y zT-yX*M^YabaZ#C8A}3c4<%+VKC?;D55{1`v$JwVo0Ivu~WiegIb~t<$AHo*m(J)7m z$NQW3b@?4ill$A{n-x`?R+U{zS^6Hgi>vtxRKkfP@6^}e3oL%mWtte({cq? z;<2;Tq$83ndnxp~Zaic#&0G4oWNagTI|I83`)V|8BHZ1+=b8R0DcJS9r7W8r{0 z*uWP?ir^H`;OfcLjGQ+LF214KsA{V$uxqd_0`Z5L zcJX%2sRvNma&(x%_UNCz+(LG46lAQqtvS3JWDr6pEnf*C9=))-(n^zdAuPR}$&-#r zru4b2pt`x>C_VoVGV-4(HQhXe;er?iz7-YXo zW?X7HM@QeGXA?6^nM7VE`>6`cc zgsB7oM{Aa@s*RoxxT-?&}VHyMJ1gGH#g-=q_t;$*{4qw4GUbJ6utE_vE{?JI zjk3!`a&-BxHiW8NppW4ny7Sa=;$5K3b~A`>A2yDX>EefX-CMw!&p<$&$?+_037mcM zc>TI}hJ$dBhx?qrg~sE#&-~u@Jd^CBWk5}mxpJ4yV16;SvqN>c?I$O%!MPg+THRAr zT_u0tiQz-^dz{QebaE5Upj)10PnUxCKO4!PRlxKZg2vOW3QH{uDEPbf{4^! zE%TMvO+~22jRfB{z%W~ybl$>6_4ri9ciP;&m)PPp-+OY_WU;wlp>}yhf6FNAB*r1s z#4Z^zX*EB>7)N9hL122fGnt}{;~PvqSRByc23-jDeT$$#3=h_xAe-CqQNfwZAHw#^ z7c;iJoCj~|={LHkck$)(*+G73T-)8hxjz-W)ysWbqREd6vIP&PC8SEZa>V&*3YTa#s1= z+D2^t#(`VS?IdG&*AqwY26)lQb7T+6;v3B4K`_)xIaD$j8TttDEdQNP#Oz$<5GGmz z_4C0>u|YJEt|+KE@c5QIZs~MbncDAW^Uh%=4u~1M)boUr*zq{p)?zIBMwhlW$CKn{ zeBR(Hfv?i_%1VLGRSe^Ga0-wltIgO4|F|fe=*PXyar&yQNwR3Gal%7qp-0S%65}PT z<{AwO+sdCB3}gzRds=>}IIrBVRCSf{mQmfHuHA&2AZP$m8oqF&AJJ9AWVFxVv$0laUSoGCqh!9Q<+UsBS>EtC97I&LHWG<{dY~d6ApAofr1MN*@D{xhI~^RT=vB82PF1B8INgMv0(rO7m33~< zG!|g6I&iyu=^rvT4O)3Y%;I=}g7a9#2dpF4WcE#^z|7;ZbV?&}i4Oz>bpwdBQlWWu zJybsNG45ANwQa;I$BZ~M=>1Fy58J%tG(M>(jp>#umIKj|6w%Po5K>@&0U4?84DA4# z3R$yx9$!ThCj==U}684hF-x9d@ zW4v;~%DFchBCG+Y^Q3YkV_4R%QR&JGBXUTC@;wxVJOC8|gPH9+9EM7>35hGa$)dHp zh5UGj@{T7XuX{+#p)@qa@RZJ`^|HR8W?e=h%rcDymoT;$hHdd%rarAGbjEeWFbRZG zbTlnM^!4iMpbeo_BYdpBwhG@|(CcAt{5i$Jbi(>W*RbZExNRr)k@8{y{3>y-p7Kv) zm}Yg3WdbK|p-iJe);-+@*(3KG>E#e}fqOx58BOMLaK)iG=h#vh%4om1E<}MqMq&Hs z2=mw|T>6)se8uVP?8CimBlyi@sp48jZN$#qL{oPMc-k;6Thgj7^9Xeq)bi-6Y$8)n zEB*RfttTP1?W$3pQ;py9)oKjE!>GxQUK3;h%E@Htrwc)oUC{FxzAMh=M`__&UrPVy z1LFXhY+`a%fv(iG&T&-oyG{z< zlt#8Hl5yzx5Xha;E*N<9NIoN5VFBbq1pT4-=@qkd?3I+bm)*PUGHiB+Cd--q8I^>z z)mQj~mWIqX0v*r6_xNr?&Mnkdh2v+vc8gM^^+O@{p5WB@n3m;L#25QFD}9`obq@pG zkxn?@qG1lc%h;~4TctK`?)};ZaJ|b;aOZq0Zzt{rrO@;7%vAD&K(|Zb55sH`fBPE| zF#?%(sN3w$T5IMKAVy+%!`BGqRr+yuj*f8h)s2H_7oGd?wyM_#^%ceKY#rm{d1i;s z3G2d868~%#+xnHhbEdGg4oxm_)$Dg|A#4Bs&k4c(09eM#BA72NVzJud3Qk%Grri>N zFJ=X$3s`b)>2Ojhs?#>!3KTm=@L&Occy$NYS3j#naaoh)#$$ndj@{DxDfXc(y*GL% zipA4q?%T^O#9QM+_Slc`+O$mF;VI6c5=3+-?qU4Nmy$~hrX^FTWv2vQ`sGR6u1E9P zbAuoFo8}6lbv4DmiHKw9XYg?vTU^sVD^@FNbc_TQ&dWv#VAS10;flFVb1tGmH&?($ zH_HVkw@Z7l7upRCk4y1d_X(%o<6;cps>A*GGAY|tH)rB?qgO#G!C6g9A8_Z45fgRj z_@28{R(!1xbJMxg=B+NqARTgM{M=+qR13)fM*VU{dAHu^rN{(F0RH-mAKF1a20Aig zsE1gZVg@i?e_F#nQ>7FQB^=$;tP?Kdiz-yn%$)UPmPLHN*Ixg<2a{iH)%QAd ziF!J@Dx2mnvtl`bvA)%=(!;U_oibjPjswmwOtNmx8aK#a?K5#5>8@d(UvH?a0Ro4i zbN!fyqhD8)AL-*KN)Kq*QnxV=e8xKWk@x%5k&D7oaOe;T7zrx~Do6vaaSk31;k{bD zN{DRr(pR+TCZN*2vxm%hN=W7;NQShiwx=#NUwzWD<$3kCT=>z2ca_$XtK<%?>{*A3 zJe=D%9C-}F(m@4KlxQo!`Lq0HOq9|UKTEQsSz591JzU3kUuz=pj-7%8Ic16yFqcK* zVp3OT88G`Rw0hwT{uno{#aUh`lay-qE}EqP_U9$vXV3f^qEPN6^xa)xqGGt)2{xev zSHu%`r-b*Gxck&ipoJ7`$QctbCh6@F?BWUu9}v)=CacUN+41$6A6N(}*Z+~#bvukh5hdApnqe0Z9CdL9uiy(_w{4FRbtebpx+>D+GGsmKfbvq4NjIV=gj zArg5!n}wV1-${97cMh_8`NG%*97Bv6OLjo8@l!b7#msvtb9ga#n{W8FqOm;(B$3gFaS&mL&9ch_1b){~dx7F;m9O279Z+cu``##b6?@l;SlS zic)n3@)*5l8DH;L>4M=;&PO$-pcJ6K#X#4);RYE6Ypi(QunD3ag6X_o^EkaGGFK)6 z)cUa>2Gyn%#iO`sC>`YVIF?&IB_VQbmgahW7v6&ND9Dn7&+qsn%lAfS9^7cjF=eC= z!M?W?n4KCs&TEnnekz+TzEVuCvNG3EcU2oa2l5JmyZxG7DQAA>_xisyg);L{usXN6 zj*&d_UGzCY2@q)$<$}`bl(vO&5-<8yY2=FqeC;1-rn;OTR)FL0b&e>%lD;!0$7)I$ z6^=N6en;N0Dpv7_1N*tx>}zg883#PQH3d=!|1{UOG7BQOvbRXqo%_^^%Pj;=2h|e~ zcfw&ve7%3gA*|*-bCJnM@+SFR$(y)$sIB26@k@u-`Np=0CY5^@>Q}vHas5F|*cihc z4A_Qh`I=iZ{0u-=mdBEf=K%hJg=(wxCAtC1a)x}+(m3x(aTvFWDWM98b<$p#z*wm?2E|fYzfYoJi7U@ zGA5FHX^F6a&*rpS#Yrjt6>fge`XjSNk_T9N@=DNbrWC`N!0h}2U}8O`ZV zLv(H!_6MC1Yn4&P6-M}egEU_J0s#ioa6OGOb2Hr~u?MKi5WX-qxltHLpL;)*8ZD;f zg;mS@u%GrReJv`ru5YHx`G~-@L!$KckxMRdx~R{A994@zv}!epmJ(UcnKPXa8M*cS z1=P$l*D8uJ3ofta-@?oN8f>Nz9N9nph^21S3hure?I7F?c)RAkT(66xB*Wc;Np7AA zbD}61b2iNP-wGf{LU9Mv+?Lh!uTzzgZlP^0S?XJC*pX^Muz9Ww^t?E!nC8X3V_k~5gq+IE7F*!i_J+jF=jL+ToWEdu{pQMVWZM7mQ ze=41bT+neb*M=h?9_px@xTVqP&Tp-$i8}uZ7d*zOAb~MwGKRlLroe@&f}#o*4n_%P z3TA{tghCPLehRTM&$J=J=RTYDS?nS|Z8G$W_Du?+3Q;TJaE(Yvn3r6aFAFK)u zIqnMwhx0$AFr`L@TKqH_=TXa?@VU%jbB&eqG#%4(Y~YQ(wWurA$=6Ni?)I((5(sZE z>XvO0c9`B$&s$+tRe9Uo?DoL9Z+^wb=;#8K8Gu?0Ks5#wDjr*x0b5QFJ5Ij*(>~xA z+KsD{jVr5-tNo2D{EaKv_I*U}G0fw1%2ig?)@Luj`1ACX ztGV|5GVigiHCFZ)_N28{dV<^E_mijTc?+zfWd=b`Ezg}!4-Xqx=o?q4?fZnEpWmR_ ziqom9?3t zg9p1c#Aw%@or{Bu17h{dE-%Td=4R*QVP@`N#j0lJ%c|+&Y0kQkU_%J4r0#C1(XorGGqj(dqN)V2Y}3#gorI$ z`FJow+=4wlY$1Nl4C0QC5T9~QGdC|QH%M60&B_eo;|yTXG_y8yv-__I#4KFO%mdPp zlpv%?Ziu@t2Nx$NA2+W)2M0amhTxQ)E&qa`{Qn{U$E@1N&B_|0cMar5;rQ_*_UtZUL%LjzWb^q(RyqpjP@W10A`8XhoAwu8(zyW~}P4K_PIQY4E{+W-1pZDMSAlljgs*mgcfrGU6&zd=Txq1FI7b0%{ z&+m$p_df_Kf2$AD8`r;k!g=v$F?8#vMWt z(SW#dJ6StJs5AdLkb-!a3j@u}%z4cDIJx<_fdYIGrE)HAb3OqMPBR`Jb8B;MOAyNc d4Ef)(@9rLu75pEohMOPA&4ofs3vrT0`Cm!-!o2_h literal 0 HcmV?d00001 diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.extract b/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.extract new file mode 100644 index 0000000..9f54acb --- /dev/null +++ b/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.extract @@ -0,0 +1,11 @@ + +2023-12-03 * "Paycheck" + filing_account: "Income:Salary:FakeCompany" + Assets:Checking:ABCBank 4228.00 USD + Expenses:Taxes:FederalIncome 416.00 USD + Expenses:Taxes:Medicare 128.00 USD + Expenses:Taxes:SocialSecurity 96.00 USD + Expenses:Taxes:StateIncome 32.00 USD + Income:Bonus:FakeCompany -3000.00 USD + Income:Overtime:FakeCompany -300.00 USD + Income:Salary:FakeCompany -1600.00 USD diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_account b/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_account new file mode 100644 index 0000000..e80daef --- /dev/null +++ b/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_account @@ -0,0 +1 @@ +Income:Salary:FakeCompany diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_date b/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_date new file mode 100644 index 0000000..ba67902 --- /dev/null +++ b/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_date @@ -0,0 +1 @@ +2023-12-03 diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_name b/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_name new file mode 100644 index 0000000..0307945 --- /dev/null +++ b/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_name @@ -0,0 +1 @@ +paystub.sample.pdf diff --git a/beancount_reds_importers/libreader/pdfreader.py b/beancount_reds_importers/libreader/pdfreader.py index f9dc6a5..ec8f1a4 100644 --- a/beancount_reds_importers/libreader/pdfreader.py +++ b/beancount_reds_importers/libreader/pdfreader.py @@ -1,7 +1,5 @@ - from pprint import pformat import pdfplumber -import pandas as pd import petl as etl from beancount_reds_importers.libreader import csvreader @@ -12,9 +10,10 @@ BLACK = (0, 0, 0) RED = (255, 0, 0) -PURPLE = (135,0,255) +PURPLE = (135, 0, 255) TRANSPARENT = (0, 0, 0, 0) + class Importer(csvreader.Importer): """ A reader that converts a pdf with tables into a multi-petl-table format understood by transaction builders. @@ -40,15 +39,15 @@ class Importer(csvreader.Importer): self.debug: `boolean` When debug is True a few images and text file are generated: - .debug-pdf-metadata-page_#.png + .debug-pdf-metadata-page_#.png shows the text available in self.meta_text with table data blacked out - + .debug-pdf-table-detection-page_#.png shows the tables detected with cells outlined in red, and the background light blue. The purple box shows where we are looking for the table title. - + .debug-pdf-data.txt is a printout of the meta_text and table data found before being processed into petl tables, as well as some generated helper objects to add to new importers or import configs - + ### Outputs self.meta_text: `str` contains all text found in the document outside of tables @@ -56,21 +55,22 @@ class Importer(csvreader.Importer): self.alltables: `{'table_1': , ...}` contains all the tables found in the document keyed by the extracted title if available, otherwise by the 1-based index in the form of `table_#` """ - FILE_EXTS = ['pdf'] + + FILE_EXTS = ["pdf"] def initialize_reader(self, file): - if getattr(self, 'file', None) != file: - self.pdf_table_extraction_settings = {} + if getattr(self, "file", None) != file: + self.pdf_table_extraction_settings = {} self.pdf_table_extraction_crop = (0, 0, 0, 0) self.pdf_table_title_height = 20 self.pdf_page_break_top = 45 self.debug = False - self.meta_text = '' + self.meta_text = "" self.file = file self.file_read_done = False self.reader_ready = True - + def file_date(self, file): raise "Not implemented, must overwrite, check self.alltables, or self.meta_text for the data" pass @@ -87,23 +87,23 @@ def read_file(self, file): adjusted_crop = ( min(0 + self.pdf_table_extraction_crop[LEFT], page.width), min(0 + self.pdf_table_extraction_crop[TOP], page.height), - max(page.width - self.pdf_table_extraction_crop[RIGHT],0), - max(page.height - self.pdf_table_extraction_crop[BOTTOM],0) + max(page.width - self.pdf_table_extraction_crop[RIGHT], 0), + max(page.height - self.pdf_table_extraction_crop[BOTTOM], 0), ) - + # Debug image image = page.crop(adjusted_crop).to_image() image.debug_tablefinder(tf=self.pdf_table_extraction_settings) - + table_ref = page.crop(adjusted_crop).find_tables(table_settings=self.pdf_table_extraction_settings) - page_tables = [{'table':i.extract(), 'bbox': i.bbox} for i in table_ref] + page_tables = [{"table": i.extract(), "bbox": i.bbox} for i in table_ref] # Get Metadata (all data outside tables) meta_page = page meta_image = meta_page.to_image() for table in page_tables: - meta_page = meta_page.outside_bbox(table['bbox']) - meta_image.draw_rect(table['bbox'], BLACK, RED) + meta_page = meta_page.outside_bbox(table["bbox"]) + meta_image.draw_rect(table["bbox"], BLACK, RED) meta_text = meta_page.extract_text() self.meta_text = self.meta_text + meta_text @@ -111,75 +111,83 @@ def read_file(self, file): # Attach section headers for table_idx, table in enumerate(page_tables): section_title_bbox = ( - table['bbox'][LEFT], - max(table['bbox'][TOP] - self.pdf_table_title_height, 0), - table['bbox'][RIGHT], - table['bbox'][TOP] + table["bbox"][LEFT], + max(table["bbox"][TOP] - self.pdf_table_title_height, 0), + table["bbox"][RIGHT], + table["bbox"][TOP], ) - section_title = meta_page.crop(section_title_bbox).extract_text() - image.draw_rect(section_title_bbox, TRANSPARENT, PURPLE) - page_tables[table_idx]['section'] = section_title + + bbox_area = pdfplumber.utils.calculate_area(section_title_bbox) + if bbox_area > 0: + section_title = meta_page.crop(section_title_bbox).extract_text() + image.draw_rect(section_title_bbox, TRANSPARENT, PURPLE) + page_tables[table_idx]["section"] = section_title + else: + page_tables[table_idx]["section"] = "" + + # replace None with '' + for row_idx, row in enumerate(table["table"]): + page_tables[table_idx]["table"][row_idx] = ["" if v is None else v for v in row] tables = tables + page_tables if self.debug: - image.save('.debug-pdf-table-detection-page_{}.png'.format(page_idx)) - meta_image.save('.debug-pdf-metadata-page_{}.png'.format(page_idx)) + image.save(".debug-pdf-table-detection-page_{}.png".format(page_idx)) + meta_image.save(".debug-pdf-metadata-page_{}.png".format(page_idx)) - # Find and fix page broken tables for table_idx, table in enumerate(tables[:]): if ( - table_idx >= 1 and # if not the first table, - table['bbox'][TOP] < self.pdf_page_break_top and # and the top of the table is close to the top of the page - table['section'] == '' and # and there is no section title - tables[table_idx - 1]['table'][0] == tables[table_idx]['table'][0] # and the header rows are the same, - ): #assume a page break - tables[table_idx - 1]['table'] = tables[table_idx - 1]['table'] + tables[table_idx]['table'][1:] + # if not the first table, + table_idx >= 1 + # and the top of the table is close to the top of the page + and table["bbox"][TOP] < self.pdf_page_break_top + # and there is no section title + and table["section"] == "" + # and the header rows are the same, + and tables[table_idx - 1]["table"][0] == tables[table_idx]["table"][0] + ): # assume a page break + tables[table_idx - 1]["table"] = tables[table_idx - 1]["table"] + tables[table_idx]["table"][1:] del tables[table_idx] continue # if there is no table section give it one - if table['section'] == '': - tables[table_idx]['section'] = 'table_{}'.format(table_idx + 1) - + if table["section"] == "": + tables[table_idx]["section"] = "table_{}".format(table_idx + 1) + if self.debug: # generate helpers paycheck_template = {} header_map = {} for table in tables: - for header in table['table'][0]: - header_map[header]='overwrite_me' - paycheck_template[table['section']] = {} - for row_idx, row in enumerate(table['table']): + for header in table["table"][0]: + header_map[header] = "overwrite_me" + paycheck_template[table["section"]] = {} + for row_idx, row in enumerate(table["table"]): if row_idx == 0: continue - paycheck_template[table['section']][row[0]] = 'overwrite_me' - if not hasattr(self, 'header_map'): - self.header_map = header_map - if not hasattr(self, 'paycheck_template'): - self.paycheck_template = paycheck_template - with open('.debug-pdf-data.txt', "w") as debug_file: - debug_file.write(pformat({ - '_output': { - 'tables':tables, - 'meta_text':self.meta_text - }, - '_input': { - 'table_settings': self.pdf_table_extraction_settings, - 'crop_settings': self.pdf_table_extraction_crop - }, - 'helpers': { - 'header_map':self.header_map, - 'paycheck_template':self.paycheck_template - } - })) - - - - self.alltables = {} - for table in tables: - self.alltables[table['section']] = etl.fromdataframe(pd.DataFrame(table['table'][1:], columns=table['table'][0])) + paycheck_template[table["section"]][row[0]] = "overwrite_me" + with open(".debug-pdf-data.txt", "w") as debug_file: + debug_file.write( + pformat( + { + "_output": {"tables": tables, "meta_text": self.meta_text}, + "_input": { + "table_settings": self.pdf_table_extraction_settings, + "crop_settings": self.pdf_table_extraction_crop, + "pdf_table_title_height": self.pdf_table_title_height, + "pdf_page_break_top": self.pdf_page_break_top, + }, + "helpers": {"header_map_generated": header_map, "paycheck_template_generated": paycheck_template}, + } + ) + ) + self.alltables = {table["section"]: etl.wrap(table["table"]) for table in tables} self.prepare_tables() + + if self.debug: + with open(".debug-pdf-prepared-tables.txt", "w") as debug_file: + debug_file.write(pformat({"prepared_tables": self.alltables})) + self.file_read_done = True diff --git a/requirements.txt b/requirements.txt index ffd5956..4c59517 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,12 @@ # Automatically generated by https://github.com/damnever/pigar. -beancount>=2.3.5 -beautifulsoup4>=4.12.2 -click>=8.1.4 -click-aliases>=1.0.1 -importlib-metadata>=6.8.0 +beancount>=2.3.6 +bs4>=0.0.2 +click-aliases>=1.0.4 +dateparser>=1.2.0 ofxparse>=0.21 openpyxl>=3.1.2 -packaging>=23.1 -pdbpp>=0.10.3 -petl>=1.7.12 -setuptools>=69.0.2 -setuptools-scm>=8.0.4 +pdfplumber>=0.11.0 +petl>=1.7.15 tabulate>=0.9.0 -tomli>=2.0.1 -tqdm>=4.65.0 -typing_extensions>=4.7.1 -xlrd>=2.0.1 +tqdm>=4.66.2 From 45fb8554a08742e758b89e2f542753513dd1c698 Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 3 Apr 2024 18:55:07 -0700 Subject: [PATCH 21/59] fix: one-file-per-account broke with smart_importer #97 --- .../importers/workday/__init__.py | 3 --- .../libtransactionbuilder/banking.py | 6 ------ .../libtransactionbuilder/investments.py | 12 +++--------- .../libtransactionbuilder/paycheck.py | 6 ------ .../libtransactionbuilder/transactionbuilder.py | 11 +++++++++++ 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/beancount_reds_importers/importers/workday/__init__.py b/beancount_reds_importers/importers/workday/__init__.py index ff06c3f..e8c4261 100644 --- a/beancount_reds_importers/importers/workday/__init__.py +++ b/beancount_reds_importers/importers/workday/__init__.py @@ -50,6 +50,3 @@ def valid_header_label(label): for header in table.header(): table = table.rename(header, valid_header_label(header)) self.alltables[section] = table - - def build_metadata(self, file, metatype=None, data={}): - return {"filing_account": self.config["main_account"]} diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index abddbca..b4816b9 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -43,12 +43,6 @@ def build_account_map(self): # } pass - def build_metadata(self, file, metatype=None, data={}): - """This method is for importers to override. The overridden method can - look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) - and the data dictionary to return additional metadata""" - return {} - def match_account_number(self, file_account, config_account): return file_account.endswith(config_account) diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 7dca20d..81b3466 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -141,12 +141,6 @@ def build_account_map(self): ) # fmt: on - def build_metadata(self, file, metatype=None, data={}): - """This method is for importers to override. The overridden method can - look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) - and the data dictionary to return additional metadata""" - return {} - def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 @@ -276,9 +270,9 @@ def generate_trade_entry(self, ot, file, counter): if "sell" in ot.type: units = -1 * abs(ot.units) if not is_money_market: - metadata["todo"] = ( - "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 - ) + metadata[ + "todo" + ] = "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 if ot.type in [ "reinvest" ]: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index b487b07..a7f26ea 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -131,12 +131,6 @@ def build_postings(self, entry): newentry = entry._replace(postings=postings) return newentry - def build_metadata(self, file, metatype=None, data={}): - """This method is for importers to override. The overridden method can - look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) - and the data dictionary to return additional metadata""" - return {} - def extract(self, file, existing_entries=None): self.initialize(file) config = self.config diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index f789e9b..07694eb 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -44,3 +44,14 @@ def set_config_variables(self, substs): self.config["filing_account"] = self.remove_empty_subaccounts( filing_account ) + + def build_metadata(self, file, metatype=None, data={}): + """This method is for importers to override. The overridden method can + look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) + and the data dictionary to return additional metadata""" + + # This 'filing_account' is read by a patch to bean-extract so it can output transactions to + # a file that corresponds with filing_account, when the one-file-per-account feature is + # used. + acct = self.config.get("filing_account", self.config.get("main_account", None)) + return {"filing_account": acct} From 7f387c36740361f87f5b5a5e0d12f3dc5ead4d9a Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 3 Apr 2024 19:15:19 -0700 Subject: [PATCH 22/59] fix: only emit filing account metadata if configured #97 --- beancount_reds_importers/importers/workday/__init__.py | 4 ++++ .../libtransactionbuilder/transactionbuilder.py | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/beancount_reds_importers/importers/workday/__init__.py b/beancount_reds_importers/importers/workday/__init__.py index e8c4261..2a1f08b 100644 --- a/beancount_reds_importers/importers/workday/__init__.py +++ b/beancount_reds_importers/importers/workday/__init__.py @@ -50,3 +50,7 @@ def valid_header_label(label): for header in table.header(): table = table.rename(header, valid_header_label(header)) self.alltables[section] = table + + def build_metadata(self, file, metatype=None, data={}): + acct = self.config.get("filing_account", self.config.get("main_account", None)) + return {"filing_account": acct} diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index 07694eb..c0082ec 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -53,5 +53,9 @@ def build_metadata(self, file, metatype=None, data={}): # This 'filing_account' is read by a patch to bean-extract so it can output transactions to # a file that corresponds with filing_account, when the one-file-per-account feature is # used. - acct = self.config.get("filing_account", self.config.get("main_account", None)) - return {"filing_account": acct} + if self.config.get("emit_filing_account_metadata"): + acct = self.config.get( + "filing_account", self.config.get("main_account", None) + ) + return {"filing_account": acct} + return {} From cc214e2a3831c6795b3d1baf8fe43ccef6241ef9 Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 3 Apr 2024 19:21:01 -0700 Subject: [PATCH 23/59] style: reformat to 99 col width (previously 88 col) - trying this out, and will revert/change based on how it works out - toml only in this commit. reformatting in the next commit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d2d873b..574e924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.ruff] -line-length = 88 +line-length = 99 [tool.ruff.format] docstring-code-format = true From 246427fe8c7635ea02ac10a43e6904953cca4f1d Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 3 Apr 2024 19:21:26 -0700 Subject: [PATCH 24/59] style: reformat to 99 col width (previously 88 col) --- .../importers/amazongc/__init__.py | 4 +- .../importers/dcu/__init__.py | 8 +-- .../importers/fidelity/__init__.py | 8 +-- .../importers/fidelity/fidelity_cma_csv.py | 12 ++--- .../importers/schwab/schwab_csv_brokerage.py | 9 ++-- .../importers/schwab/schwab_csv_checking.py | 4 +- .../importers/schwab/schwab_csv_creditline.py | 4 +- .../importers/stanchart/scbbank.py | 11 ++-- .../importers/stanchart/scbcard.py | 17 ++---- .../importers/unitedoverseas/uobbank.py | 13 ++--- .../importers/unitedoverseas/uobcard.py | 5 +- .../importers/unitedoverseas/uobsrs.py | 13 ++--- .../vanguard/vanguard_screenscrape.py | 4 +- .../libreader/csv_multitable_reader.py | 14 ++--- .../libreader/csvreader.py | 4 +- .../libreader/ofxreader.py | 4 +- beancount_reds_importers/libreader/reader.py | 4 +- .../libtransactionbuilder/banking.py | 8 +-- .../libtransactionbuilder/investments.py | 52 +++++-------------- .../libtransactionbuilder/paycheck.py | 16 ++---- .../transactionbuilder.py | 11 ++-- .../util/bean_download.py | 4 +- beancount_reds_importers/util/needs_update.py | 33 +++--------- .../util/ofx_summarize.py | 8 +-- 24 files changed, 78 insertions(+), 192 deletions(-) diff --git a/beancount_reds_importers/importers/amazongc/__init__.py b/beancount_reds_importers/importers/amazongc/__init__.py index e05e0b6..031e2f1 100644 --- a/beancount_reds_importers/importers/amazongc/__init__.py +++ b/beancount_reds_importers/importers/amazongc/__init__.py @@ -84,9 +84,7 @@ def extract(self, file, existing_entries=None): data.EMPTY_SET, [], ) - data.create_simple_posting( - entry, config["main_account"], number, self.currency - ) + data.create_simple_posting(entry, config["main_account"], number, self.currency) data.create_simple_posting(entry, config["target_account"], None, None) new_entries.append(entry) diff --git a/beancount_reds_importers/importers/dcu/__init__.py b/beancount_reds_importers/importers/dcu/__init__.py index 6837ddf..3a31104 100644 --- a/beancount_reds_importers/importers/dcu/__init__.py +++ b/beancount_reds_importers/importers/dcu/__init__.py @@ -12,7 +12,9 @@ def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = ".*Account_Transactions" self.header_identifier = "" - self.column_labels_line = '"DATE","TRANSACTION TYPE","DESCRIPTION","AMOUNT","ID","MEMO","CURRENT BALANCE"' + self.column_labels_line = ( + '"DATE","TRANSACTION TYPE","DESCRIPTION","AMOUNT","ID","MEMO","CURRENT BALANCE"' + ) self.date_format = "%m/%d/%Y" # fmt: off self.header_map = { @@ -35,6 +37,4 @@ def get_balance_statement(self, file=None): date = self.get_balance_assertion_date() if date: - yield banking.Balance( - date, self.rdr.namedtuples()[0].balance, self.currency - ) + yield banking.Balance(date, self.rdr.namedtuples()[0].balance, self.currency) diff --git a/beancount_reds_importers/importers/fidelity/__init__.py b/beancount_reds_importers/importers/fidelity/__init__.py index d71ff9c..96d869a 100644 --- a/beancount_reds_importers/importers/fidelity/__init__.py +++ b/beancount_reds_importers/importers/fidelity/__init__.py @@ -13,18 +13,14 @@ def custom_init(self): self.max_rounding_error = 0.18 self.filename_pattern_def = ".*fidelity" self.get_ticker_info = self.get_ticker_info_from_id - self.get_payee = ( - lambda ot: ot.memo.split(";", 1)[0] if ";" in ot.memo else ot.memo - ) + self.get_payee = lambda ot: ot.memo.split(";", 1)[0] if ";" in ot.memo else ot.memo def security_narration(self, ot): ticker, ticker_long_name = self.get_ticker_info(ot.security) return f"[{ticker}]" def file_name(self, file): - return "fidelity-{}-{}".format( - self.config["account_number"], ntpath.basename(file.name) - ) + return "fidelity-{}-{}".format(self.config["account_number"], ntpath.basename(file.name)) def get_target_acct_custom(self, transaction, ticker=None): if transaction.memo.startswith("CONTRIBUTION"): diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py index 7380253..d2b9076 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py @@ -13,7 +13,9 @@ def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = ".*History" self.date_format = "%m/%d/%Y" - header_s0 = ".*Run Date,Action,Symbol,Security Description,Security Type,Quantity,Price \\(\\$\\)," + header_s0 = ( + ".*Run Date,Action,Symbol,Security Description,Security Type,Quantity,Price \\(\\$\\)," + ) header_s1 = "Commission \\(\\$\\),Fees \\(\\$\\),Accrued Interest \\(\\$\\),Amount \\(\\$\\),Settlement Date" header_sum = header_s0 + header_s1 self.header_identifier = header_sum @@ -42,12 +44,8 @@ def prepare_raw_columns(self, rdr): for field in ["Action"]: rdr = rdr.convert(field, lambda x: x.lstrip()) - rdr = rdr.capture( - "Action", "(?:\\s)(?:\\w*)(.*)", ["memo"], include_original=True - ) - rdr = rdr.capture( - "Action", "(\\S+(?:\\s+\\S+)?)", ["payee"], include_original=True - ) + rdr = rdr.capture("Action", "(?:\\s)(?:\\w*)(.*)", ["memo"], include_original=True) + rdr = rdr.capture("Action", "(\\S+(?:\\s+\\S+)?)", ["payee"], include_original=True) for field in ["memo", "payee"]: rdr = rdr.convert(field, lambda x: x.lstrip()) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index 1c08ec8..ee01119 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -13,7 +13,9 @@ def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = ".*_Transactions_" self.header_identifier = "" - self.column_labels_line = '"Date","Action","Symbol","Description","Quantity","Price","Fees & Comm","Amount"' + self.column_labels_line = ( + '"Date","Action","Symbol","Description","Quantity","Price","Fees & Comm","Amount"' + ) self.get_ticker_info = self.get_ticker_info_from_id self.date_format = "%m/%d/%Y" self.funds_db_txt = "funds_by_ticker" @@ -59,10 +61,7 @@ def custom_init(self): def deep_identify(self, file): last_three = self.config.get("account_number", "")[-3:] - return ( - re.match(self.header_identifier, file.head()) - and f"XX{last_three}" in file.name - ) + return re.match(self.header_identifier, file.head()) and f"XX{last_three}" in file.name def skip_transaction(self, ot): return ot.type in ["", "Journal"] diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py index 8284364..a26db2c 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_checking.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_checking.py @@ -49,6 +49,4 @@ def get_balance_statement(self, file=None): date = self.get_balance_assertion_date() if date: - yield banking.Balance( - date, self.rdr.namedtuples()[0].balance, self.currency - ) + yield banking.Balance(date, self.rdr.namedtuples()[0].balance, self.currency) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py index 6dc55b7..cac8038 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py @@ -9,4 +9,6 @@ class Importer(schwab_csv_checking.Importer): def custom_init(self): super().custom_init() self.filename_pattern_def = ".*_Transactions_" - self.column_labels_line = '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' + self.column_labels_line = ( + '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' + ) diff --git a/beancount_reds_importers/importers/stanchart/scbbank.py b/beancount_reds_importers/importers/stanchart/scbbank.py index 9d344bb..06f6319 100644 --- a/beancount_reds_importers/importers/stanchart/scbbank.py +++ b/beancount_reds_importers/importers/stanchart/scbbank.py @@ -14,10 +14,10 @@ class Importer(csvreader.Importer, banking.Importer): def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = "AccountTransactions[0-9]*" - self.header_identifier = self.config.get( - "custom_header", "Account transactions shown:" + self.header_identifier = self.config.get("custom_header", "Account transactions shown:") + self.column_labels_line = ( + "Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance" ) - self.column_labels_line = "Date,Transaction,Currency,Deposit,Withdrawal,Running Balance,SGD Equivalent Balance" self.balance_column_labels_line = ( "Account Name,Account Number,Currency,Current Balance,Available Balance" ) @@ -40,10 +40,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): diff --git a/beancount_reds_importers/importers/stanchart/scbcard.py b/beancount_reds_importers/importers/stanchart/scbcard.py index 4dfc4da..7aba70b 100644 --- a/beancount_reds_importers/importers/stanchart/scbcard.py +++ b/beancount_reds_importers/importers/stanchart/scbcard.py @@ -31,10 +31,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() def skip_transaction(self, row): return "[UNPOSTED]" in row.payee @@ -59,19 +56,13 @@ def prepare_table(self, rdr): rdr = rdr.cutout("Foreign Currency Amount") # parse SGD Amount: "SGD 141.02 CR" into a single amount column - rdr = rdr.capture( - "SGD Amount", "(.*) (.*) (.*)", ["currency", "amount", "crdr"] - ) + rdr = rdr.capture("SGD Amount", "(.*) (.*) (.*)", ["currency", "amount", "crdr"]) # change DR into -ve. TODO: move this into csvreader or csvreader.utils crdrdict = {"DR": "-", "CR": ""} - rdr = rdr.convert( - "amount", lambda i, row: crdrdict[row.crdr] + i, pass_row=True - ) + rdr = rdr.convert("amount", lambda i, row: crdrdict[row.crdr] + i, pass_row=True) - rdr = rdr.addfield( - "memo", lambda x: "" - ) # TODO: make this non-mandatory in csvreader + rdr = rdr.addfield("memo", lambda x: "") # TODO: make this non-mandatory in csvreader return rdr def prepare_raw_file(self, rdr): diff --git a/beancount_reds_importers/importers/unitedoverseas/uobbank.py b/beancount_reds_importers/importers/unitedoverseas/uobbank.py index 44fb7e0..753faf6 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobbank.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobbank.py @@ -18,7 +18,9 @@ def custom_init(self): "custom_header", "United Overseas Bank Limited.*Account Type:Uniplus Account", ) - self.column_labels_line = "Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance" + self.column_labels_line = ( + "Transaction Date,Transaction Description,Withdrawal,Deposit,Available Balance" + ) self.date_format = "%d %b %Y" # fmt: off self.header_map = { @@ -32,10 +34,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() # TODO: move these into utils, since this is probably a common operation def prepare_table(self, rdr): @@ -47,9 +46,7 @@ def Ds(x): rdr = rdr.addfield( "amount", - lambda x: -1 * Ds(x["Withdrawal"]) - if x["Withdrawal"] != 0 - else Ds(x["Deposit"]), + lambda x: -1 * Ds(x["Withdrawal"]) if x["Withdrawal"] != 0 else Ds(x["Deposit"]), ) rdr = rdr.addfield("memo", lambda x: "") return rdr diff --git a/beancount_reds_importers/importers/unitedoverseas/uobcard.py b/beancount_reds_importers/importers/unitedoverseas/uobcard.py index e1f840a..5a50f2c 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobcard.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobcard.py @@ -47,10 +47,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() # TODO: move into utils, since this is probably a common operation def prepare_table(self, rdr): diff --git a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py index 3e459ef..ac6a761 100644 --- a/beancount_reds_importers/importers/unitedoverseas/uobsrs.py +++ b/beancount_reds_importers/importers/unitedoverseas/uobsrs.py @@ -17,9 +17,7 @@ def custom_init(self): self.header_identifier = self.config.get( "custom_header", "United Overseas Bank Limited.*Account Type:SRS Account" ) - self.column_labels_line = ( - "Transaction Date,Transaction Description,Withdrawal,Deposit" - ) + self.column_labels_line = "Transaction Date,Transaction Description,Withdrawal,Deposit" self.date_format = "%Y%m%d" # fmt: off self.header_map = { @@ -32,10 +30,7 @@ def custom_init(self): def deep_identify(self, file): account_number = self.config.get("account_number", "") - return ( - re.match(self.header_identifier, file.head()) - and account_number in file.head() - ) + return re.match(self.header_identifier, file.head()) and account_number in file.head() def prepare_table(self, rdr): # Remove carriage returns in description @@ -46,9 +41,7 @@ def Ds(x): rdr = rdr.addfield( "amount", - lambda x: -1 * Ds(x["Withdrawal"]) - if x["Withdrawal"] != "" - else Ds(x["Deposit"]), + lambda x: -1 * Ds(x["Withdrawal"]) if x["Withdrawal"] != "" else Ds(x["Deposit"]), ) rdr = rdr.addfield("memo", lambda x: "") return rdr diff --git a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py index 02ab167..75ba296 100644 --- a/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py +++ b/beancount_reds_importers/importers/vanguard/vanguard_screenscrape.py @@ -59,9 +59,7 @@ def extract_numbers(x): ) rdr = rdr.pushheader(header) - rdr = rdr.addfield( - "action", lambda x: x["description"].rsplit(" ", 2)[1].strip() - ) + rdr = rdr.addfield("action", lambda x: x["description"].rsplit(" ", 2)[1].strip()) for field in [ "date", diff --git a/beancount_reds_importers/libreader/csv_multitable_reader.py b/beancount_reds_importers/libreader/csv_multitable_reader.py index 94a0433..6cf320b 100644 --- a/beancount_reds_importers/libreader/csv_multitable_reader.py +++ b/beancount_reds_importers/libreader/csv_multitable_reader.py @@ -59,18 +59,16 @@ def read_file(self, file): self.raw_rdr = rdr = self.read_raw(file) - rdr = rdr.skip( - getattr(self, "skip_head_rows", 0) - ) # chop unwanted file header rows + rdr = rdr.skip(getattr(self, "skip_head_rows", 0)) # chop unwanted file header rows rdr = rdr.head( len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 ) # chop unwanted file footer rows # [0, 2, 10] <-- starts # [-1, 1, 9] <-- ends - table_starts = [ - i for (i, row) in enumerate(rdr) if self.is_section_title(row) - ] + [len(rdr)] + table_starts = [i for (i, row) in enumerate(rdr) if self.is_section_title(row)] + [ + len(rdr) + ] table_ends = [r - 1 for r in table_starts][1:] table_indexes = zip(table_starts, table_ends) @@ -85,9 +83,7 @@ def read_file(self, file): for section, table in self.alltables.items(): table = table.rowlenselect(0, complement=True) # clean up empty rows - table = table.cut( - *[h for h in table.header() if h] - ) # clean up empty columns + table = table.cut(*[h for h in table.header() if h]) # clean up empty columns self.alltables[section] = table self.prepare_tables() # to be overridden by importer diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 414d54c..92f8611 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -186,9 +186,7 @@ def read_file(self, file): rdr = self.prepare_raw_file(rdr) # extract main table - rdr = rdr.skip( - getattr(self, "skip_head_rows", 0) - ) # chop unwanted header rows + rdr = rdr.skip(getattr(self, "skip_head_rows", 0)) # chop unwanted header rows rdr = rdr.head( len(rdr) - getattr(self, "skip_tail_rows", 0) - 1 ) # chop unwanted footer rows diff --git a/beancount_reds_importers/libreader/ofxreader.py b/beancount_reds_importers/libreader/ofxreader.py index 45fd204..d4b2a69 100644 --- a/beancount_reds_importers/libreader/ofxreader.py +++ b/beancount_reds_importers/libreader/ofxreader.py @@ -167,9 +167,7 @@ def get_balance_assertion_date(self): if not return_date: return None - return return_date + datetime.timedelta( - days=1 - ) # Next day, as defined by Beancount + return return_date + datetime.timedelta(days=1) # Next day, as defined by Beancount def get_max_transaction_date(self): """ diff --git a/beancount_reds_importers/libreader/reader.py b/beancount_reds_importers/libreader/reader.py index 7309978..f93aa2b 100644 --- a/beancount_reds_importers/libreader/reader.py +++ b/beancount_reds_importers/libreader/reader.py @@ -18,9 +18,7 @@ def identify(self, file): # print("No match on extension") return False self.custom_init() - self.filename_pattern = self.config.get( - "filename_pattern", self.filename_pattern_def - ) + self.filename_pattern = self.config.get("filename_pattern", self.filename_pattern_def) if not re.match(self.filename_pattern, path.basename(file.name)): # print("No match on filename_pattern", self.filename_pattern, path.basename(file.name)) return False diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index b4816b9..e3616ae 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -110,9 +110,7 @@ def extract(self, file, existing_entries=None): metadata = data.new_metadata(file.name, next(counter)) # metadata['type'] = ot.type # Optional metadata, useful for debugging #TODO metadata.update( - self.build_metadata( - file, metatype="transaction", data={"transaction": ot} - ) + self.build_metadata(file, metatype="transaction", data={"transaction": ot}) ) # description fields: With OFX, ot.payee tends to be the "main" description field, @@ -149,9 +147,7 @@ def extract(self, file, existing_entries=None): ot.foreign_currency, ) else: - data.create_simple_posting( - entry, main_account, ot.amount, self.get_currency(ot) - ) + data.create_simple_posting(entry, main_account, ot.amount, self.get_currency(ot)) # smart_importer can fill this in if the importer doesn't override self.get_target_acct() target_acct = self.get_target_account(ot) diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 81b3466..2bab989 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -96,9 +96,7 @@ def initialize(self, file): "fund_data" ] # [(ticker, id, long_name), ...] self.funds_by_id = {i: (ticker, desc) for ticker, i, desc in self.fund_data} - self.funds_by_ticker = { - ticker: (ticker, desc) for ticker, _, desc in self.fund_data - } + self.funds_by_ticker = {ticker: (ticker, desc) for ticker, _, desc in self.fund_data} # Most ofx/csv files refer to funds by id (cusip/isin etc.) Some use tickers instead self.funds_db = getattr(self, getattr(self, "funds_db_txt", "funds_by_id")) @@ -199,10 +197,7 @@ def get_target_acct(self, transaction, ticker): target = self.get_target_acct_custom(transaction, ticker) if target: return target - if ( - transaction.type == "income" - and getattr(transaction, "income_type", None) == "DIV" - ): + if transaction.type == "income" and getattr(transaction, "income_type", None) == "DIV": return self.target_account_map.get("dividends", None) return self.target_account_map.get(transaction.type, None) @@ -232,9 +227,7 @@ def get_acct(self, acct, ot, ticker): """Get an account from self.config, resolve variables, and return""" template = self.config.get(acct) if not template: - raise KeyError( - f"{acct} not set in importer configuration. Config: {self.config}" - ) + raise KeyError(f"{acct} not set in importer configuration. Config: {self.config}") return self.subst_acct_vars(template, ot, ticker) # extract() and supporting methods @@ -251,14 +244,9 @@ def generate_trade_entry(self, ot, file, counter): # Build metadata metadata = data.new_metadata(file.name, next(counter)) metadata.update( - self.build_metadata( - file, metatype="transaction_trade", data={"transaction": ot} - ) + self.build_metadata(file, metatype="transaction_trade", data={"transaction": ot}) ) - if ( - getattr(ot, "settleDate", None) is not None - and ot.settleDate != ot.tradeDate - ): + if getattr(ot, "settleDate", None) is not None and ot.settleDate != ot.tradeDate: metadata["settlement_date"] = str(ot.settleDate.date()) narration = self.security_narration(ot) @@ -315,11 +303,7 @@ def generate_trade_entry(self, ot, file, counter): else: # buy stock/fund unit_price = getattr(ot, "unit_price", 0) # annoyingly, vanguard reinvests have ot.unit_price set to zero. so manually compute it - if ( - (hasattr(ot, "security") and ot.security) - and ot.units - and not ot.unit_price - ): + if (hasattr(ot, "security") and ot.security) and ot.units and not ot.unit_price: unit_price = round(abs(ot.total) / ot.units, 4) common.create_simple_posting_with_cost( entry, @@ -357,9 +341,7 @@ def generate_transfer_entry(self, ot, file, counter): config = self.config metadata = data.new_metadata(file.name, next(counter)) metadata.update( - self.build_metadata( - file, metatype="transaction_transfer", data={"transaction": ot} - ) + self.build_metadata(file, metatype="transaction_transfer", data={"transaction": ot}) ) ticker = None date = getattr(ot, "tradeDate", None) @@ -427,9 +409,7 @@ def generate_transfer_entry(self, ot, file, counter): "fee", ]: # cash amount = ot.total if hasattr(ot, "total") else ot.amount - data.create_simple_posting( - entry, config["cash_account"], amount, self.currency - ) + data.create_simple_posting(entry, config["cash_account"], amount, self.currency) data.create_simple_posting(entry, target_acct, -1 * amount, self.currency) else: data.create_simple_posting(entry, main_acct, units, ticker) @@ -503,9 +483,7 @@ def extract_balances_and_prices(self, file, counter): for pos in self.get_balance_positions(): ticker, ticker_long_name = self.get_ticker_info(pos.security) metadata = data.new_metadata(file.name, next(counter)) - metadata.update( - self.build_metadata(file, metatype="balance", data={"pos": pos}) - ) + metadata.update(self.build_metadata(file, metatype="balance", data={"pos": pos})) # if there are no transactions, use the date in the source file for the balance. This gives us the # bonus of an updated, recent balance assertion @@ -526,9 +504,7 @@ def extract_balances_and_prices(self, file, counter): # extract price info if available if hasattr(pos, "unit_price") and hasattr(pos, "date"): metadata = data.new_metadata(file.name, next(counter)) - metadata.update( - self.build_metadata(file, metatype="price", data={"pos": pos}) - ) + metadata.update(self.build_metadata(file, metatype="price", data={"pos": pos})) price_entry = data.Price( metadata, pos.date.date(), @@ -564,13 +540,9 @@ def add_fee_postings(self, entry, ot): config = self.config if hasattr(ot, "fees") or hasattr(ot, "commission"): if getattr(ot, "fees", 0) != 0: - data.create_simple_posting( - entry, config["fees"], ot.fees, self.currency - ) + data.create_simple_posting(entry, config["fees"], ot.fees, self.currency) if getattr(ot, "commission", 0) != 0: - data.create_simple_posting( - entry, config["fees"], ot.commission, self.currency - ) + data.create_simple_posting(entry, config["fees"], ot.commission, self.currency) def add_custom_postings(self, entry, ot): pass diff --git a/beancount_reds_importers/libtransactionbuilder/paycheck.py b/beancount_reds_importers/libtransactionbuilder/paycheck.py index a7f26ea..a6b3b25 100644 --- a/beancount_reds_importers/libtransactionbuilder/paycheck.py +++ b/beancount_reds_importers/libtransactionbuilder/paycheck.py @@ -58,9 +58,7 @@ def flip_if_needed(amount, account): account.startswith(prefix) for prefix in ["Income:", "Equity:", "Liabilities:"] ): amount *= -1 - if amount < 0 and any( - account.startswith(prefix) for prefix in ["Expenses:", "Assets:"] - ): + if amount < 0 and any(account.startswith(prefix) for prefix in ["Expenses:", "Assets:"]): amount *= -1 return amount @@ -81,22 +79,16 @@ def build_postings(self, entry): continue for row in table.namedtuples(): # TODO: 'bank' is workday specific; move it there - row_description = getattr( - row, "description", getattr(row, "bank", None) - ) + row_description = getattr(row, "description", getattr(row, "bank", None)) row_pattern = next( - filter( - lambda ts: row_description.startswith(ts), template[section] - ), + filter(lambda ts: row_description.startswith(ts), template[section]), None, ) if not row_pattern: template_missing[section].add(row_description) else: accounts = template[section][row_pattern] - accounts = ( - [accounts] if not isinstance(accounts, list) else accounts - ) + accounts = [accounts] if not isinstance(accounts, list) else accounts for account in accounts: # TODO: 'amount_in_pay_group_currency' is workday specific; move it there amount = getattr( diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index c0082ec..288998b 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -32,8 +32,7 @@ def set_config_variables(self, substs): } """ self.config = { - k: v.format(**substs) if isinstance(v, str) else v - for k, v in self.config.items() + k: v.format(**substs) if isinstance(v, str) else v for k, v in self.config.items() } # Prevent the replacement fields from appearing in the output of @@ -41,9 +40,7 @@ def set_config_variables(self, substs): if "filing_account" not in self.config: kwargs = {k: "" for k in substs} filing_account = self.config["main_account"].format(**kwargs) - self.config["filing_account"] = self.remove_empty_subaccounts( - filing_account - ) + self.config["filing_account"] = self.remove_empty_subaccounts(filing_account) def build_metadata(self, file, metatype=None, data={}): """This method is for importers to override. The overridden method can @@ -54,8 +51,6 @@ def build_metadata(self, file, metatype=None, data={}): # a file that corresponds with filing_account, when the one-file-per-account feature is # used. if self.config.get("emit_filing_account_metadata"): - acct = self.config.get( - "filing_account", self.config.get("main_account", None) - ) + acct = self.config.get("filing_account", self.config.get("main_account", None)) return {"filing_account": acct} return {} diff --git a/beancount_reds_importers/util/bean_download.py b/beancount_reds_importers/util/bean_download.py index fb05eb8..2cfe3af 100755 --- a/beancount_reds_importers/util/bean_download.py +++ b/beancount_reds_importers/util/bean_download.py @@ -120,9 +120,7 @@ def pverbose(*args, **kwargs): sites = config.sections() if site_types: site_types = site_types.split(",") - sites_lists = [ - get_sites(sites, site_type, config) for site_type in site_types - ] + sites_lists = [get_sites(sites, site_type, config) for site_type in site_types] sites = [j for i in sites_lists for j in i] errors = [] diff --git a/beancount_reds_importers/util/needs_update.py b/beancount_reds_importers/util/needs_update.py index f602dac..ed61da7 100755 --- a/beancount_reds_importers/util/needs_update.py +++ b/beancount_reds_importers/util/needs_update.py @@ -21,22 +21,16 @@ def get_config(entries, args): e for e in entries if isinstance(e, Custom) and e.type == "reds-importers" ] config_meta = { - entry.values[0].value: ( - entry.values[1].value if (len(entry.values) == 2) else None - ) + entry.values[0].value: (entry.values[1].value if (len(entry.values) == 2) else None) for entry in _extension_entries } - config = { - k: ast.literal_eval(v) for k, v in config_meta.items() if "needs-updates" in k - } + config = {k: ast.literal_eval(v) for k, v in config_meta.items() if "needs-updates" in k} config = config.get("needs-updates", {}) if args["all_accounts"]: config["included_account_pats"] = [] config["excluded_account_pats"] = ["$-^"] - included_account_pats = config.get( - "included_account_pats", ["^Assets:", "^Liabilities:"] - ) + included_account_pats = config.get("included_account_pats", ["^Assets:", "^Liabilities:"]) excluded_account_pats = config.get( "excluded_account_pats", ["$-^"] ) # exclude nothing by default @@ -45,11 +39,7 @@ def get_config(entries, args): def is_interesting_account(account, closes): - return ( - account not in closes - and included_re.match(account) - and not excluded_re.match(account) - ) + return account not in closes and included_re.match(account) and not excluded_re.match(account) def handle_commodity_leaf_accounts(last_balance): @@ -104,9 +94,7 @@ def acc_or_parent(acc): def pretty_print_table(not_updated_accounts, sort_by_date): field = 0 if sort_by_date else 1 - output = sorted( - [(v.date, k) for k, v in not_updated_accounts.items()], key=lambda x: x[field] - ) + output = sorted([(v.date, k) for k, v in not_updated_accounts.items()], key=lambda x: x[field]) headers = ["Last Updated", "Account"] print(click.style(tabulate.tabulate(output, headers=headers, **tbl_options))) @@ -118,9 +106,7 @@ def pretty_print_table(not_updated_accounts, sort_by_date): help="How many days ago should the last balance assertion be to be considered old", default=15, ) -@click.option( - "--sort-by-date", help="Sort output by date (instead of account name)", is_flag=True -) +@click.option("--sort-by-date", help="Sort output by date (instead of account name)", is_flag=True) @click.option( "--all-accounts", help="Show all account (ignore include/exclude in config)", @@ -169,9 +155,7 @@ def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts get_config(entries, locals()) closes = [a.account for a in entries if isinstance(a, Close)] balance_entries = [ - a - for a in entries - if isinstance(a, Balance) and is_interesting_account(a.account, closes) + a for a in entries if isinstance(a, Balance) and is_interesting_account(a.account, closes) ] last_balance = {v.account: v for v in balance_entries} d = handle_commodity_leaf_accounts(last_balance) @@ -190,8 +174,7 @@ def accounts_needing_updates(beancount_file, recency, sort_by_date, all_accounts headers = ["Accounts without balance entries:"] print( click.style( - "\n" - + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options) + "\n" + tabulate.tabulate(sorted(accs_no_bal), headers=headers, **tbl_options) ) ) diff --git a/beancount_reds_importers/util/ofx_summarize.py b/beancount_reds_importers/util/ofx_summarize.py index 1bde0ad..e83a9a7 100755 --- a/beancount_reds_importers/util/ofx_summarize.py +++ b/beancount_reds_importers/util/ofx_summarize.py @@ -27,9 +27,7 @@ def analyze(filename, ttype="dividends", pdb_explore=False): @click.command() @click.argument("filename", type=click.Path(exists=True)) -@click.option( - "-n", "--num-transactions", default=5, help="Number of transactions to show" -) +@click.option("-n", "--num-transactions", default=5, help="Number of transactions to show") @click.option("-e", "--pdb-explore", is_flag=True, help="Open a pdb shell to explore") @click.option( "--stats-only", @@ -101,9 +99,7 @@ def summarize(filename, pdb_explore, num_transactions, stats_only): # noqa: C90 if pdb_explore: print("Hints:") print("- try dir(acc), dir(acc.statement.transactions)") - print( - "- try the 'interact' command to start an interactive python interpreter" - ) + print("- try the 'interact' command to start an interactive python interpreter") if len(ofx.accounts) > 1: print("- type 'c' to explore the next account in this file") import pdb From 0db8f8ed3ae3fd85659db35fa0bf85fc7302043a Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 6 Apr 2024 19:13:43 -0700 Subject: [PATCH 25/59] feat: add_custom_postings to banking.py From request here: https://github.com/redstreet/reds-ramblings-comments/issues/21#issuecomment-2041211354 --- beancount_reds_importers/libtransactionbuilder/banking.py | 1 + beancount_reds_importers/libtransactionbuilder/investments.py | 3 --- .../libtransactionbuilder/transactionbuilder.py | 4 ++++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/beancount_reds_importers/libtransactionbuilder/banking.py b/beancount_reds_importers/libtransactionbuilder/banking.py index e3616ae..c97791a 100644 --- a/beancount_reds_importers/libtransactionbuilder/banking.py +++ b/beancount_reds_importers/libtransactionbuilder/banking.py @@ -154,6 +154,7 @@ def extract(self, file, existing_entries=None): if target_acct: data.create_simple_posting(entry, target_acct, None, None) + self.add_custom_postings(entry, ot) new_entries.append(entry) new_entries += self.extract_balance(file, counter) diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 2bab989..f74a3ab 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -544,9 +544,6 @@ def add_fee_postings(self, entry, ot): if getattr(ot, "commission", 0) != 0: data.create_simple_posting(entry, config["fees"], ot.commission, self.currency) - def add_custom_postings(self, entry, ot): - pass - def extract_custom_entries(self, file, counter): """For custom importers to override""" return [] diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index 288998b..d9f9992 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -42,6 +42,10 @@ def set_config_variables(self, substs): filing_account = self.config["main_account"].format(**kwargs) self.config["filing_account"] = self.remove_empty_subaccounts(filing_account) + def add_custom_postings(self, entry, ot): + """This method is for importers to override. Add arbitrary posting to each entry.""" + pass + def build_metadata(self, file, metatype=None, data={}): """This method is for importers to override. The overridden method can look at the metatype ('transaction', 'balance', 'account', 'commodity', etc.) From e82d4c56323e009ae660b538c4705e97e419dbbb Mon Sep 17 00:00:00 2001 From: Ad Timmering Date: Sun, 7 Apr 2024 23:36:14 +0900 Subject: [PATCH 26/59] fix: add file encoding support to csvreader fixed support for importers on non-UTF8 (e.g. Shift-JIS) text files. Fixes #99. --- beancount_reds_importers/libreader/csvreader.py | 7 +++++-- .../libtransactionbuilder/investments.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 92f8611..0bd11af 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -74,7 +74,10 @@ def initialize_reader(self, file): # print(self.header_identifier, file.head()) def deep_identify(self, file): - return re.match(self.header_identifier, file.head()) + return re.match( + self.header_identifier, + file.head(encoding=getattr(self, "file_encoding", None)), + ) def file_date(self, file): "Get the maximum date from the file." @@ -135,7 +138,7 @@ def convert_date(d): return rdr def read_raw(self, file): - return etl.fromcsv(file.name) + return etl.fromcsv(file.name, encoding=getattr(self, "file_encoding", None)) def skip_until_main_table(self, rdr, col_labels=None): """Skip csv lines until the header line is found.""" diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index f74a3ab..12ecd63 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -258,9 +258,9 @@ def generate_trade_entry(self, ot, file, counter): if "sell" in ot.type: units = -1 * abs(ot.units) if not is_money_market: - metadata[ - "todo" - ] = "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 + metadata["todo"] = ( + "TODO: this entry is incomplete until lots are selected (bean-doctor context )" # noqa: E501 + ) if ot.type in [ "reinvest" ]: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI From cc6feaf7abc1331a337b2729c4b5c024e0c4c242 Mon Sep 17 00:00:00 2001 From: Red S Date: Wed, 10 Apr 2024 16:49:00 -0700 Subject: [PATCH 27/59] fix: schwab_csv_creditline balance sign --- .../importers/schwab/schwab_csv_creditline.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py index cac8038..be9e836 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_creditline.py @@ -1,6 +1,7 @@ """Schwab Credit Line (eg: Pledged Asset Line) .csv importer.""" from beancount_reds_importers.importers.schwab import schwab_csv_checking +from beancount_reds_importers.libtransactionbuilder import banking class Importer(schwab_csv_checking.Importer): @@ -12,3 +13,9 @@ def custom_init(self): self.column_labels_line = ( '"Date","Type","CheckNumber","Description","Withdrawal","Deposit","RunningBalance"' ) + + def get_balance_statement(self, file=None): + """Return the balance on the first and last dates""" + + for i in super().get_balance_statement(file): + yield banking.Balance(i.date, -1 * i.amount, i.currency) From 44690951c65c69fbc216055dbb24fb62d25ea51a Mon Sep 17 00:00:00 2001 From: Ammon Sarver Date: Sun, 14 Apr 2024 11:33:41 -0600 Subject: [PATCH 28/59] fix: update requirements to add back lost packages I'm not sure what happened, but pigar would not detect imports from other tests. So I manually updated the requirements.txt to include all needed files. --- .../importers/genericpdf/tests/genericpdf_test.py | 1 - requirements.txt | 12 ++++++++++-- setup.py | 2 ++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py b/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py index 9c64531..8ac67e9 100644 --- a/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py +++ b/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py @@ -2,7 +2,6 @@ from beancount.ingest import regression_pytest as regtest from beancount_reds_importers.importers import genericpdf - @regtest.with_importer( genericpdf.Importer( { diff --git a/requirements.txt b/requirements.txt index 4c59517..53470d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,20 @@ # Automatically generated by https://github.com/damnever/pigar. -beancount>=2.3.6 -bs4>=0.0.2 +beancount>=2.3.5 +beautifulsoup4>=4.12.3 +click>=8.1.7 click-aliases>=1.0.4 dateparser>=1.2.0 +importlib-metadata>=6.8.0 ofxparse>=0.21 openpyxl>=3.1.2 +packaging>=23.1 pdfplumber>=0.11.0 petl>=1.7.15 +setuptools>=69.0.2 +setuptools-scm>=8.0.4 tabulate>=0.9.0 +tomli>=2.0.1 tqdm>=4.66.2 +typing_extensions>=4.7.1 +xlrd>=2.0.1 diff --git a/setup.py b/setup.py index fd3debf..a23c92f 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,11 @@ 'Click >= 7.0', 'beancount >= 2.3.5', 'click_aliases >= 1.0.1', + 'dateparser >= 1.2.0', 'ofxparse >= 0.21', 'openpyxl >= 3.0.9', 'packaging >= 20.3', + 'pdfplumber>=0.11.0', 'petl >= 1.7.4', 'tabulate >= 0.8.9', 'tqdm >= 4.64.0', From fbba9a083751c9907f0fe7491787cf963bc931ac Mon Sep 17 00:00:00 2001 From: Red S Date: Mon, 15 Apr 2024 22:39:10 -0700 Subject: [PATCH 29/59] ci: enable workflows to run automatically on PRs --- .github/workflows/conventionalcommits.yml | 1 + .github/workflows/pythonpackage.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/conventionalcommits.yml b/.github/workflows/conventionalcommits.yml index add48c6..ae2d6a7 100644 --- a/.github/workflows/conventionalcommits.yml +++ b/.github/workflows/conventionalcommits.yml @@ -3,6 +3,7 @@ name: Conventional Commits on: pull_request: branches: [ main ] + types: [opened, reopened, edited] jobs: build: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 0312cb2..05a9b9a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -8,6 +8,7 @@ on: branches: [ main ] pull_request: branches: [ main ] + types: [opened, reopened, edited] jobs: build: From 6432b2025455657686f1cec9f66edfc8223f8597 Mon Sep 17 00:00:00 2001 From: Red S Date: Mon, 15 Apr 2024 22:39:10 -0700 Subject: [PATCH 30/59] ci: enable workflows to run automatically on PRs --- .github/workflows/conventionalcommits.yml | 1 + .github/workflows/pythonpackage.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/conventionalcommits.yml b/.github/workflows/conventionalcommits.yml index add48c6..ae2d6a7 100644 --- a/.github/workflows/conventionalcommits.yml +++ b/.github/workflows/conventionalcommits.yml @@ -3,6 +3,7 @@ name: Conventional Commits on: pull_request: branches: [ main ] + types: [opened, reopened, edited] jobs: build: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 0312cb2..05a9b9a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -8,6 +8,7 @@ on: branches: [ main ] pull_request: branches: [ main ] + types: [opened, reopened, edited] jobs: build: From ebbcfeb7e1cd8bb1f1d948abba3d405c5cf0c0c1 Mon Sep 17 00:00:00 2001 From: Ammon Sarver Date: Tue, 16 Apr 2024 10:46:56 -0600 Subject: [PATCH 31/59] chore: formatting --- .../importers/bamboohr/__init__.py | 2 ++ .../importers/genericpdf/__init__.py | 1 + .../genericpdf/tests/genericpdf_test.py | 7 +++--- .../libreader/pdfreader.py | 19 ++++++++++++---- setup.py | 22 +++++++++---------- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/beancount_reds_importers/importers/bamboohr/__init__.py b/beancount_reds_importers/importers/bamboohr/__init__.py index 6d05acc..a1ac1bc 100644 --- a/beancount_reds_importers/importers/bamboohr/__init__.py +++ b/beancount_reds_importers/importers/bamboohr/__init__.py @@ -1,7 +1,9 @@ """BambooHR paycheck importer""" import re + from dateparser.search import search_dates + from beancount_reds_importers.libreader import pdfreader from beancount_reds_importers.libtransactionbuilder import paycheck diff --git a/beancount_reds_importers/importers/genericpdf/__init__.py b/beancount_reds_importers/importers/genericpdf/__init__.py index b5f7595..91b210c 100644 --- a/beancount_reds_importers/importers/genericpdf/__init__.py +++ b/beancount_reds_importers/importers/genericpdf/__init__.py @@ -1,6 +1,7 @@ """Generic pdf paycheck importer""" import datetime + from beancount_reds_importers.libreader import pdfreader from beancount_reds_importers.libtransactionbuilder import paycheck diff --git a/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py b/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py index 8ac67e9..2d9eaa2 100644 --- a/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py +++ b/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py @@ -1,7 +1,10 @@ from os import path + from beancount.ingest import regression_pytest as regtest + from beancount_reds_importers.importers import genericpdf + @regtest.with_importer( genericpdf.Importer( { @@ -19,9 +22,7 @@ "Federal Withholding": "Expenses:Taxes:FederalIncome", "State Withholding": "Expenses:Taxes:StateIncome", }, - "table_6": { - "CURRENT": "Assets:Checking:ABCBank" - } + "table_6": {"CURRENT": "Assets:Checking:ABCBank"}, }, "currency": "USD", } diff --git a/beancount_reds_importers/libreader/pdfreader.py b/beancount_reds_importers/libreader/pdfreader.py index ec8f1a4..20a2815 100644 --- a/beancount_reds_importers/libreader/pdfreader.py +++ b/beancount_reds_importers/libreader/pdfreader.py @@ -1,6 +1,8 @@ from pprint import pformat + import pdfplumber import petl as etl + from beancount_reds_importers.libreader import csvreader LEFT = 0 @@ -95,7 +97,9 @@ def read_file(self, file): image = page.crop(adjusted_crop).to_image() image.debug_tablefinder(tf=self.pdf_table_extraction_settings) - table_ref = page.crop(adjusted_crop).find_tables(table_settings=self.pdf_table_extraction_settings) + table_ref = page.crop(adjusted_crop).find_tables( + table_settings=self.pdf_table_extraction_settings + ) page_tables = [{"table": i.extract(), "bbox": i.bbox} for i in table_ref] # Get Metadata (all data outside tables) @@ -127,7 +131,9 @@ def read_file(self, file): # replace None with '' for row_idx, row in enumerate(table["table"]): - page_tables[table_idx]["table"][row_idx] = ["" if v is None else v for v in row] + page_tables[table_idx]["table"][row_idx] = [ + "" if v is None else v for v in row + ] tables = tables + page_tables @@ -147,7 +153,9 @@ def read_file(self, file): # and the header rows are the same, and tables[table_idx - 1]["table"][0] == tables[table_idx]["table"][0] ): # assume a page break - tables[table_idx - 1]["table"] = tables[table_idx - 1]["table"] + tables[table_idx]["table"][1:] + tables[table_idx - 1]["table"] = ( + tables[table_idx - 1]["table"] + tables[table_idx]["table"][1:] + ) del tables[table_idx] continue @@ -178,7 +186,10 @@ def read_file(self, file): "pdf_table_title_height": self.pdf_table_title_height, "pdf_page_break_top": self.pdf_page_break_top, }, - "helpers": {"header_map_generated": header_map, "paycheck_template_generated": paycheck_template}, + "helpers": { + "header_map_generated": header_map, + "paycheck_template_generated": paycheck_template, + }, } ) ) diff --git a/setup.py b/setup.py index 78474f3..2018a5c 100644 --- a/setup.py +++ b/setup.py @@ -26,17 +26,17 @@ ] }, install_requires=[ - 'Click >= 7.0', - 'beancount >= 2.3.5', - 'click_aliases >= 1.0.1', - 'dateparser >= 1.2.0', - 'ofxparse >= 0.21', - 'openpyxl >= 3.0.9', - 'packaging >= 20.3', - 'pdfplumber >= 0.11.0', - 'petl >= 1.7.4', - 'tabulate >= 0.8.9', - 'tqdm >= 4.64.0', + "Click >= 7.0", + "beancount >= 2.3.5", + "click_aliases >= 1.0.1", + "dateparser >= 1.2.0", + "ofxparse >= 0.21", + "openpyxl >= 3.0.9", + "packaging >= 20.3", + "pdfplumber>=0.11.0", + "petl >= 1.7.4", + "tabulate >= 0.8.9", + "tqdm >= 4.64.0", ], entry_points={ "console_scripts": [ From b674597a60e6b0f9736fd4b92573428246b066c9 Mon Sep 17 00:00:00 2001 From: Red S Date: Mon, 13 May 2024 21:01:08 -0700 Subject: [PATCH 32/59] ci: fix CHANGELOG version --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8ac163..9d92c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## (unreleased) +## 0.8.0 (2023-11-24) ### Improvements From 2d47a25ae876fc6f1b9b81066fcddc1acf433fae Mon Sep 17 00:00:00 2001 From: Red S Date: Tue, 14 May 2024 01:18:36 -0700 Subject: [PATCH 33/59] ci: CHANGELOG --- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d92c6d..806c02e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,47 @@ # Changelog +## 0.9.0 (2024-05-14) + +### New Features + +- add genericpdf paycheck importer. [Ammon Sarver] +- add bamboohr paycheck importer. [Ammon Sarver] +- add pdfreader libreader importer. [Ammon Sarver] +- minor: add 'show_configured' in paycheck transaction builder. [Red S] + +### New Importers +- add CSV importer for Digital Credit Union (#89) [Harlan Lieberman-Berg] +- add importer for Alliant Federal Credit Union (#88) [Harlan Lieberman-Berg] +- add schwab csv credit line importer. [Red S] + +### Improvements + +- minor: add_custom_postings to banking.py. [Red S] +- minor: add identification based on filename to schwab importers. [Red S] +- minor: overridable add_custom_postings() [Red S] +- minor: add tdameritrade to template.cfg. [Red S] + + +### Fixes + +- update requirements to add back lost packages. [Ammon Sarver] +- schwab_csv_creditline balance sign. [Red S] +- add file encoding support to csvreader. [Ad Timmering] +- only emit filing account metadata if configured #97. [Red S] +- one-file-per-account broke with smart_importer #97. [Red S] +- timestamp issue in balance assertions. [Red S] +- balance date on test. [Red S] +- schwab tests. [Red S] +- schwab doesn't use a header in their csv any more. [Red S] +- schwab_csv_checking format changed #91. [Red S] + +### Other +- enforce formatting with ruff. [Rane Brown] +- format with isort, use pyproject.toml. [Rane Brown] +- switch to ruff for linting (#90) [Harlan Lieberman-Berg] +- style: reformat to 99 col width (previously 88 col) [Red S] + + ## 0.8.0 (2023-11-24) From 1f1943c29d66710e195a57993e1feec294feaf27 Mon Sep 17 00:00:00 2001 From: Red S Date: Mon, 10 Jun 2024 15:33:14 -0700 Subject: [PATCH 34/59] fix: add interest to schwab_csv_brokerage --- .../importers/schwab/schwab_csv_brokerage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index ee01119..e5c5a3c 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -34,6 +34,7 @@ def custom_init(self): } self.transaction_type_map = { "Bank Interest": "income", + "Credit Interest": "income", "Bank Transfer": "cash", "Buy": "buystock", "Journaled Shares": "buystock", # These are in-kind tranfers From be7f43a30ed9bc3d42c45703f7a5624891af5bc0 Mon Sep 17 00:00:00 2001 From: Red S Date: Sun, 14 Jul 2024 18:24:50 -0700 Subject: [PATCH 35/59] feat: xml reader + ibkr flex query importer --- .../importers/ibkr/__init__.py | 162 ++++++++++++++++++ beancount_reds_importers/libreader/reader.py | 9 +- .../libreader/xmlreader.py | 58 +++++++ 3 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 beancount_reds_importers/importers/ibkr/__init__.py create mode 100644 beancount_reds_importers/libreader/xmlreader.py diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py new file mode 100644 index 0000000..80fd50c --- /dev/null +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -0,0 +1,162 @@ +"""IBKR Flex Query importer for beancount. + +Activity Flex Query Details +Query ID XXX +Query Name XXX + +Sections +======== + +Account Information +------------------- +1.ClientAccountID +2.CurrencyPrimary + +Cash Transactions +----------------- + +Options: Dividends, Payment in Lieu of Dividends, Withholding Tax, 871(m) Withholding, Advisor Fees, Other Fees, Deposits & Withdrawals, Carbon Credits, Bill Pay, Broker Interest Paid, Broker Interest Received, Broker Fees, Bond Interest Paid, Bond Interest Received, Price Adjustments, Commission Adjustments, Detail + +1.Date/Time +2.Amount +3.Type +4.CurrencyPrimary +5.Symbol +6.CommodityType +7.ISIN + +Net Stock Position Summary +-------------------------- +1.Symbol +2.CUSIP + +Open Dividend Accruals +---------------------- +1.Symbol +2.GrossAmount +3.NetAmount +4.PayDate +5.Quantity +6.ISIN + +Trades +------ +Options: Execution +1.SecurityID +2.DateTime +3.TransactionType +4.Quantity +5.TradePrice +6.TradeMoney +7.Proceeds +8.IBCommission +9.IBCommissionCurrency +10.NetCash +11.CostBasis +12.FifoPnlRealized +13.Buy/Sell +14.CurrencyPrimary +15.ISIN + + +Delivery Configuration +---------------------- +Accounts +Format XML +Period Last N Calendar Days +Number of Days 120 + + +General Configuration +--------------------- +Profit and Loss Default +Include Canceled Trades? No +Include Currency Rates? No +Include Audit Trail Fields? No +Display Account Alias in Place of Account ID? No +Breakout by Day? No +Date Format yyyy-MM-dd +Time Format HH:mm:ss TimeZone +Date/Time Separator ' ' (single-space) + +""" + +import datetime +from beancount_reds_importers.libreader import xmlreader +from beancount_reds_importers.libtransactionbuilder import investments +from beancount.core.number import D + +class DictToObject: + def __init__(self, dictionary): + for key, value in dictionary.items(): + setattr(self, key, value) + +# xml on left, ofx on right +ofx_type_map = { + 'BUY': 'buystock', + 'SELL': 'selltock', +} + + +class Importer(investments.Importer, xmlreader.Importer): + IMPORTER_NAME = "IBKR Flex Query" + + def custom_init(self): + if not self.custom_init_run: + self.max_rounding_error = 0.04 + self.filename_pattern_def = "Transaction_report" + self.custom_init_run = True + self.date_format = '%Y-%m-%d' + self.get_ticker_info = self.get_ticker_info_from_id + + def set_currency(self): + self.currency = list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['currency'] + + # fixup dates + def convert_date(self, d): + d = d.split(' ')[0] + return datetime.datetime.strptime(d, self.date_format) + + def trade_to_ofx_dict(self, xml_data): + # Mapping the input dictionary to the OFX dictionary format + ofx_dict = { + 'security': xml_data['isin'], + 'tradeDate': self.convert_date(xml_data['dateTime']), + 'memo': xml_data['transactionType'], + 'type': ofx_type_map[xml_data['buySell']], + 'units': D(xml_data['quantity']), + 'unit_price': D(xml_data['tradePrice']), + 'commission': D(xml_data['ibCommission']), + 'total': D(xml_data['proceeds']), + } + return ofx_dict + + def cash_to_ofx_dict(self, xml_data): + # Mapping the input dictionary to the OFX dictionary format + ofx_dict = { + 'tradeDate': self.convert_date(xml_data['dateTime']), + 'amount': D(xml_data['amount']), + 'security': getattr(xml_data, 'isin', None), + 'type': 'cash', + 'memo': xml_data['type'], + } + + if xml_data['type'] == 'Dividends': + ofx_dict['type'] = 'dividends' + ofx_dict['total'] = ofx_dict['amount'] + + return ofx_dict + + def xml_trade_interpreter(self, element): + ot = self.trade_to_ofx_dict(element) + return DictToObject(ot) + + def xml_cash_interpreter(self, element): + ot = self.cash_to_ofx_dict(element) + return DictToObject(ot) + + def get_transactions(self): + yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade', + xml_interpreter=self.xml_trade_interpreter) + yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction', + xml_interpreter=self.xml_cash_interpreter) diff --git a/beancount_reds_importers/libreader/reader.py b/beancount_reds_importers/libreader/reader.py index f93aa2b..98ef183 100644 --- a/beancount_reds_importers/libreader/reader.py +++ b/beancount_reds_importers/libreader/reader.py @@ -24,9 +24,13 @@ def identify(self, file): return False self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED") self.initialize_reader(file) - # print("reader_ready:", self.reader_ready) + # print("reader_ready:", self.reader_ready, self.IMPORTER_NAME) return self.reader_ready + def set_currency(self): + """For overriding""" + self.currency = self.config.get("currency", "CURRENCY_NOT_CONFIGURED") + def file_name(self, file): return "{}".format(ntpath.basename(file.name)) @@ -55,5 +59,8 @@ def get_balance_statement(self, file=None): def get_balance_positions(self): return [] + def get_balance_assertion_date(self): + return None + def get_available_cash(self, settlement_fund_balance=0): return None diff --git a/beancount_reds_importers/libreader/xmlreader.py b/beancount_reds_importers/libreader/xmlreader.py new file mode 100644 index 0000000..718491f --- /dev/null +++ b/beancount_reds_importers/libreader/xmlreader.py @@ -0,0 +1,58 @@ +"""XML reader for beancount-reds-importers. + +XML files have widely varying specifications, and thus, this is a very generic reader, and most of +the logic will have to be the institution specific readers. + +""" + +import datetime +import warnings +from collections import namedtuple +from lxml import etree + +from beancount.ingest import importer +from beancount_reds_importers.libreader import reader + + + +class Importer(reader.Reader, importer.ImporterProtocol): + FILE_EXTS = ["xml"] + + def initialize_reader(self, file): + if getattr(self, "file", None) != file: + self.file = file + self.reader_ready = False + try: + self.xmltree = etree.parse(file.name) + except: + return + self.reader_ready = self.deep_identify() + if self.reader_ready: + self.set_currency() + + def deep_identify(self): + """For overriding by institution specific importer which can check if an account name + matches, and oother such things.""" + return True + + def file_date(self, file): + """Get the ending date of the statement.""" + if not getattr(self, "xmltree", None): + self.initialize(file) + # TODO: + return None + + def read_file(self, file): + self.xmltree = etree.parse(file.name) + + def get_xpath_elements(self, xpath_expr, xml_interpreter=lambda x: x): + """Extract a list of elements in the XML file at the given XPath expression. Typically, + transactions are stored in an xml path, and this extracts them.""" + elements = self.xmltree.xpath(xpath_expr) + for elem in elements: + yield xml_interpreter(elem.attrib) + + def get_transactions(self): + """/Transactions/Transaction is a dummy default path for transactions that needs to be + overriden in the institution specific importer.""" + yield from self.get_xpath_elements("/Transactions/Transaction") From 2d4aaea8304fa507a8a532d809079db4a19cbcc9 Mon Sep 17 00:00:00 2001 From: Red S Date: Sun, 14 Jul 2024 20:19:56 -0700 Subject: [PATCH 36/59] feat: ibkr balance assertion for cash balance --- .../importers/ibkr/__init__.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py index 80fd50c..4eab64d 100644 --- a/beancount_reds_importers/importers/ibkr/__init__.py +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -1,5 +1,10 @@ """IBKR Flex Query importer for beancount. +TODO: +- balance assertions for positions +- balance assertions for cash +- Flex Web Service API to programmatically retrieve all of this + Activity Flex Query Details Query ID XXX Query Name XXX @@ -136,7 +141,7 @@ def cash_to_ofx_dict(self, xml_data): ofx_dict = { 'tradeDate': self.convert_date(xml_data['dateTime']), 'amount': D(xml_data['amount']), - 'security': getattr(xml_data, 'isin', None), + 'security': xml_data.get('isin', None), 'type': 'cash', 'memo': xml_data['type'], } @@ -160,3 +165,14 @@ def get_transactions(self): xml_interpreter=self.xml_trade_interpreter) yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction', xml_interpreter=self.xml_cash_interpreter) + + def get_balance_assertion_date(self): + ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0] + return self.convert_date(ac['toDate']).date() + + def get_available_cash(self, settlement_fund_balance=0): + """Assumes there's only one cash currency. + TODO: get investments transaction builder to accept date from get_available_cash + """ + ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0] + return D(ac['slbNetCash']) From e06f53bad61c25a5f9801b8b6cf391241fb5f0e9 Mon Sep 17 00:00:00 2001 From: Red S Date: Sun, 14 Jul 2024 20:20:16 -0700 Subject: [PATCH 37/59] fix: ibkr trade commissions fix --- beancount_reds_importers/importers/ibkr/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py index 4eab64d..b166fd3 100644 --- a/beancount_reds_importers/importers/ibkr/__init__.py +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -131,8 +131,8 @@ def trade_to_ofx_dict(self, xml_data): 'type': ofx_type_map[xml_data['buySell']], 'units': D(xml_data['quantity']), 'unit_price': D(xml_data['tradePrice']), - 'commission': D(xml_data['ibCommission']), - 'total': D(xml_data['proceeds']), + 'commission': -1 * D(xml_data['ibCommission']), + 'total': D(xml_data['netCash']), } return ofx_dict From f7d086420894edfca6fea444bc718dd9aa5a45b3 Mon Sep 17 00:00:00 2001 From: Red S Date: Sun, 14 Jul 2024 20:31:41 -0700 Subject: [PATCH 38/59] feat: IBKR add balance assertions for positions --- beancount_reds_importers/importers/ibkr/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py index b166fd3..a21015e 100644 --- a/beancount_reds_importers/importers/ibkr/__init__.py +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -176,3 +176,11 @@ def get_available_cash(self, settlement_fund_balance=0): """ ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0] return D(ac['slbNetCash']) + + def get_balance_positions(self): + for pos in self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/OpenPositions/OpenPosition'): + balance = { + 'security': pos['isin'], + 'units': D(pos['position']), + } + yield DictToObject(balance) From 95d4ffca6dfde038dfdf1ce27e01b3651075d003 Mon Sep 17 00:00:00 2001 From: Red S Date: Sun, 14 Jul 2024 20:35:20 -0700 Subject: [PATCH 39/59] ci: flake --- beancount_reds_importers/importers/ibkr/__init__.py | 7 ++++++- beancount_reds_importers/importers/vanguard/__init__.py | 2 +- beancount_reds_importers/libreader/xmlreader.py | 9 +-------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py index a21015e..5d8c115 100644 --- a/beancount_reds_importers/importers/ibkr/__init__.py +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -20,7 +20,10 @@ Cash Transactions ----------------- -Options: Dividends, Payment in Lieu of Dividends, Withholding Tax, 871(m) Withholding, Advisor Fees, Other Fees, Deposits & Withdrawals, Carbon Credits, Bill Pay, Broker Interest Paid, Broker Interest Received, Broker Fees, Bond Interest Paid, Bond Interest Received, Price Adjustments, Commission Adjustments, Detail +Options: Dividends, Payment in Lieu of Dividends, Withholding Tax, 871(m) Withholding, +Advisor Fees, Other Fees, Deposits & Withdrawals, Carbon Credits, Bill Pay, Broker +Interest Paid, Broker Interest Received, Broker Fees, Bond Interest Paid, Bond Interest +Received, Price Adjustments, Commission Adjustments, Detail 1.Date/Time 2.Amount @@ -91,11 +94,13 @@ from beancount_reds_importers.libtransactionbuilder import investments from beancount.core.number import D + class DictToObject: def __init__(self, dictionary): for key, value in dictionary.items(): setattr(self, key, value) + # xml on left, ofx on right ofx_type_map = { 'BUY': 'buystock', diff --git a/beancount_reds_importers/importers/vanguard/__init__.py b/beancount_reds_importers/importers/vanguard/__init__.py index 76f48e9..47faab5 100644 --- a/beancount_reds_importers/importers/vanguard/__init__.py +++ b/beancount_reds_importers/importers/vanguard/__init__.py @@ -45,7 +45,7 @@ def cleanup_memo(self, ot): # some vanguard files have memos repeated like this: # 'DIVIDEND REINVESTMENTDIVIDEND REINVESTMENT' retval = ot.memo - if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2) :]: + if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2):]: retval = ot.memo[: int(len(ot.memo) / 2)] return retval diff --git a/beancount_reds_importers/libreader/xmlreader.py b/beancount_reds_importers/libreader/xmlreader.py index 718491f..9667f9d 100644 --- a/beancount_reds_importers/libreader/xmlreader.py +++ b/beancount_reds_importers/libreader/xmlreader.py @@ -5,16 +5,12 @@ """ -import datetime -import warnings -from collections import namedtuple from lxml import etree from beancount.ingest import importer from beancount_reds_importers.libreader import reader - class Importer(reader.Reader, importer.ImporterProtocol): FILE_EXTS = ["xml"] @@ -22,10 +18,7 @@ def initialize_reader(self, file): if getattr(self, "file", None) != file: self.file = file self.reader_ready = False - try: - self.xmltree = etree.parse(file.name) - except: - return + self.xmltree = etree.parse(file.name) self.reader_ready = self.deep_identify() if self.reader_ready: self.set_currency() From ef1da4a9a0e3198f66a3314327cc7112f31a63d5 Mon Sep 17 00:00:00 2001 From: Red S Date: Tue, 16 Jul 2024 01:10:02 -0700 Subject: [PATCH 40/59] doc: ibkr flex query config --- .../importers/ibkr/__init__.py | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py index 5d8c115..ddab13c 100644 --- a/beancount_reds_importers/importers/ibkr/__init__.py +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -1,30 +1,23 @@ """IBKR Flex Query importer for beancount. TODO: -- balance assertions for positions -- balance assertions for cash -- Flex Web Service API to programmatically retrieve all of this +- Flex Web Service API to programmatically retrieve data Activity Flex Query Details -Query ID XXX -Query Name XXX Sections -======== - Account Information -------------------- 1.ClientAccountID 2.CurrencyPrimary -Cash Transactions ------------------ - -Options: Dividends, Payment in Lieu of Dividends, Withholding Tax, 871(m) Withholding, -Advisor Fees, Other Fees, Deposits & Withdrawals, Carbon Credits, Bill Pay, Broker -Interest Paid, Broker Interest Received, Broker Fees, Bond Interest Paid, Bond Interest -Received, Price Adjustments, Commission Adjustments, Detail +Cash Report +1.CurrencyPrimary +2.StartingCash +3.EndingCash +4.NetCashBalanceSLB +5.ToDate +Cash Transactions 1.Date/Time 2.Amount 3.Type @@ -34,12 +27,12 @@ 7.ISIN Net Stock Position Summary --------------------------- 1.Symbol -2.CUSIP +2.ISIN +3.ReportDate +4.NetShares Open Dividend Accruals ----------------------- 1.Symbol 2.GrossAmount 3.NetAmount @@ -47,8 +40,13 @@ 5.Quantity 6.ISIN +Open Positions +Options: Summary +1.Symbol +2.ISIN +3.Quantity + Trades ------- Options: Execution 1.SecurityID 2.DateTime @@ -66,17 +64,22 @@ 14.CurrencyPrimary 15.ISIN +Transfers +Options: Transfer +1.Symbol +2.ISIN +3.DateTime +4.Quantity +5.TransferPrice + Delivery Configuration ----------------------- -Accounts -Format XML +Accounts Format XML Period Last N Calendar Days Number of Days 120 General Configuration ---------------------- Profit and Loss Default Include Canceled Trades? No Include Currency Rates? No From 13f0ddc5704adb638dadd97cf62b7c9d1ccc710d Mon Sep 17 00:00:00 2001 From: Red S Date: Thu, 18 Jul 2024 09:22:46 -0700 Subject: [PATCH 41/59] feat: ibkr: add transfers; minor refactor --- .../importers/ibkr/__init__.py | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py index ddab13c..a591211 100644 --- a/beancount_reds_importers/importers/ibkr/__init__.py +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -130,8 +130,19 @@ def convert_date(self, d): d = d.split(' ')[0] return datetime.datetime.strptime(d, self.date_format) - def trade_to_ofx_dict(self, xml_data): - # Mapping the input dictionary to the OFX dictionary format + def xml_transfer_interpreter(self, xml_data): + # map, with ofx fields on the left and xml fields on the right + ofx_dict = { + 'security': xml_data['isin'], + 'tradeDate': self.convert_date(xml_data['dateTime']), + 'units': D(xml_data['quantity']), + 'memo': 'Transfer in kind', + 'type': 'transfer', + } + return DictToObject(ofx_dict) + + def xml_trade_interpreter(self, xml_data): + # map, with ofx fields on the left and xml fields on the right ofx_dict = { 'security': xml_data['isin'], 'tradeDate': self.convert_date(xml_data['dateTime']), @@ -142,10 +153,10 @@ def trade_to_ofx_dict(self, xml_data): 'commission': -1 * D(xml_data['ibCommission']), 'total': D(xml_data['netCash']), } - return ofx_dict + return DictToObject(ofx_dict) - def cash_to_ofx_dict(self, xml_data): - # Mapping the input dictionary to the OFX dictionary format + def xml_cash_interpreter(self, xml_data): + # map, with ofx fields on the left and xml fields on the right ofx_dict = { 'tradeDate': self.convert_date(xml_data['dateTime']), 'amount': D(xml_data['amount']), @@ -158,21 +169,15 @@ def cash_to_ofx_dict(self, xml_data): ofx_dict['type'] = 'dividends' ofx_dict['total'] = ofx_dict['amount'] - return ofx_dict - - def xml_trade_interpreter(self, element): - ot = self.trade_to_ofx_dict(element) - return DictToObject(ot) - - def xml_cash_interpreter(self, element): - ot = self.cash_to_ofx_dict(element) - return DictToObject(ot) + return DictToObject(ofx_dict) def get_transactions(self): yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade', xml_interpreter=self.xml_trade_interpreter) yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction', xml_interpreter=self.xml_cash_interpreter) + yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Transfers/Transfer', + xml_interpreter=self.xml_transfer_interpreter) def get_balance_assertion_date(self): ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0] From c08e6ebd6f55c9a8612eeabd13ba0dc1e4be73dc Mon Sep 17 00:00:00 2001 From: Red S Date: Thu, 18 Jul 2024 23:31:18 -0700 Subject: [PATCH 42/59] feat: ibkr flex query web API downloader (reds-ibkr-flexquery-download) the new command is: reds-ibkr-flexquery-download --- .../importers/ibkr/flexquery_download.py | 41 +++++++++++++++++++ setup.py | 1 + 2 files changed, 42 insertions(+) create mode 100755 beancount_reds_importers/importers/ibkr/flexquery_download.py diff --git a/beancount_reds_importers/importers/ibkr/flexquery_download.py b/beancount_reds_importers/importers/ibkr/flexquery_download.py new file mode 100755 index 0000000..3512f2c --- /dev/null +++ b/beancount_reds_importers/importers/ibkr/flexquery_download.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""IBKR Flex Query Downloader""" + +import requests +import click + +@click.command() +@click.argument('token', required=True) +@click.argument('query_id', required=True) +def flexquery_download(token, query_id): + url = "https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.SendRequest" + + # Request Flex Query + request_payload = { + "v": "3", + "t": token, + "q": query_id + } + + response = requests.post(url, data=request_payload) + + if response.status_code == 200: + request_id = response.text.split("")[1].split("")[0] + # print(f"Request ID: {request_id}") + + # Construct URL to get the query result + result_url = f"https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.GetStatement?q={request_id}&t={token}&v=3" + + result_response = requests.get(result_url) + + if result_response.status_code == 200: + print(result_response.text) + else: + print(f"Failed to get the query result. Status Code: {result_response.status_code}") + return None + else: + print(f"Failed to request the query. Status Code: {response.status_code}") + return None + +if __name__ == '__main__': + flexquery_download() diff --git a/setup.py b/setup.py index 2018a5c..a60d832 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ "console_scripts": [ "ofx-summarize = beancount_reds_importers.util.ofx_summarize:summarize", "bean-download = beancount_reds_importers.util.bean_download:cli", + "reds-ibkr-flexquery-download = beancount_reds_importers.importers.ibkr.flexquery_download:flexquery_download", ] }, zip_safe=False, From b60e97fb80ca1d14cb1e65321b896721f0277da1 Mon Sep 17 00:00:00 2001 From: Red S Date: Thu, 1 Aug 2024 08:37:41 -0700 Subject: [PATCH 43/59] feat(minor): add deep_identify to ibkr --- beancount_reds_importers/importers/ibkr/__init__.py | 13 ++++++++++++- beancount_reds_importers/libreader/xmlreader.py | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py index a591211..4ab1978 100644 --- a/beancount_reds_importers/importers/ibkr/__init__.py +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -117,11 +117,22 @@ class Importer(investments.Importer, xmlreader.Importer): def custom_init(self): if not self.custom_init_run: self.max_rounding_error = 0.04 - self.filename_pattern_def = "Transaction_report" + self.filename_pattern_def = "ibkr" self.custom_init_run = True self.date_format = '%Y-%m-%d' self.get_ticker_info = self.get_ticker_info_from_id + def deep_identify(self, file): + try: + if self.config.get('account_number', None): + # account number specific matching + return self.config['account_number'] == list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['accountId'] + else: + # base check: simply ensure this looks like a valid IBKR Flex Query file + return list(self.get_xpath_elements("/FlexQueryResponse"))[0] is not None + except IndexError: + return False + def set_currency(self): self.currency = list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['currency'] diff --git a/beancount_reds_importers/libreader/xmlreader.py b/beancount_reds_importers/libreader/xmlreader.py index 9667f9d..f7d1446 100644 --- a/beancount_reds_importers/libreader/xmlreader.py +++ b/beancount_reds_importers/libreader/xmlreader.py @@ -19,11 +19,11 @@ def initialize_reader(self, file): self.file = file self.reader_ready = False self.xmltree = etree.parse(file.name) - self.reader_ready = self.deep_identify() + self.reader_ready = self.deep_identify(file) if self.reader_ready: self.set_currency() - def deep_identify(self): + def deep_identify(self, file): """For overriding by institution specific importer which can check if an account name matches, and oother such things.""" return True From de199f0bca7a0e1717d613bf92bc2a6389656cf1 Mon Sep 17 00:00:00 2001 From: Red S Date: Mon, 23 Sep 2024 21:46:08 -0700 Subject: [PATCH 44/59] fix: remove blank ticker, common with brokerage-bank combos --- beancount_reds_importers/libtransactionbuilder/investments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 12ecd63..aa20633 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -163,7 +163,8 @@ def get_ticker_info_from_id(self, security_id): except IndexError: print(f"Error: fund info not found for {security_id}", file=sys.stderr) securities = self.get_security_list() - securities_missing = [s for s in securities] + securities.remove('') + securities_missing = list(securities) for s in securities: for k in self.funds_db: if s in k: From 9109855da6947667542abf8f140ddc86f8558a7d Mon Sep 17 00:00:00 2001 From: Red S Date: Mon, 23 Sep 2024 21:47:18 -0700 Subject: [PATCH 45/59] fix: wip, fix fidelity_cma_csv TODO: - this needs a pytest so tests don't go stale - needs much more fixing, or removal --- .../importers/fidelity/fidelity_cma_csv.py | 1 + .../importers/fidelity/fidelity_cma_csv_examples/README.md | 4 ++++ .../fidelity_cma_csv_examples/fidelity-cma-csv.import | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 beancount_reds_importers/importers/fidelity/fidelity_cma_csv_examples/README.md diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py index d2b9076..af0c58d 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv.py @@ -35,6 +35,7 @@ def custom_init(self): "Symbol": "security", "Price ($)": "unit_price", } + self.skip_transaction_types = [] # fmt: on def deep_identify(self, file): diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv_examples/README.md b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv_examples/README.md new file mode 100644 index 0000000..1a4aafc --- /dev/null +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv_examples/README.md @@ -0,0 +1,4 @@ +fidelity_cma_csv doesn't currently work, and needs to be fixed. Patches welcome. + +See [this link](https://groups.google.com/g/beancount/c/XqdlCfnZdQU/m/TksJQqjPAQAJ) for +details. diff --git a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv_examples/fidelity-cma-csv.import b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv_examples/fidelity-cma-csv.import index 4301fe4..63afbe8 100755 --- a/beancount_reds_importers/importers/fidelity/fidelity_cma_csv_examples/fidelity-cma-csv.import +++ b/beancount_reds_importers/importers/fidelity/fidelity_cma_csv_examples/fidelity-cma-csv.import @@ -6,7 +6,7 @@ from os import path sys.path.insert(0, path.join(path.dirname(__file__))) -from beancount_reds_importers import fidelity_cma_csv +from beancount_reds_importers.importers.fidelity import fidelity_cma_csv fund_data = [ ('FZFXX', 'helllo', 'Fideilty Zero', ), From a9e5323159716a633f4cfc4ea412f75bfb52f4a2 Mon Sep 17 00:00:00 2001 From: Red S Date: Mon, 23 Sep 2024 21:51:43 -0700 Subject: [PATCH 46/59] doc: recommend bleeding edge --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 95351b2..9f3c1cd 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,8 @@ for f in ~/.zcomplete/*; do source $f; done pip3 install beancount-reds-importers ``` -Or to install the bleeding edge version from git: +Or to install the bleeding edge version from git (which I recommend, as long as you are +willing to understand there might be a bug or two): ``` pip3 install git+https://github.com/redstreet/beancount_reds_importers ``` From fe279d0207ba626145d45a3c69f16e13c20bb049 Mon Sep 17 00:00:00 2001 From: Red S Date: Tue, 1 Oct 2024 23:57:21 -0700 Subject: [PATCH 47/59] fix: csvreader: strip before parsing dates --- beancount_reds_importers/libreader/csvreader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beancount_reds_importers/libreader/csvreader.py b/beancount_reds_importers/libreader/csvreader.py index 0bd11af..4388281 100644 --- a/beancount_reds_importers/libreader/csvreader.py +++ b/beancount_reds_importers/libreader/csvreader.py @@ -128,7 +128,8 @@ def remove_non_numeric(x): # fixup dates def convert_date(d): - return datetime.datetime.strptime(d, self.date_format) + """Remove spaces and convert to datetime""" + return datetime.datetime.strptime(d.strip(), self.date_format) dates = getattr(self, "date_fields", []) + ["date", "tradeDate", "settleDate"] for i in dates: From f1e224f2228d652ce7870161346f59cfc22896ee Mon Sep 17 00:00:00 2001 From: Red S Date: Tue, 1 Oct 2024 23:58:08 -0700 Subject: [PATCH 48/59] fix: '' in securities bug --- beancount_reds_importers/libtransactionbuilder/investments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index aa20633..38d1986 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -163,7 +163,8 @@ def get_ticker_info_from_id(self, security_id): except IndexError: print(f"Error: fund info not found for {security_id}", file=sys.stderr) securities = self.get_security_list() - securities.remove('') + if '' in securities: + securities.remove('') securities_missing = list(securities) for s in securities: for k in self.funds_db: From 0cef611c0d601e1861e86b7533787e309ad0f0d9 Mon Sep 17 00:00:00 2001 From: Red S Date: Tue, 1 Oct 2024 23:58:21 -0700 Subject: [PATCH 49/59] feat: fidelity_brokerage_csv importer (ofx is limited to 90 days) --- .../fidelity/fidelity_brokerage_csv.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py diff --git a/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py new file mode 100644 index 0000000..cfefedc --- /dev/null +++ b/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py @@ -0,0 +1,58 @@ +"""Fidelity Brokerage csv importer for beancount.""" + +import re + +from beancount_reds_importers.libreader import csvreader +from beancount_reds_importers.libtransactionbuilder import investments + + +class Importer(investments.Importer, csvreader.Importer): + IMPORTER_NAME = "Fidelity Cash Management Account" + + def custom_init(self): + self.max_rounding_error = 0.04 + self.filename_pattern_def = ".*History" + self.date_format = "%m/%d/%Y" + self.header_identifier = "" + self.column_labels_line = ( + "Run Date,Action,Symbol,Description,Type,Quantity,Price ($),Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date" + ) + self.header_map = { + "Run Date": "date", + "Action": "memo", + "Symbol": "security", + "Amount ($)": "amount", + "Settlement Date": "settleDate", + "Quantity": "units", + "Accrued Interest ($)": "accrued_interest", + "Fees ($)": "fees", + "Commission ($)": "commission", + "Cash Balance ($)": "balance", + "Price ($)": "unit_price", + } + self.transaction_type_map = { + "DIVIDEND RECEIVED": "dividends", + "TRANSFERRED FROM": "cash", + "YOU BOUGHT": "buystock", + "YOU SOLD": "sellstock", + } + self.skip_transaction_types = [] + # fmt: on + + def deep_identify(self, file): + last_four = self.config.get("account_number", "")[-4:] + return re.match(self.header_identifier, file.head(), flags=re.DOTALL) and f"{last_four}" in file.name + + def prepare_table(self, rdr): + for field in ["Action", "Symbol", "Description"]: + rdr = rdr.convert(field, lambda x: x.lstrip()) + + rdr = rdr.addfield("total", lambda x: x["Amount ($)"]) + rdr = rdr.addfield("tradeDate", lambda x: x["Run Date"]) + rdr = rdr.cutout('Type') + rdr = rdr.capture("Action", "(\\S+(?:\\s+\\S+)?)", ["type"], include_original=True) + + # for field in ["memo"]: + # rdr = rdr.convert(field, lambda x: x.lstrip()) + + return rdr From 67b09f417b0f43d31481b2b3179531bec6291e1e Mon Sep 17 00:00:00 2001 From: dev Date: Sat, 5 Oct 2024 01:00:14 +0200 Subject: [PATCH 50/59] fix: constrain Beancount dependency version The project doesn't work with beancount v3. --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 53470d3..6a151c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Automatically generated by https://github.com/damnever/pigar. -beancount>=2.3.5 +beancount >=2.3.5,<3.0.0 beautifulsoup4>=4.12.3 click>=8.1.7 click-aliases>=1.0.4 diff --git a/setup.py b/setup.py index a60d832..0df449a 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ }, install_requires=[ "Click >= 7.0", - "beancount >= 2.3.5", + "beancount >=2.3.5,<3.0.0", "click_aliases >= 1.0.1", "dateparser >= 1.2.0", "ofxparse >= 0.21", From 8194ac405a3e933915af789ba43f193b5accfe46 Mon Sep 17 00:00:00 2001 From: dev Date: Sat, 5 Oct 2024 01:47:04 +0200 Subject: [PATCH 51/59] feat! : add generic json reader BREAKING CHANGE: old jsonreader.py is rename into schwabjsonreader.py --- .../libreader/jsonreader.py | 106 ++++++------------ .../libreader/schwabjsonreader.py | 92 +++++++++++++++ 2 files changed, 129 insertions(+), 69 deletions(-) create mode 100644 beancount_reds_importers/libreader/schwabjsonreader.py diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 38ffc8f..0d046d3 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -1,92 +1,60 @@ -"""JSON importer module for beancount to be used along with investment/banking/other importer modules in -beancount_reds_importers. +#!/usr/bin/env python3 ------------------------------- -This is WIP and incomplete. ------------------------------- +"""JSON reader for beancount-reds-importers. -JSON schemas vary widely. This one is based on Charles Schwab's json format. In the future, the -goal is to make this reader automatically "understand" the schema of any json given to it. +JSON files have widely varying specifications, and thus, this is a very generic reader, and most of +the logic will have to be the institution specific readers. -Until that happens, perhaps this file should be renamed to schwabjsonreader.py. """ import json -# import re -import warnings - -# import datetime -# import ofxparse -# from collections import namedtuple from beancount.ingest import importer -from bs4.builder import XMLParsedAsHTMLWarning - from beancount_reds_importers.libreader import reader -warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) - - class Importer(reader.Reader, importer.ImporterProtocol): FILE_EXTS = ["json"] def initialize_reader(self, file): if getattr(self, "file", None) != file: self.file = file + self.reader_ready = False + with open(file.name, 'r') as f: + self.json_data = json.load(f) self.reader_ready = self.deep_identify(file) - if self.reader_ready: - self.file_read_done = False + if self.reader_ready: + self.set_currency() def deep_identify(self, file): - # identify based on filename - return True + """For overriding by institution specific importer which can check if an account name + matches, and other such things.""" + # default value to False, else jsonreader.initialize_reader fail to execute because missing attribut "config" + return False def file_date(self, file): - "Get the maximum date from the file." - self.initialize(file) # self.date_format gets set via this - self.read_file(file) - return max(ot.date for ot in self.get_transactions()).date() + """Get the ending date of the statement.""" + if not getattr(self, "json_data", None): + self.initialize(file) + # TODO: + return None def read_file(self, file): - with open(file.name) as fh: - self.rdr = json.load(fh) - - # transactions = [] - # for transaction in self.rdr['BrokerageTransactions']: - # raw_ot = Transaction( - # date = transaction['Date'], - # type = transaction['Action'], - # security = transaction['Symbol'], - # memo = transaction['Description'], - # unit_price = transaction['Price'], - # units = transaction['Quantity'], - # fees = transaction['Fees & Comm'], - # total = transaction['Amount'] - # ) - - # def get_transactions(self): - # Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', - # 'units', 'fees', 'total']) - # for transaction in self.rdr['BrokerageTransactions']: - # raw_ot = Transaction( - # date = transaction['Date'], - # type = transaction['Action'], - # security = transaction['Symbol'], - # memo = transaction['Description'], - # unit_price = transaction['Price'], - # units = transaction['Quantity'], - # fees = transaction['Fees & Comm'], - # total = transaction['Amount'] - # ) - # ot = self.fixup(ot) - # import pdb; pdb.set_trace() - # yield ot - - def fixup(self, ot): - ot.date = self.convert_date(ot.date) - - # def convert_date(d): - # return datetime.datetime.strptime(d, self.date_format) - - def get_balance_assertion_date(self): - return None + with open(file.name, 'r') as f: + self.json_data = json.load(f) + + def get_json_elements(self, json_path, json_interpreter=lambda x: x): + """Extract a list of elements in the JSON file at the given JSON path. Typically, + transactions are stored in a JSON path, and this extracts them.""" + elements = self.json_data + for key in json_path.split('.'): + if key in elements: + elements = elements[key] + else: + return [] + for elem in elements: + yield json_interpreter(elem) + + def get_transactions(self): + """/Transactions/Transaction is a dummy default path for transactions that needs to be + overriden in the institution specific importer.""" + yield from self.get_json_elements("Transactions.Transaction") diff --git a/beancount_reds_importers/libreader/schwabjsonreader.py b/beancount_reds_importers/libreader/schwabjsonreader.py new file mode 100644 index 0000000..38ffc8f --- /dev/null +++ b/beancount_reds_importers/libreader/schwabjsonreader.py @@ -0,0 +1,92 @@ +"""JSON importer module for beancount to be used along with investment/banking/other importer modules in +beancount_reds_importers. + +------------------------------ +This is WIP and incomplete. +------------------------------ + +JSON schemas vary widely. This one is based on Charles Schwab's json format. In the future, the +goal is to make this reader automatically "understand" the schema of any json given to it. + +Until that happens, perhaps this file should be renamed to schwabjsonreader.py. +""" + +import json + +# import re +import warnings + +# import datetime +# import ofxparse +# from collections import namedtuple +from beancount.ingest import importer +from bs4.builder import XMLParsedAsHTMLWarning + +from beancount_reds_importers.libreader import reader + +warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) + + +class Importer(reader.Reader, importer.ImporterProtocol): + FILE_EXTS = ["json"] + + def initialize_reader(self, file): + if getattr(self, "file", None) != file: + self.file = file + self.reader_ready = self.deep_identify(file) + if self.reader_ready: + self.file_read_done = False + + def deep_identify(self, file): + # identify based on filename + return True + + def file_date(self, file): + "Get the maximum date from the file." + self.initialize(file) # self.date_format gets set via this + self.read_file(file) + return max(ot.date for ot in self.get_transactions()).date() + + def read_file(self, file): + with open(file.name) as fh: + self.rdr = json.load(fh) + + # transactions = [] + # for transaction in self.rdr['BrokerageTransactions']: + # raw_ot = Transaction( + # date = transaction['Date'], + # type = transaction['Action'], + # security = transaction['Symbol'], + # memo = transaction['Description'], + # unit_price = transaction['Price'], + # units = transaction['Quantity'], + # fees = transaction['Fees & Comm'], + # total = transaction['Amount'] + # ) + + # def get_transactions(self): + # Transaction = namedtuple('Transaction', ['date', 'type', 'security', 'memo', 'unit_price', + # 'units', 'fees', 'total']) + # for transaction in self.rdr['BrokerageTransactions']: + # raw_ot = Transaction( + # date = transaction['Date'], + # type = transaction['Action'], + # security = transaction['Symbol'], + # memo = transaction['Description'], + # unit_price = transaction['Price'], + # units = transaction['Quantity'], + # fees = transaction['Fees & Comm'], + # total = transaction['Amount'] + # ) + # ot = self.fixup(ot) + # import pdb; pdb.set_trace() + # yield ot + + def fixup(self, ot): + ot.date = self.convert_date(ot.date) + + # def convert_date(d): + # return datetime.datetime.strptime(d, self.date_format) + + def get_balance_assertion_date(self): + return None From 85080cd461aa021dba464d209003b1f55d71e26f Mon Sep 17 00:00:00 2001 From: dev Date: Sun, 6 Oct 2024 23:27:55 +0200 Subject: [PATCH 52/59] refactor: formatting whole project --- .../fidelity/fidelity_brokerage_csv.py | 39 +++--- .../importers/ibkr/__init__.py | 113 +++++++++++------- .../importers/ibkr/flexquery_download.py | 32 ++--- .../importers/vanguard/__init__.py | 2 +- .../libreader/jsonreader.py | 8 +- .../libreader/xmlreader.py | 2 +- .../libtransactionbuilder/investments.py | 4 +- 7 files changed, 116 insertions(+), 84 deletions(-) diff --git a/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py b/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py index cfefedc..79b8d9f 100644 --- a/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py +++ b/beancount_reds_importers/importers/fidelity/fidelity_brokerage_csv.py @@ -14,34 +14,35 @@ def custom_init(self): self.filename_pattern_def = ".*History" self.date_format = "%m/%d/%Y" self.header_identifier = "" - self.column_labels_line = ( - "Run Date,Action,Symbol,Description,Type,Quantity,Price ($),Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date" - ) + self.column_labels_line = "Run Date,Action,Symbol,Description,Type,Quantity,Price ($),Commission ($),Fees ($),Accrued Interest ($),Amount ($),Cash Balance ($),Settlement Date" self.header_map = { - "Run Date": "date", - "Action": "memo", - "Symbol": "security", - "Amount ($)": "amount", - "Settlement Date": "settleDate", - "Quantity": "units", + "Run Date": "date", + "Action": "memo", + "Symbol": "security", + "Amount ($)": "amount", + "Settlement Date": "settleDate", + "Quantity": "units", "Accrued Interest ($)": "accrued_interest", - "Fees ($)": "fees", - "Commission ($)": "commission", - "Cash Balance ($)": "balance", - "Price ($)": "unit_price", + "Fees ($)": "fees", + "Commission ($)": "commission", + "Cash Balance ($)": "balance", + "Price ($)": "unit_price", } self.transaction_type_map = { - "DIVIDEND RECEIVED": "dividends", - "TRANSFERRED FROM": "cash", - "YOU BOUGHT": "buystock", - "YOU SOLD": "sellstock", + "DIVIDEND RECEIVED": "dividends", + "TRANSFERRED FROM": "cash", + "YOU BOUGHT": "buystock", + "YOU SOLD": "sellstock", } self.skip_transaction_types = [] # fmt: on def deep_identify(self, file): last_four = self.config.get("account_number", "")[-4:] - return re.match(self.header_identifier, file.head(), flags=re.DOTALL) and f"{last_four}" in file.name + return ( + re.match(self.header_identifier, file.head(), flags=re.DOTALL) + and f"{last_four}" in file.name + ) def prepare_table(self, rdr): for field in ["Action", "Symbol", "Description"]: @@ -49,7 +50,7 @@ def prepare_table(self, rdr): rdr = rdr.addfield("total", lambda x: x["Amount ($)"]) rdr = rdr.addfield("tradeDate", lambda x: x["Run Date"]) - rdr = rdr.cutout('Type') + rdr = rdr.cutout("Type") rdr = rdr.capture("Action", "(\\S+(?:\\s+\\S+)?)", ["type"], include_original=True) # for field in ["memo"]: diff --git a/beancount_reds_importers/importers/ibkr/__init__.py b/beancount_reds_importers/importers/ibkr/__init__.py index 4ab1978..999cb48 100644 --- a/beancount_reds_importers/importers/ibkr/__init__.py +++ b/beancount_reds_importers/importers/ibkr/__init__.py @@ -93,9 +93,11 @@ """ import datetime + +from beancount.core.number import D + from beancount_reds_importers.libreader import xmlreader from beancount_reds_importers.libtransactionbuilder import investments -from beancount.core.number import D class DictToObject: @@ -106,8 +108,8 @@ def __init__(self, dictionary): # xml on left, ofx on right ofx_type_map = { - 'BUY': 'buystock', - 'SELL': 'selltock', + "BUY": "buystock", + "SELL": "selltock", } @@ -119,14 +121,21 @@ def custom_init(self): self.max_rounding_error = 0.04 self.filename_pattern_def = "ibkr" self.custom_init_run = True - self.date_format = '%Y-%m-%d' + self.date_format = "%Y-%m-%d" self.get_ticker_info = self.get_ticker_info_from_id def deep_identify(self, file): try: - if self.config.get('account_number', None): + if self.config.get("account_number", None): # account number specific matching - return self.config['account_number'] == list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['accountId'] + return ( + self.config["account_number"] + == list( + self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation" + ) + )[0]["accountId"] + ) else: # base check: simply ensure this looks like a valid IBKR Flex Query file return list(self.get_xpath_elements("/FlexQueryResponse"))[0] is not None @@ -134,77 +143,97 @@ def deep_identify(self, file): return False def set_currency(self): - self.currency = list(self.get_xpath_elements("/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation"))[0]['currency'] + self.currency = list( + self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/AccountInformation" + ) + )[0]["currency"] # fixup dates def convert_date(self, d): - d = d.split(' ')[0] + d = d.split(" ")[0] return datetime.datetime.strptime(d, self.date_format) def xml_transfer_interpreter(self, xml_data): # map, with ofx fields on the left and xml fields on the right ofx_dict = { - 'security': xml_data['isin'], - 'tradeDate': self.convert_date(xml_data['dateTime']), - 'units': D(xml_data['quantity']), - 'memo': 'Transfer in kind', - 'type': 'transfer', + "security": xml_data["isin"], + "tradeDate": self.convert_date(xml_data["dateTime"]), + "units": D(xml_data["quantity"]), + "memo": "Transfer in kind", + "type": "transfer", } return DictToObject(ofx_dict) def xml_trade_interpreter(self, xml_data): # map, with ofx fields on the left and xml fields on the right ofx_dict = { - 'security': xml_data['isin'], - 'tradeDate': self.convert_date(xml_data['dateTime']), - 'memo': xml_data['transactionType'], - 'type': ofx_type_map[xml_data['buySell']], - 'units': D(xml_data['quantity']), - 'unit_price': D(xml_data['tradePrice']), - 'commission': -1 * D(xml_data['ibCommission']), - 'total': D(xml_data['netCash']), + "security": xml_data["isin"], + "tradeDate": self.convert_date(xml_data["dateTime"]), + "memo": xml_data["transactionType"], + "type": ofx_type_map[xml_data["buySell"]], + "units": D(xml_data["quantity"]), + "unit_price": D(xml_data["tradePrice"]), + "commission": -1 * D(xml_data["ibCommission"]), + "total": D(xml_data["netCash"]), } return DictToObject(ofx_dict) def xml_cash_interpreter(self, xml_data): # map, with ofx fields on the left and xml fields on the right ofx_dict = { - 'tradeDate': self.convert_date(xml_data['dateTime']), - 'amount': D(xml_data['amount']), - 'security': xml_data.get('isin', None), - 'type': 'cash', - 'memo': xml_data['type'], + "tradeDate": self.convert_date(xml_data["dateTime"]), + "amount": D(xml_data["amount"]), + "security": xml_data.get("isin", None), + "type": "cash", + "memo": xml_data["type"], } - if xml_data['type'] == 'Dividends': - ofx_dict['type'] = 'dividends' - ofx_dict['total'] = ofx_dict['amount'] + if xml_data["type"] == "Dividends": + ofx_dict["type"] = "dividends" + ofx_dict["total"] = ofx_dict["amount"] return DictToObject(ofx_dict) def get_transactions(self): - yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade', - xml_interpreter=self.xml_trade_interpreter) - yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction', - xml_interpreter=self.xml_cash_interpreter) - yield from self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/Transfers/Transfer', - xml_interpreter=self.xml_transfer_interpreter) + yield from self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/Trades/Trade", + xml_interpreter=self.xml_trade_interpreter, + ) + yield from self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/CashTransactions/CashTransaction", + xml_interpreter=self.xml_cash_interpreter, + ) + yield from self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/Transfers/Transfer", + xml_interpreter=self.xml_transfer_interpreter, + ) def get_balance_assertion_date(self): - ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0] - return self.convert_date(ac['toDate']).date() + ac = list( + self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency" + ) + )[0] + return self.convert_date(ac["toDate"]).date() def get_available_cash(self, settlement_fund_balance=0): """Assumes there's only one cash currency. TODO: get investments transaction builder to accept date from get_available_cash """ - ac = list(self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency'))[0] - return D(ac['slbNetCash']) + ac = list( + self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/CashReport/CashReportCurrency" + ) + )[0] + return D(ac["slbNetCash"]) def get_balance_positions(self): - for pos in self.get_xpath_elements('/FlexQueryResponse/FlexStatements/FlexStatement/OpenPositions/OpenPosition'): + for pos in self.get_xpath_elements( + "/FlexQueryResponse/FlexStatements/FlexStatement/OpenPositions/OpenPosition" + ): balance = { - 'security': pos['isin'], - 'units': D(pos['position']), + "security": pos["isin"], + "units": D(pos["position"]), } yield DictToObject(balance) diff --git a/beancount_reds_importers/importers/ibkr/flexquery_download.py b/beancount_reds_importers/importers/ibkr/flexquery_download.py index 3512f2c..b4be179 100755 --- a/beancount_reds_importers/importers/ibkr/flexquery_download.py +++ b/beancount_reds_importers/importers/ibkr/flexquery_download.py @@ -1,33 +1,32 @@ #!/usr/bin/env python3 """IBKR Flex Query Downloader""" -import requests import click +import requests + @click.command() -@click.argument('token', required=True) -@click.argument('query_id', required=True) +@click.argument("token", required=True) +@click.argument("query_id", required=True) def flexquery_download(token, query_id): - url = "https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.SendRequest" - + url = ( + "https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.SendRequest" + ) + # Request Flex Query - request_payload = { - "v": "3", - "t": token, - "q": query_id - } - + request_payload = {"v": "3", "t": token, "q": query_id} + response = requests.post(url, data=request_payload) - + if response.status_code == 200: request_id = response.text.split("")[1].split("")[0] # print(f"Request ID: {request_id}") - + # Construct URL to get the query result result_url = f"https://gdcdyn.interactivebrokers.com/Universal/servlet/FlexStatementService.GetStatement?q={request_id}&t={token}&v=3" - + result_response = requests.get(result_url) - + if result_response.status_code == 200: print(result_response.text) else: @@ -37,5 +36,6 @@ def flexquery_download(token, query_id): print(f"Failed to request the query. Status Code: {response.status_code}") return None -if __name__ == '__main__': + +if __name__ == "__main__": flexquery_download() diff --git a/beancount_reds_importers/importers/vanguard/__init__.py b/beancount_reds_importers/importers/vanguard/__init__.py index 47faab5..76f48e9 100644 --- a/beancount_reds_importers/importers/vanguard/__init__.py +++ b/beancount_reds_importers/importers/vanguard/__init__.py @@ -45,7 +45,7 @@ def cleanup_memo(self, ot): # some vanguard files have memos repeated like this: # 'DIVIDEND REINVESTMENTDIVIDEND REINVESTMENT' retval = ot.memo - if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2):]: + if ot.memo[: int(len(ot.memo) / 2)] == ot.memo[int(len(ot.memo) / 2) :]: retval = ot.memo[: int(len(ot.memo) / 2)] return retval diff --git a/beancount_reds_importers/libreader/jsonreader.py b/beancount_reds_importers/libreader/jsonreader.py index 0d046d3..d469439 100644 --- a/beancount_reds_importers/libreader/jsonreader.py +++ b/beancount_reds_importers/libreader/jsonreader.py @@ -10,8 +10,10 @@ import json from beancount.ingest import importer + from beancount_reds_importers.libreader import reader + class Importer(reader.Reader, importer.ImporterProtocol): FILE_EXTS = ["json"] @@ -19,7 +21,7 @@ def initialize_reader(self, file): if getattr(self, "file", None) != file: self.file = file self.reader_ready = False - with open(file.name, 'r') as f: + with open(file.name, "r") as f: self.json_data = json.load(f) self.reader_ready = self.deep_identify(file) if self.reader_ready: @@ -39,14 +41,14 @@ def file_date(self, file): return None def read_file(self, file): - with open(file.name, 'r') as f: + with open(file.name, "r") as f: self.json_data = json.load(f) def get_json_elements(self, json_path, json_interpreter=lambda x: x): """Extract a list of elements in the JSON file at the given JSON path. Typically, transactions are stored in a JSON path, and this extracts them.""" elements = self.json_data - for key in json_path.split('.'): + for key in json_path.split("."): if key in elements: elements = elements[key] else: diff --git a/beancount_reds_importers/libreader/xmlreader.py b/beancount_reds_importers/libreader/xmlreader.py index f7d1446..237cebc 100644 --- a/beancount_reds_importers/libreader/xmlreader.py +++ b/beancount_reds_importers/libreader/xmlreader.py @@ -5,9 +5,9 @@ """ +from beancount.ingest import importer from lxml import etree -from beancount.ingest import importer from beancount_reds_importers.libreader import reader diff --git a/beancount_reds_importers/libtransactionbuilder/investments.py b/beancount_reds_importers/libtransactionbuilder/investments.py index 38d1986..f54eec2 100644 --- a/beancount_reds_importers/libtransactionbuilder/investments.py +++ b/beancount_reds_importers/libtransactionbuilder/investments.py @@ -163,8 +163,8 @@ def get_ticker_info_from_id(self, security_id): except IndexError: print(f"Error: fund info not found for {security_id}", file=sys.stderr) securities = self.get_security_list() - if '' in securities: - securities.remove('') + if "" in securities: + securities.remove("") securities_missing = list(securities) for s in securities: for k in self.funds_db: From 69b95c2a6f55a74980f53f52bb1793531a75816b Mon Sep 17 00:00:00 2001 From: Red S Date: Sun, 13 Oct 2024 10:30:55 -0700 Subject: [PATCH 53/59] fix: schwab_csv_brokerage: add more transaction types --- .../importers/schwab/schwab_csv_brokerage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py index e5c5a3c..8c8be70 100644 --- a/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py +++ b/beancount_reds_importers/importers/schwab/schwab_csv_brokerage.py @@ -57,6 +57,9 @@ def custom_init(self): "Funds Received": "cash", "Stock Split": "cash", "Cash In Lieu": "cash", + "Wire Sent": "cash", + "Misc Cash Entry": "cash", + "Service Fee": "fee", } # fmt: on From 07d972d95e535b0705d52e7ce36a25c40d4babaf Mon Sep 17 00:00:00 2001 From: Red S Date: Sun, 13 Oct 2024 10:39:57 -0700 Subject: [PATCH 54/59] feat: turn on emit_filing_account by default Else, we are dependent on the order of postings determined by Python --- .../libtransactionbuilder/transactionbuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index d9f9992..ee88d3e 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -54,7 +54,7 @@ def build_metadata(self, file, metatype=None, data={}): # This 'filing_account' is read by a patch to bean-extract so it can output transactions to # a file that corresponds with filing_account, when the one-file-per-account feature is # used. - if self.config.get("emit_filing_account_metadata"): + if not self.config.get("emit_filing_account_metadata", False): acct = self.config.get("filing_account", self.config.get("main_account", None)) return {"filing_account": acct} return {} From 3cab8517b1471cd90dda8d8511226d43c07eb15b Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 16 Nov 2024 16:02:16 -0800 Subject: [PATCH 55/59] fix: #109: raise notimplemented for get_transactions() --- beancount_reds_importers/libreader/reader.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/beancount_reds_importers/libreader/reader.py b/beancount_reds_importers/libreader/reader.py index 98ef183..9d55dc6 100644 --- a/beancount_reds_importers/libreader/reader.py +++ b/beancount_reds_importers/libreader/reader.py @@ -64,3 +64,8 @@ def get_balance_assertion_date(self): def get_available_cash(self, settlement_fund_balance=0): return None + + def get_transactions(self): + raise NotImplementedError( + "get_transactions() must be implemented by a subclass (usually the reader, but sometimes the importer)." + ) From 9daa22b0827c8deaef551a598cd047647b5bd61c Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 16 Nov 2024 16:47:32 -0800 Subject: [PATCH 56/59] fix: failing unit tests because of emit_filing_account change - emit_filing_account was turned on by default in 07d972d, which broke unit tests --- beancount_reds_importers/importers/ally/tests/ally_test.py | 1 + .../importers/capitalonebank/tests/capitalone_test.py | 1 + beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py | 1 + .../importers/etrade/tests/etrade_qfx_brokerage_test.py | 1 + .../tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py | 1 + .../tests/schwab_csv_checking/schwab_csv_checking_test.py | 1 + .../importers/unitedoverseas/tests/uobbank_test.py | 1 + .../importers/vanguard/tests/vanguard_test.py | 1 + .../last_transaction/last_transaction_date_test.py | 1 + .../tests/balance_assertion_date/ofx_date/ofx_date_test.py | 1 + .../tests/balance_assertion_date/smart/smart_date_test.py | 1 + .../libtransactionbuilder/transactionbuilder.py | 2 +- 12 files changed, 12 insertions(+), 1 deletion(-) diff --git a/beancount_reds_importers/importers/ally/tests/ally_test.py b/beancount_reds_importers/importers/ally/tests/ally_test.py index 8f7e87f..527707f 100644 --- a/beancount_reds_importers/importers/ally/tests/ally_test.py +++ b/beancount_reds_importers/importers/ally/tests/ally_test.py @@ -10,6 +10,7 @@ { "account_number": "23456", "main_account": "Assets:Banks:Checking", + "emit_filing_account_metadata": False, } ) ) diff --git a/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py b/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py index a7215c0..2f2f82c 100644 --- a/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py +++ b/beancount_reds_importers/importers/capitalonebank/tests/capitalone_test.py @@ -10,6 +10,7 @@ { "account_number": "9876", "main_account": "Assets:Banks:CapitalOne", + "emit_filing_account_metadata": False, } ) ) diff --git a/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py b/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py index c05e35c..9c59d33 100644 --- a/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py +++ b/beancount_reds_importers/importers/dcu/tests/dcu_csv_test.py @@ -12,6 +12,7 @@ { "main_account": "Assets:Banks:DCU:Checking", "currency": "USD", + "emit_filing_account_metadata": False, } ) ) diff --git a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py index 656fa32..a3dea56 100644 --- a/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py +++ b/beancount_reds_importers/importers/etrade/tests/etrade_qfx_brokerage_test.py @@ -41,6 +41,7 @@ def build_config(): "rounding_error": "Equity:Rounding-Errors:Imports", "fund_info": fund_info, "currency": currency, + "emit_filing_account_metadata": False, } return config diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py index ea91049..3869273 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_brokerage/schwab_csv_brokerage_test.py @@ -42,6 +42,7 @@ def build_config(): "rounding_error": "Equity:Rounding-Errors:Imports", "fund_info": fund_info, "currency": currency, + "emit_filing_account_metadata": False, } return config diff --git a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py index 04e3bb7..44ed462 100644 --- a/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py +++ b/beancount_reds_importers/importers/schwab/tests/schwab_csv_checking/schwab_csv_checking_test.py @@ -13,6 +13,7 @@ "account_number": "1234", "main_account": "Assets:Banks:Schwab", "currency": "USD", + "emit_filing_account_metadata": False, } ) ) diff --git a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py index f989eb4..d78f2ac 100644 --- a/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py +++ b/beancount_reds_importers/importers/unitedoverseas/tests/uobbank_test.py @@ -12,6 +12,7 @@ "account_number": "1234567890", "currency": "SGD", "rounding_error": "Equity:Rounding-Errors:Imports", + "emit_filing_account_metadata": False, } ) ) diff --git a/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py b/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py index 8473b32..02c962b 100644 --- a/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py +++ b/beancount_reds_importers/importers/vanguard/tests/vanguard_test.py @@ -26,6 +26,7 @@ ], "money_market": ["VMFXX"], }, + "emit_filing_account_metadata": False, } ) ) diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py index fb4b451..b8ec7db 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/last_transaction/last_transaction_date_test.py @@ -11,6 +11,7 @@ "account_number": "23456", "main_account": "Assets:Banks:Checking", "balance_assertion_date_type": "last_transaction", + "emit_filing_account_metadata": False, } ) ) diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py index b13ac82..8f120c3 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/ofx_date/ofx_date_test.py @@ -11,6 +11,7 @@ "account_number": "23456", "main_account": "Assets:Banks:Checking", "balance_assertion_date_type": "ofx_date", + "emit_filing_account_metadata": False, } ) ) diff --git a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py index a194f50..1f6fbfb 100644 --- a/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py +++ b/beancount_reds_importers/libreader/tests/balance_assertion_date/smart/smart_date_test.py @@ -11,6 +11,7 @@ { "account_number": "23456", "main_account": "Assets:Banks:Checking", + "emit_filing_account_metadata": False, } ) ) diff --git a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py index ee88d3e..5299703 100644 --- a/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py +++ b/beancount_reds_importers/libtransactionbuilder/transactionbuilder.py @@ -54,7 +54,7 @@ def build_metadata(self, file, metatype=None, data={}): # This 'filing_account' is read by a patch to bean-extract so it can output transactions to # a file that corresponds with filing_account, when the one-file-per-account feature is # used. - if not self.config.get("emit_filing_account_metadata", False): + if self.config.get("emit_filing_account_metadata", True) is not False: acct = self.config.get("filing_account", self.config.get("main_account", None)) return {"filing_account": acct} return {} From 738b0ea090e2093fefb60db78f7e89665b263ff0 Mon Sep 17 00:00:00 2001 From: Red S Date: Sat, 16 Nov 2024 19:55:31 -0800 Subject: [PATCH 57/59] refactor: rename genericpdf to genericpdfpaycheck #109 --- .../{genericpdf => genericpdfpaycheck}/__init__.py | 2 +- .../tests/genericpdf_test.py | 6 +++--- .../tests/paystub.sample.pdf | Bin .../tests/paystub.sample.pdf.extract | 0 .../tests/paystub.sample.pdf.file_account | 0 .../tests/paystub.sample.pdf.file_date | 0 .../tests/paystub.sample.pdf.file_name | 0 7 files changed, 4 insertions(+), 4 deletions(-) rename beancount_reds_importers/importers/{genericpdf => genericpdfpaycheck}/__init__.py (97%) rename beancount_reds_importers/importers/{genericpdf => genericpdfpaycheck}/tests/genericpdf_test.py (86%) rename beancount_reds_importers/importers/{genericpdf => genericpdfpaycheck}/tests/paystub.sample.pdf (100%) rename beancount_reds_importers/importers/{genericpdf => genericpdfpaycheck}/tests/paystub.sample.pdf.extract (100%) rename beancount_reds_importers/importers/{genericpdf => genericpdfpaycheck}/tests/paystub.sample.pdf.file_account (100%) rename beancount_reds_importers/importers/{genericpdf => genericpdfpaycheck}/tests/paystub.sample.pdf.file_date (100%) rename beancount_reds_importers/importers/{genericpdf => genericpdfpaycheck}/tests/paystub.sample.pdf.file_name (100%) diff --git a/beancount_reds_importers/importers/genericpdf/__init__.py b/beancount_reds_importers/importers/genericpdfpaycheck/__init__.py similarity index 97% rename from beancount_reds_importers/importers/genericpdf/__init__.py rename to beancount_reds_importers/importers/genericpdfpaycheck/__init__.py index 91b210c..3d04b10 100644 --- a/beancount_reds_importers/importers/genericpdf/__init__.py +++ b/beancount_reds_importers/importers/genericpdfpaycheck/__init__.py @@ -8,7 +8,7 @@ # Generic pdf paystub importer. Use this to build your own pdf paystub importer. # Call this importer with a config that looks like: # -# genericpdf.Importer({"desc":"Paycheck (My Company)", +# genericpdfpaycheck.Importer({"desc":"Paycheck (My Company)", # "main_account":"Income:Employment", # "paycheck_template": {}, # See beancount_reds_importers/libtransactionbuilder/paycheck.py for sample template # "currency": "PENNIES", diff --git a/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py b/beancount_reds_importers/importers/genericpdfpaycheck/tests/genericpdf_test.py similarity index 86% rename from beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py rename to beancount_reds_importers/importers/genericpdfpaycheck/tests/genericpdf_test.py index 2d9eaa2..ea87a22 100644 --- a/beancount_reds_importers/importers/genericpdf/tests/genericpdf_test.py +++ b/beancount_reds_importers/importers/genericpdfpaycheck/tests/genericpdf_test.py @@ -2,11 +2,11 @@ from beancount.ingest import regression_pytest as regtest -from beancount_reds_importers.importers import genericpdf +from beancount_reds_importers.importers import genericpdfpaycheck @regtest.with_importer( - genericpdf.Importer( + genericpdfpaycheck.Importer( { "desc": "Paycheck", "main_account": "Income:Salary:FakeCompany", @@ -29,5 +29,5 @@ ) ) @regtest.with_testdir(path.dirname(__file__)) -class TestGenericPDF(regtest.ImporterTestBase): +class TestGenericPDFPaycheck(regtest.ImporterTestBase): pass diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf b/beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf similarity index 100% rename from beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf rename to beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.extract b/beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf.extract similarity index 100% rename from beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.extract rename to beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf.extract diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_account b/beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf.file_account similarity index 100% rename from beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_account rename to beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf.file_account diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_date b/beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf.file_date similarity index 100% rename from beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_date rename to beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf.file_date diff --git a/beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_name b/beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf.file_name similarity index 100% rename from beancount_reds_importers/importers/genericpdf/tests/paystub.sample.pdf.file_name rename to beancount_reds_importers/importers/genericpdfpaycheck/tests/paystub.sample.pdf.file_name From ee9a787b499a140d03fbd402f60c3efec941fb21 Mon Sep 17 00:00:00 2001 From: Red S Date: Fri, 22 Nov 2024 09:30:10 -0800 Subject: [PATCH 58/59] doc: README.md for pdf --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9f3c1cd..270f06a 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ File format readers included are: - `.ofx` - `.csv` (single and multitable support) - `.xlsx` (single and multitable support) (`pip3 install xlrd` if you plan to use this) +- `.pdf` (single and multitable support) + Transaction builders included are: - Banking (for banks and credit cards, which benefit from a postings predictor like From 5573a63eeca572d3ac7f190e62a489427bc8bcbd Mon Sep 17 00:00:00 2001 From: Red S Date: Fri, 22 Nov 2024 22:36:51 -0800 Subject: [PATCH 59/59] fix: #114: rerun checks on push to PRs --- .github/workflows/conventionalcommits.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conventionalcommits.yml b/.github/workflows/conventionalcommits.yml index ae2d6a7..9373e5d 100644 --- a/.github/workflows/conventionalcommits.yml +++ b/.github/workflows/conventionalcommits.yml @@ -3,7 +3,7 @@ name: Conventional Commits on: pull_request: branches: [ main ] - types: [opened, reopened, edited] + types: [opened, reopened, edited, synchronize] jobs: build: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 05a9b9a..ff7bd9e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -8,7 +8,7 @@ on: branches: [ main ] pull_request: branches: [ main ] - types: [opened, reopened, edited] + types: [opened, reopened, edited, synchronize] jobs: build: