Fixes some James bugs (#2969)

* Fixed bug

* Fix vwap

* Hid future warning

* Added fix to pandas

* Quality improvement to customer and supplier

* Fixed some tests

* Test fix

openbb_terminal/stocks/backtesting/bt_model.py

@@ -2,6 +2,7 @@
 __docformat__ = "numpy"
 
 import logging
+import warnings
 
 import bt
 import pandas as pd
@@ -252,18 +253,21 @@ def rsi_strategy(
     merged_data = bt.merge(signal, prices)
     merged_data.columns = ["signal", "price"]
 
+    warnings.simplefilter(action="ignore", category=FutureWarning)
     bt_strategy = bt.Strategy(
         "RSI Reversion", [bt.algos.WeighTarget(signal), bt.algos.Rebalance()]
     )
     bt_backtest = bt.Backtest(bt_strategy, prices)
     bt_backtest = bt.Backtest(bt_strategy, prices)
     backtests = [bt_backtest]
-    if spy_bt:
-        spy_bt = buy_and_hold("spy", start_date, "SPY Hold")
-        backtests.append(spy_bt)
-    if not no_bench:
-        stock_bt = buy_and_hold(symbol, start_date, symbol.upper() + " Hold")
-        backtests.append(stock_bt)
+    # Once the bt package replaces pd iteritems with items we can remove this
+    with warnings.catch_warnings():
+        if spy_bt:
+            spy_bt = buy_and_hold("spy", start_date, "SPY Hold")
+            backtests.append(spy_bt)
+        if not no_bench:
+            stock_bt = buy_and_hold(symbol, start_date, symbol.upper() + " Hold")
+            backtests.append(stock_bt)
 
     res = bt.run(*backtests)
     return res

openbb_terminal/stocks/due_diligence/csimarket_model.py

@@ -2,69 +2,85 @@
 __docformat__ = "numpy"
 
 import logging
-from typing import List
 
-import requests
-from bs4 import BeautifulSoup
+import pandas as pd
 
 from openbb_terminal.decorators import log_start_end
 
 logger = logging.getLogger(__name__)
 
 
+def clean_table(df: pd.DataFrame) -> pd.DataFrame:
+    """Clean up the table from CSIMarket
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        Dataframe to clean
+
+    Returns
+    -------
+    df: pd.DataFrame
+        Cleaned dataframe
+    """
+    if df.empty:
+        return df
+    if "TICKER" in df.columns:
+        df = df.set_index("TICKER")
+    df.columns = [x.title() for x in df.columns]
+    if "Company Name.1" in df.columns:
+        df = df.drop("Company Name.1", axis=1)
+    if "SUBTOTAL" in df.index:
+        df = df.drop("SUBTOTAL", axis=0)
+    if "Revenue" in df.columns:
+        df = df.sort_values("Revenue", ascending=False)
+    df = df[df.index.notnull()]
+    return df
+
+
 @log_start_end(log=logger)
-def get_suppliers(symbol: str) -> List[str]:
+def get_suppliers(symbol: str, limit: int = 50) -> pd.DataFrame:
     """Get suppliers from ticker provided. [Source: CSIMarket]
 
     Parameters
     ----------
     symbol: str
         Ticker to select suppliers from
+    limit: int
+        The maximum number of rows to show
 
     Returns
     -------
-    list[str]
-        List of suppliers for ticker provided
+    pd.DataFrame
+        A dataframe of suppliers
     """
-    # TODO: This link has a lot more data that we can display
-    # TODO: We could at least sort the tickers based on market cap
-    url_supply_chain = (
-        f"https://csimarket.com/stocks/competitionNO3.php?supply&code={symbol.upper()}"
-    )
-    text_supplier_chain = BeautifulSoup(requests.get(url_supply_chain).text, "lxml")
+    url = f"https://csimarket.com/stocks/competition2.php?supply&code={symbol.upper()}"
+    dfs = pd.read_html(url, header=0)
+    df = dfs[10]
+    df = clean_table(df)
 
-    l_supplier = list()
-    for supplier in text_supplier_chain.findAll(
-        "td", {"class": "svjetlirub11 block al"}
-    ):
-        l_supplier.append(supplier.text.replace("\n", "").strip())
-
-    return l_supplier
+    return df.head(limit)
 
 
 @log_start_end(log=logger)
-def get_customers(symbol: str) -> List[str]:
+def get_customers(symbol: str, limit: int = 50) -> pd.DataFrame:
     """Print customers from ticker provided
 
     Parameters
     ----------
     symbol: str
         Ticker to select customers from
+    limit: int
+        The maximum number of rows to show
 
     Returns
     -------
-    list[str]
-        List of customers for ticker provided
+    pd.DataFrame
+        A dataframe of suppliers
     """
-    # TODO: This link has a lot more data that we can display
-    # TODO: We could at least sort the tickers based on market cap
-    url_customer_chain = (
-        f"https://csimarket.com/stocks/custexNO.php?markets&code={symbol.upper()}"
-    )
-    text_customer_chain = BeautifulSoup(requests.get(url_customer_chain).text, "lxml")
-
-    l_customer = list()
-    for customer in text_customer_chain.findAll("td", {"class": "plava svjetlirub"}):
-        l_customer.append(customer.text)
-
-    return l_customer
+    url = f"https://csimarket.com/stocks/custexNO.php?markets&code={symbol.upper()}"
+    dfs = pd.read_html(url, header=0)
+    df = dfs[9]
+    df = clean_table(df)
+
+    return df.head(limit)

openbb_terminal/stocks/due_diligence/csimarket_view.py

@@ -7,15 +7,15 @@
 import pandas as pd
 
 from openbb_terminal.decorators import log_start_end
-from openbb_terminal.helper_funcs import export_data
+from openbb_terminal.helper_funcs import export_data, print_rich_table
 from openbb_terminal.rich_config import console
 from openbb_terminal.stocks.due_diligence import csimarket_model
 
 logger = logging.getLogger(__name__)
 
 
 @log_start_end(log=logger)
-def suppliers(symbol: str, export: str = ""):
+def suppliers(symbol: str, export: str = "", limit: int = 10) -> None:
     """Display suppliers from ticker provided. [Source: CSIMarket]
 
     Parameters
@@ -24,18 +24,26 @@ def suppliers(symbol: str, export: str = ""):
         Ticker to select suppliers from
     export : str
         Export dataframe data to csv,json,xlsx file
+    limit: int
+        The maximum number of rows to show
     """
-    tickers = csimarket_model.get_suppliers(symbol)
-    if tickers:
-        console.print(f"List of suppliers: {', '.join(tickers)}\n")
-    else:
+    tickers = csimarket_model.get_suppliers(symbol, limit=limit)
+    if tickers.empty:
         console.print("No suppliers found.\n")
+    else:
+        console.print(f"List of suppliers: {', '.join(tickers)}\n")
+        print_rich_table(
+            tickers,
+            headers=list(tickers.columns),
+            show_index=True,
+            title=f"Suppliers for {symbol.upper()}",
+        )
 
     export_data(
         export,
         os.path.dirname(os.path.abspath(__file__)),
         "supplier",
-        pd.DataFrame(tickers),
+        tickers,
     )
 
 
@@ -51,10 +59,15 @@ def customers(symbol: str, export: str = ""):
         Export dataframe data to csv,json,xlsx file
     """
     tickers = csimarket_model.get_customers(symbol)
-    if tickers:
-        console.print(f"List of customers: {', '.join(tickers)}\n")
-    else:
+    if tickers.empty:
         console.print("No customers found.\n")
+    else:
+        print_rich_table(
+            tickers,
+            headers=list(tickers.columns),
+            show_index=True,
+            title=f"Customers for {symbol.upper()}",
+        )
 
     export_data(
         export,

openbb_terminal/stocks/government/quiverquant_view.py

@@ -423,7 +423,9 @@ def display_contracts(
         else:
             return
 
-        df_contracts.groupby("Date").sum().div(1000).plot(kind="bar", rot=0, ax=ax)
+        df_contracts.groupby("Date").sum(numeric_only=True).div(1000).plot(
+            kind="bar", rot=0, ax=ax
+        )
         ax.set_ylabel("Amount ($1k)")
         ax.set_title(f"Sum of latest government contracts to {symbol}")
 

openbb_terminal/stocks/quantitative_analysis/beta_model.py

@@ -13,6 +13,7 @@ def beta_model(
     ref_symbol: str,
     data: pd.DataFrame = None,
     ref_data: pd.DataFrame = None,
+    interval: int = 1440,
 ) -> Tuple[pd.Series, pd.Series, float, float]:
     """Calculate beta for a ticker and a reference ticker.
 
@@ -26,6 +27,8 @@ def beta_model(
         The selected ticker symbols price data
     ref_data: pd.DataFrame
         The reference ticker symbols price data
+    interval: int
+        The interval of the ref_data. This will ONLY be used if ref_data is None
 
     Returns
     -------
@@ -44,7 +47,7 @@ def beta_model(
         # with an uppercase char. This should be consistent.
         data = data.rename({"close": "Close"}, axis=1)
     if ref_data is None:
-        ref_data = stocks_helper.load(ref_symbol)
+        ref_data = stocks_helper.load(ref_symbol, interval=interval)
         if ref_data.empty:
             raise Exception("Invalid ref_symbol ticker")
     else:

openbb_terminal/stocks/quantitative_analysis/beta_view.py

@@ -20,6 +20,7 @@ def beta_view(
     ref_symbol: str,
     data: pd.DataFrame = None,
     ref_data: pd.DataFrame = None,
+    interval: int = 1440,
     export: str = "",
 ) -> None:
     """Display the beta scatterplot + linear regression.
@@ -34,9 +35,13 @@ def beta_view(
         The selected ticker symbols price data
     ref_data : pd.DataFrame
         The reference ticker symbols price data
+    interval: int
+        The interval of the ref_data. This will ONLY be used if ref_data is None
     """
     try:
-        sr, rr, beta, alpha = beta_model(symbol, ref_symbol, data, ref_data)
+        sr, rr, beta, alpha = beta_model(
+            symbol, ref_symbol, data, ref_data, interval=interval
+        )
     except Exception as e:
         if str(e) == "Invalid ref ticker":
             console.print(str(e) + "\n")

openbb_terminal/stocks/quantitative_analysis/qa_controller.py

@@ -26,6 +26,7 @@
 from openbb_terminal.rich_config import console, MenuText, get_ordered_list_sources
 from openbb_terminal.stocks.quantitative_analysis.beta_view import beta_view
 from openbb_terminal.stocks.quantitative_analysis.factors_view import capm_view
+from openbb_terminal.stocks.quantitative_analysis.qa_model import full_stock_df
 
 # pylint: disable=C0302
 
@@ -85,17 +86,7 @@ def __init__(
         """Constructor"""
         super().__init__(queue)
 
-        # TODO: Move these calculations to a model
-        stock["Returns"] = stock["Adj Close"].pct_change()
-        stock["LogRet"] = np.log(stock["Adj Close"]) - np.log(
-            stock["Adj Close"].shift(1)
-        )
-        stock["LogPrice"] = np.log(stock["Adj Close"])
-        stock = stock.rename(columns={"Adj Close": "AdjClose"})
-        stock = stock.dropna()
-        stock.columns = [x.lower() for x in stock.columns]
-
-        self.stock = stock
+        self.stock = full_stock_df(stock)
         self.ticker = ticker
         self.start = start
         self.interval = interval
@@ -911,9 +902,17 @@ def call_beta(self, other_args: List[str]):
         ns_parser = self.parse_known_args_and_warn(
             parser, other_args, EXPORT_ONLY_RAW_DATA_ALLOWED
         )
+        # This assumes all intervals convert from string to int well
+        # This should handle weekly and monthly because the merge would only
+        # Work on the intersections
+        interval = "".join(c for c in self.interval if c.isdigit())
         if ns_parser:
             beta_view(
-                self.ticker, ns_parser.ref, data=self.stock, export=ns_parser.export
+                symbol=self.ticker,
+                ref_symbol=ns_parser.ref,
+                data=self.stock,
+                interval=interval,
+                export=ns_parser.export,
             )
 
     @log_start_end(log=logger)

openbb_terminal/stocks/quantitative_analysis/qa_model.py

@@ -0,0 +1,14 @@
+import pandas as pd
+import numpy as np
+
+pd.options.mode.chained_assignment = None
+
+
+def full_stock_df(df: pd.DataFrame) -> pd.DataFrame:
+    df["Returns"] = df["Adj Close"].pct_change()
+    df["LogRet"] = np.log(df["Adj Close"]) - np.log(df["Adj Close"].shift(1))
+    df["LogPrice"] = np.log(df["Adj Close"])
+    df = df.rename(columns={"Adj Close": "AdjClose"})
+    df = df.dropna()
+    df.columns = [x.lower() for x in df.columns]
+    return df

openbb_terminal/stocks/stocks_helper.py

@@ -245,7 +245,7 @@ def load(
         end_date = datetime.now()
 
     # Daily
-    if interval == 1440:
+    if int(interval) == 1440:
 
         int_string = "Daily"
         if weekly:

openbb_terminal/stocks/technical_analysis/ta_controller.py

@@ -708,12 +708,12 @@ def call_vwap(self, other_args: List[str]):
                 interval_text = self.interval
 
             overlap_view.view_vwap(
-                symbol=self.ticker,
-                s_interval=interval_text,
                 data=self.stock,
+                symbol=self.ticker,
                 start_date=ns_parser.start,
                 end_date=ns_parser.end,
                 offset=ns_parser.n_offset,
+                interval=interval_text,
                 export=ns_parser.export,
             )