changed do_asset_selection to online_usage

Makefile

@@ -19,7 +19,7 @@ endif
 
 env:  ## create environment
 	$(PYTHON) -m venv $(ENVDIR)
-	$(BINDIR)/python -m pip install --editable .[docs,dev]
+	$(BINDIR)/python -m pip install --editable .[docs,dev,examples]
 
 clean:  ## clean environment
 	-rm -rf $(BUILDDIR)/*

TODOs_ROADMAP.rst

@@ -109,6 +109,12 @@ Optimization policies
   and other aggregate statistics (as best as possible) to return ``-np.inf``
   if back-test ended in backruptcy. This is needed specifically for
   hyper-parameter optimization. Target ``1.0.4``.
+- [ ] Capture **logs** from the back-test; add ``logs`` property that returns
+  then as a string (newline separated, like a .log file). Make log level
+  changeable by a module constant (like ``cvxportfolio.result.LOG_LEVEL``) set
+  to ``INFO`` by default. Then, improve logs throughout (informative, proactive
+  on possible issues). Logs formatter should produce source module and
+  timestamp.
 
 
 Development & testing

cvxportfolio/data.py

@@ -809,7 +809,7 @@ class MarketDataInMemory(MarketData):
 
     def __init__(
         self, trading_frequency, base_location, cash_key, min_history,
-        do_asset_selection = True):
+        online_usage = False):
         """This must be called by the derived classes."""
         if (self.returns.index[-1] - self.returns.index[0]) < min_history:
             raise DataError(
@@ -828,7 +828,7 @@ def __init__(
         self.base_location = Path(base_location)
         self.cash_key = cash_key
         self._min_history_timedelta = min_history
-        self.do_asset_selection = do_asset_selection
+        self.online_usage = online_usage
 
     def _mask_dataframes(self, mask):
         """Mask internal dataframes if necessary."""
@@ -879,12 +879,7 @@ def serve(self, t):
             pandas.Series or None, pandas.Series or None)
         """
 
-        # TODO: this clauses should be improved, no need to mask with full
-        # columns
-        if self.do_asset_selection:
-            mask = self._universe_mask_at_time(t).values
-        else:
-            mask = np.ones(self.returns.shape[1], dtype=bool)
+        mask = self._universe_mask_at_time(t).values
         self._mask_dataframes(mask)
 
         tidx = self.returns.index.get_loc(t)
@@ -981,8 +976,11 @@ def trading_calendar(
     def _universe_mask_at_time(self, t):
         """Return the valid universe mask at time t."""
         past_returns = self.returns.loc[self.returns.index < t]
-        valid_universe_mask = ((past_returns.count() >= self.min_history) &
-            (~self.returns.loc[t].isnull()))
+        if self.online_usage:
+            valid_universe_mask = past_returns.count() >= self.min_history
+        else:
+            valid_universe_mask = ((past_returns.count() >= self.min_history) &
+                (~self.returns.loc[t].isnull()))
         if sum(valid_universe_mask) <= 1:
             raise DataError(
                 f'The trading universe at time {t} has size less or equal'
@@ -1191,10 +1189,9 @@ class UserProvidedMarketData(MarketDataInMemory):
         make sure your provided dataframes have a timezone aware datetime
         index. Its returns are the risk-free rate.
     :type cash_key: str
-    :param do_asset_selection: Whether to remove assets that don't meet the
-        ``min_history`` or have ``np.nan`` return for the given time. Default
-        True.
-    :type do_asset_selection: bool
+    :param online_usage: Disable removal of assets that have ``np.nan`` returns
+        for the given time. Default False.
+    :type online_usage: bool
     """
 
     # pylint: disable=too-many-arguments
@@ -1204,7 +1201,7 @@ def __init__(self, returns, volumes=None, prices=None,
                  base_location=BASE_LOCATION,
                  grace_period=pd.Timedelta('1d'),
                  cash_key='USDOLLAR',
-                 do_asset_selection=True):
+                 online_usage=False):
 
         if returns is None:
             raise SyntaxError(
@@ -1228,7 +1225,7 @@ def __init__(self, returns, volumes=None, prices=None,
             base_location=base_location,
             cash_key=cash_key,
             min_history=min_history,
-            do_asset_selection=do_asset_selection)
+            online_usage=online_usage)
 
 
 class DownloadedMarketData(MarketDataInMemory):
@@ -1262,10 +1259,9 @@ class DownloadedMarketData(MarketDataInMemory):
         We implement ``'weekly'``, ``'monthly'``, ``'quarterly'`` and
         ``'annual'``. By default (None) don't down-sample.
     :type trading_frequency: str or None
-    :param do_asset_selection: Whether to remove assets that don't meet the
-        ``min_history`` or have ``np.nan`` return for the given time. You need
-        to set this to False for online usage. Default True.
-    :type do_asset_selection: bool
+    :param online_usage: Disable removal of assets that have ``np.nan`` returns
+        for the given time. Default False.
+    :type online_usage: bool
     """
 
     # pylint: disable=too-many-arguments
@@ -1278,7 +1274,7 @@ def __init__(self,
                  min_history=pd.Timedelta('365.24d'),
                  grace_period=pd.Timedelta('1d'),
                  trading_frequency=None,
-                 do_asset_selection=True):
+                 online_usage=False):
         """Initializer."""
 
         # drop duplicates and ensure ordering
@@ -1302,7 +1298,7 @@ def __init__(self,
             base_location=base_location,
             cash_key=cash_key,
             min_history=min_history,
-            do_asset_selection=do_asset_selection)
+            online_usage=online_usage)
 
     def _get_market_data(self, universe, grace_period, storage_backend):
         """Download market data."""

cvxportfolio/policies.py

@@ -66,8 +66,8 @@ def execute(self, h, market_data, t=None):
         :param t: Time at which we execute. If None (the default), the
             last timestamp in the trading calendar provided by the
             :class:`MarketData` instance is used. Note: if you use a default
-            market data server, you probably want to set ``do_asset_selection``
-            to ``False`` for online usage.
+            market data server, you probably want to set their ``online_usage``
+            argument to ``True``.
         :type t: pandas.Timestamp or None
 
         :raises cvxportfolio.errors.DataError: Holdings vector sum to a

cvxportfolio/tests/test_policies.py

@@ -639,7 +639,7 @@ def test_execute(self):
                     returns=self.returns, volumes=self.volumes,
                     cash_key='cash', base_location=self.datadir,
                     min_history=pd.Timedelta('0d'),
-                    do_asset_selection=False)
+                    online_usage=True)
 
         execution_online = policy.execute(market_data=market_data, h=h)
         self.assertTrue(np.all(execution[0] == execution_online[0]))

examples/strategies/dow30_daily.py

@@ -31,7 +31,6 @@
 .. code:: bash
 
     python -m examples.strategies.dow30daily
-
 """
 
 import cvxportfolio as cvx

examples/strategies/strategy_executor.py

@@ -281,7 +281,7 @@ def run_execute_strategy(self):
         u, t, _ = self.execute_strategy(
             current_holdings=h,
             market_data=cvx.DownloadedMarketData(
-                self.universe, do_asset_selection=False),
+                self.universe, online_usage=True),
             **self.all_hyper_params[hp_index])
 
         assert t == self.today

examples/universes.py

@@ -20,20 +20,20 @@
 We could also save each universe in a ``json`` file.
 """
 
-# This was generated on 2023-12-12 15:18:33.706712+00:00
+# This was generated on 2023-12-27 06:55:30.344592+00:00
 
 SP500 = \
 ['A', 'AAL', 'AAPL', 'ABBV', 'ABNB', 'ABT', 'ACGL', 'ACN', 'ADBE', 'ADI',
  'ADM', 'ADP', 'ADSK', 'AEE', 'AEP', 'AES', 'AFL', 'AIG', 'AIZ', 'AJG', 'AKAM',
- 'ALB', 'ALGN', 'ALK', 'ALL', 'ALLE', 'AMAT', 'AMCR', 'AMD', 'AME', 'AMGN',
- 'AMP', 'AMT', 'AMZN', 'ANET', 'ANSS', 'AON', 'AOS', 'APA', 'APD', 'APH',
- 'APTV', 'ARE', 'ATO', 'AVB', 'AVGO', 'AVY', 'AWK', 'AXON', 'AXP', 'AZO', 'BA',
- 'BAC', 'BALL', 'BAX', 'BBWI', 'BBY', 'BDX', 'BEN', 'BF-B', 'BG', 'BIIB',
- 'BIO', 'BK', 'BKNG', 'BKR', 'BLK', 'BMY', 'BR', 'BRK-B', 'BRO', 'BSX', 'BWA',
- 'BX', 'BXP', 'C', 'CAG', 'CAH', 'CARR', 'CAT', 'CB', 'CBOE', 'CBRE', 'CCI',
- 'CCL', 'CDAY', 'CDNS', 'CDW', 'CE', 'CEG', 'CF', 'CFG', 'CHD', 'CHRW', 'CHTR',
- 'CI', 'CINF', 'CL', 'CLX', 'CMA', 'CMCSA', 'CME', 'CMG', 'CMI', 'CMS', 'CNC',
- 'CNP', 'COF', 'COO', 'COP', 'COR', 'COST', 'CPB', 'CPRT', 'CPT', 'CRL', 'CRM',
+ 'ALB', 'ALGN', 'ALL', 'ALLE', 'AMAT', 'AMCR', 'AMD', 'AME', 'AMGN', 'AMP',
+ 'AMT', 'AMZN', 'ANET', 'ANSS', 'AON', 'AOS', 'APA', 'APD', 'APH', 'APTV',
+ 'ARE', 'ATO', 'AVB', 'AVGO', 'AVY', 'AWK', 'AXON', 'AXP', 'AZO', 'BA', 'BAC',
+ 'BALL', 'BAX', 'BBWI', 'BBY', 'BDX', 'BEN', 'BF-B', 'BG', 'BIIB', 'BIO', 'BK',
+ 'BKNG', 'BKR', 'BLDR', 'BLK', 'BMY', 'BR', 'BRK-B', 'BRO', 'BSX', 'BWA', 'BX',
+ 'BXP', 'C', 'CAG', 'CAH', 'CARR', 'CAT', 'CB', 'CBOE', 'CBRE', 'CCI', 'CCL',
+ 'CDAY', 'CDNS', 'CDW', 'CE', 'CEG', 'CF', 'CFG', 'CHD', 'CHRW', 'CHTR', 'CI',
+ 'CINF', 'CL', 'CLX', 'CMA', 'CMCSA', 'CME', 'CMG', 'CMI', 'CMS', 'CNC', 'CNP',
+ 'COF', 'COO', 'COP', 'COR', 'COST', 'CPB', 'CPRT', 'CPT', 'CRL', 'CRM',
  'CSCO', 'CSGP', 'CSX', 'CTAS', 'CTLT', 'CTRA', 'CTSH', 'CTVA', 'CVS', 'CVX',
  'CZR', 'D', 'DAL', 'DD', 'DE', 'DFS', 'DG', 'DGX', 'DHI', 'DHR', 'DIS', 'DLR',
  'DLTR', 'DOV', 'DOW', 'DPZ', 'DRI', 'DTE', 'DUK', 'DVA', 'DVN', 'DXCM', 'EA',
@@ -46,44 +46,43 @@
  'HAL', 'HAS', 'HBAN', 'HCA', 'HD', 'HES', 'HIG', 'HII', 'HLT', 'HOLX', 'HON',
  'HPE', 'HPQ', 'HRL', 'HSIC', 'HST', 'HSY', 'HUBB', 'HUM', 'HWM', 'IBM', 'ICE',
  'IDXX', 'IEX', 'IFF', 'ILMN', 'INCY', 'INTC', 'INTU', 'INVH', 'IP', 'IPG',
- 'IQV', 'IR', 'IRM', 'ISRG', 'IT', 'ITW', 'IVZ', 'J', 'JBHT', 'JCI', 'JKHY',
- 'JNJ', 'JNPR', 'JPM', 'K', 'KDP', 'KEY', 'KEYS', 'KHC', 'KIM', 'KLAC', 'KMB',
- 'KMI', 'KMX', 'KO', 'KR', 'KVUE', 'L', 'LDOS', 'LEN', 'LH', 'LHX', 'LIN',
- 'LKQ', 'LLY', 'LMT', 'LNT', 'LOW', 'LRCX', 'LULU', 'LUV', 'LVS', 'LW', 'LYB',
- 'LYV', 'MA', 'MAA', 'MAR', 'MAS', 'MCD', 'MCHP', 'MCK', 'MCO', 'MDLZ', 'MDT',
- 'MET', 'META', 'MGM', 'MHK', 'MKC', 'MKTX', 'MLM', 'MMC', 'MMM', 'MNST', 'MO',
- 'MOH', 'MOS', 'MPC', 'MPWR', 'MRK', 'MRNA', 'MRO', 'MS', 'MSCI', 'MSFT',
- 'MSI', 'MTB', 'MTCH', 'MTD', 'MU', 'NCLH', 'NDAQ', 'NDSN', 'NEE', 'NEM',
- 'NFLX', 'NI', 'NKE', 'NOC', 'NOW', 'NRG', 'NSC', 'NTAP', 'NTRS', 'NUE',
+ 'IQV', 'IR', 'IRM', 'ISRG', 'IT', 'ITW', 'IVZ', 'J', 'JBHT', 'JBL', 'JCI',
+ 'JKHY', 'JNJ', 'JNPR', 'JPM', 'K', 'KDP', 'KEY', 'KEYS', 'KHC', 'KIM', 'KLAC',
+ 'KMB', 'KMI', 'KMX', 'KO', 'KR', 'KVUE', 'L', 'LDOS', 'LEN', 'LH', 'LHX',
+ 'LIN', 'LKQ', 'LLY', 'LMT', 'LNT', 'LOW', 'LRCX', 'LULU', 'LUV', 'LVS', 'LW',
+ 'LYB', 'LYV', 'MA', 'MAA', 'MAR', 'MAS', 'MCD', 'MCHP', 'MCK', 'MCO', 'MDLZ',
+ 'MDT', 'MET', 'META', 'MGM', 'MHK', 'MKC', 'MKTX', 'MLM', 'MMC', 'MMM',
+ 'MNST', 'MO', 'MOH', 'MOS', 'MPC', 'MPWR', 'MRK', 'MRNA', 'MRO', 'MS', 'MSCI',
+ 'MSFT', 'MSI', 'MTB', 'MTCH', 'MTD', 'MU', 'NCLH', 'NDAQ', 'NDSN', 'NEE',
+ 'NEM', 'NFLX', 'NI', 'NKE', 'NOC', 'NOW', 'NRG', 'NSC', 'NTAP', 'NTRS', 'NUE',
  'NVDA', 'NVR', 'NWS', 'NWSA', 'NXPI', 'O', 'ODFL', 'OKE', 'OMC', 'ON', 'ORCL',
  'ORLY', 'OTIS', 'OXY', 'PANW', 'PARA', 'PAYC', 'PAYX', 'PCAR', 'PCG', 'PEAK',
  'PEG', 'PEP', 'PFE', 'PFG', 'PG', 'PGR', 'PH', 'PHM', 'PKG', 'PLD', 'PM',
  'PNC', 'PNR', 'PNW', 'PODD', 'POOL', 'PPG', 'PPL', 'PRU', 'PSA', 'PSX', 'PTC',
  'PWR', 'PXD', 'PYPL', 'QCOM', 'QRVO', 'RCL', 'REG', 'REGN', 'RF', 'RHI',
  'RJF', 'RL', 'RMD', 'ROK', 'ROL', 'ROP', 'ROST', 'RSG', 'RTX', 'RVTY', 'SBAC',
- 'SBUX', 'SCHW', 'SEDG', 'SEE', 'SHW', 'SJM', 'SLB', 'SNA', 'SNPS', 'SO',
- 'SPG', 'SPGI', 'SRE', 'STE', 'STLD', 'STT', 'STX', 'STZ', 'SWK', 'SWKS',
- 'SYF', 'SYK', 'SYY', 'T', 'TAP', 'TDG', 'TDY', 'TECH', 'TEL', 'TER', 'TFC',
- 'TFX', 'TGT', 'TJX', 'TMO', 'TMUS', 'TPR', 'TRGP', 'TRMB', 'TROW', 'TRV',
- 'TSCO', 'TSLA', 'TSN', 'TT', 'TTWO', 'TXN', 'TXT', 'TYL', 'UAL', 'UDR', 'UHS',
- 'ULTA', 'UNH', 'UNP', 'UPS', 'URI', 'USB', 'V', 'VFC', 'VICI', 'VLO', 'VLTO',
- 'VMC', 'VRSK', 'VRSN', 'VRTX', 'VTR', 'VTRS', 'VZ', 'WAB', 'WAT', 'WBA',
- 'WBD', 'WDC', 'WEC', 'WELL', 'WFC', 'WHR', 'WM', 'WMB', 'WMT', 'WRB', 'WRK',
- 'WST', 'WTW', 'WY', 'WYNN', 'XEL', 'XOM', 'XRAY', 'XYL', 'YUM', 'ZBH', 'ZBRA',
- 'ZION', 'ZTS']
+ 'SBUX', 'SCHW', 'SHW', 'SJM', 'SLB', 'SNA', 'SNPS', 'SO', 'SPG', 'SPGI',
+ 'SRE', 'STE', 'STLD', 'STT', 'STX', 'STZ', 'SWK', 'SWKS', 'SYF', 'SYK', 'SYY',
+ 'T', 'TAP', 'TDG', 'TDY', 'TECH', 'TEL', 'TER', 'TFC', 'TFX', 'TGT', 'TJX',
+ 'TMO', 'TMUS', 'TPR', 'TRGP', 'TRMB', 'TROW', 'TRV', 'TSCO', 'TSLA', 'TSN',
+ 'TT', 'TTWO', 'TXN', 'TXT', 'TYL', 'UAL', 'UBER', 'UDR', 'UHS', 'ULTA', 'UNH',
+ 'UNP', 'UPS', 'URI', 'USB', 'V', 'VFC', 'VICI', 'VLO', 'VLTO', 'VMC', 'VRSK',
+ 'VRSN', 'VRTX', 'VTR', 'VTRS', 'VZ', 'WAB', 'WAT', 'WBA', 'WBD', 'WDC', 'WEC',
+ 'WELL', 'WFC', 'WHR', 'WM', 'WMB', 'WMT', 'WRB', 'WRK', 'WST', 'WTW', 'WY',
+ 'WYNN', 'XEL', 'XOM', 'XRAY', 'XYL', 'YUM', 'ZBH', 'ZBRA', 'ZION', 'ZTS']
 
 NDX100 = \
-['AAPL', 'ABNB', 'ADBE', 'ADI', 'ADP', 'ADSK', 'AEP', 'ALGN', 'AMAT', 'AMD',
- 'AMGN', 'AMZN', 'ANSS', 'ASML', 'AVGO', 'AZN', 'BIIB', 'BKNG', 'BKR', 'CDNS',
- 'CEG', 'CHTR', 'CMCSA', 'COST', 'CPRT', 'CRWD', 'CSCO', 'CSGP', 'CSX', 'CTAS',
- 'CTSH', 'DDOG', 'DLTR', 'DXCM', 'EA', 'EBAY', 'ENPH', 'EXC', 'FANG', 'FAST',
+['AAPL', 'ABNB', 'ADBE', 'ADI', 'ADP', 'ADSK', 'AEP', 'AMAT', 'AMD', 'AMGN',
+ 'AMZN', 'ANSS', 'ASML', 'AVGO', 'AZN', 'BIIB', 'BKNG', 'BKR', 'CCEP', 'CDNS',
+ 'CDW', 'CEG', 'CHTR', 'CMCSA', 'COST', 'CPRT', 'CRWD', 'CSCO', 'CSGP', 'CSX',
+ 'CTAS', 'CTSH', 'DASH', 'DDOG', 'DLTR', 'DXCM', 'EA', 'EXC', 'FANG', 'FAST',
  'FTNT', 'GEHC', 'GFS', 'GILD', 'GOOG', 'GOOGL', 'HON', 'IDXX', 'ILMN', 'INTC',
- 'INTU', 'ISRG', 'JD', 'KDP', 'KHC', 'KLAC', 'LCID', 'LRCX', 'LULU', 'MAR',
- 'MCHP', 'MDLZ', 'MELI', 'META', 'MNST', 'MRNA', 'MRVL', 'MSFT', 'MU', 'NFLX',
- 'NVDA', 'NXPI', 'ODFL', 'ON', 'ORLY', 'PANW', 'PAYX', 'PCAR', 'PDD', 'PEP',
- 'PYPL', 'QCOM', 'REGN', 'ROST', 'SBUX', 'SGEN', 'SIRI', 'SNPS', 'TEAM',
- 'TMUS', 'TSLA', 'TTD', 'TXN', 'VRSK', 'VRTX', 'WBA', 'WBD', 'WDAY', 'XEL',
- 'ZM', 'ZS']
+ 'INTU', 'ISRG', 'KDP', 'KHC', 'KLAC', 'LRCX', 'LULU', 'MAR', 'MCHP', 'MDB',
+ 'MDLZ', 'MELI', 'META', 'MNST', 'MRNA', 'MRVL', 'MSFT', 'MU', 'NFLX', 'NVDA',
+ 'NXPI', 'ODFL', 'ON', 'ORLY', 'PANW', 'PAYX', 'PCAR', 'PDD', 'PEP', 'PYPL',
+ 'QCOM', 'REGN', 'ROP', 'ROST', 'SBUX', 'SIRI', 'SNPS', 'SPLK', 'TEAM', 'TMUS',
+ 'TSLA', 'TTD', 'TTWO', 'TXN', 'VRSK', 'VRTX', 'WBA', 'WBD', 'WDAY', 'XEL',
+ 'ZS']
 
 DOW30 = \
 ['AAPL', 'AMGN', 'AXP', 'BA', 'CAT', 'CRM', 'CSCO', 'CVX', 'DIS', 'DOW', 'GS',

pyproject.toml

@@ -14,7 +14,7 @@ dependencies = ["pandas", "numpy", "matplotlib", "requests", "cvxpy",
 docs = ["sphinx", "furo"]
 dev = ["build", "twine", "coverage", "diff_cover", "pylint", "isort",
     "autopep8", "docformatter", "beautifulsoup4"]
-examples = ['beatifulsoup4', 'pandas_datareader', 'yfinance', 
+examples = ['beautifulsoup4', 'lxml', 'pandas_datareader', 'yfinance',
     'clarabel', 'ecos']
 
 [project.urls]