Skip to content

Commit

Permalink
removed part of data.py that was unused and caused tests to fail with…
Browse files Browse the repository at this point in the history 
… new numpy
  • Loading branch information
@enzbus
enzbus committed Aug 31, 2023
1 parent 0608c06 commit 76aa86c
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 105 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ jobs:

strategy:
matrix:
os: [ubuntu-latest] # , windows-latest, macos-latest]
python-version: ['3.11'] # ['3.9', '3.10', '3.11']
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ['3.9', '3.10', '3.11']


# Steps represent matrix sequence of tasks that will be executed as part of the job
Expand Down
186 changes: 93 additions & 93 deletions cvxportfolio/data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,98 +205,98 @@ class BaseDataStore(BaseData):
base_location = None


class SqliteDataStore(BaseDataStore):
"""Local sqlite3 database using python standard library.
Args:
location (pathlib.Path or None): pathlib.Path base location of the databases
directory or, if None, use ":memory:" for storing in RAM instead. Default
is ~/cvxportfolio/
"""

def __init__(self, base_location=BASE_LOCATION):
"""Initialize sqlite connection and if necessary create database."""
self.base_location = base_location

def __open__(self):
"""Open database connection."""
if self.base_location is None:
self.connection = sqlite3.connect(":memory:")
else:
self.connection = sqlite3.connect(
(self.base_location / self.__class__.__name__).with_suffix(".sqlite"))

def __close__(self):
"""Close database connection."""
self.connection.close()

def store(self, symbol, data):
"""Store Pandas object to sqlite.
We separately store dtypes for data consistency and safety.
Limitations: if your pandas object's index has a name it will be lost,
the index is renamed 'index'.
"""
self.__open__()
exists = pd.read_sql_query(
f"SELECT name FROM sqlite_master WHERE type='table' AND name='{symbol}'",
self.connection,
)
if len(exists):
res = self.connection.cursor().execute(f"DROP TABLE '{symbol}'")
res = self.connection.cursor().execute(
f"DROP TABLE '{symbol}___dtypes'")
self.connection.commit()

if hasattr(data.index, "levels"):
data.index = data.index.set_names(
["index"] +
[f"___level{i}" for i in range(1, len(data.index.levels))]
)
data = data.reset_index().set_index("index")
else:
data.index.name = "index"

data.to_sql(f"{symbol}", self.connection)
pd.DataFrame(data).dtypes.astype("string").to_sql(
f"{symbol}___dtypes", self.connection
)
self.__close__()

def load_raw(self, symbol):
"""Load Pandas object with datetime index from sqlite.
If data is not present in in the database, return None.
"""
try:
self.__open__()
dtypes = pd.read_sql_query(
f"SELECT * FROM {symbol}___dtypes",
self.connection,
index_col="index",
dtype={"index": "str", "0": "str"},
)
tmp = pd.read_sql_query(
f"SELECT * FROM {symbol}",
self.connection,
index_col="index",
parse_dates="index",
dtype=dict(dtypes["0"]),
)
self.__close__()
multiindex = []
for col in tmp.columns:
if col[:8] == "___level":
multiindex.append(col)
else:
break
if len(multiindex):
multiindex = [tmp.index.name] + multiindex
tmp = tmp.reset_index().set_index(multiindex)
return tmp.iloc[:, 0] if tmp.shape[1] == 1 else tmp
except pd.errors.DatabaseError:
return None
# class SqliteDataStore(BaseDataStore):
# """Local sqlite3 database using python standard library.
#
# Args:
# location (pathlib.Path or None): pathlib.Path base location of the databases
# directory or, if None, use ":memory:" for storing in RAM instead. Default
# is ~/cvxportfolio/
# """
#
# def __init__(self, base_location=BASE_LOCATION):
# """Initialize sqlite connection and if necessary create database."""
# self.base_location = base_location
#
# def __open__(self):
# """Open database connection."""
# if self.base_location is None:
# self.connection = sqlite3.connect(":memory:")
# else:
# self.connection = sqlite3.connect(
# (self.base_location / self.__class__.__name__).with_suffix(".sqlite"))
#
# def __close__(self):
# """Close database connection."""
# self.connection.close()
#
# def store(self, symbol, data):
# """Store Pandas object to sqlite.
#
# We separately store dtypes for data consistency and safety.
#
# Limitations: if your pandas object's index has a name it will be lost,
# the index is renamed 'index'.
# """
# self.__open__()
# exists = pd.read_sql_query(
# f"SELECT name FROM sqlite_master WHERE type='table' AND name='{symbol}'",
# self.connection,
# )
# if len(exists):
# res = self.connection.cursor().execute(f"DROP TABLE '{symbol}'")
# res = self.connection.cursor().execute(
# f"DROP TABLE '{symbol}___dtypes'")
# self.connection.commit()
#
# if hasattr(data.index, "levels"):
# data.index = data.index.set_names(
# ["index"] +
# [f"___level{i}" for i in range(1, len(data.index.levels))]
# )
# data = data.reset_index().set_index("index")
# else:
# data.index.name = "index"
#
# data.to_sql(f"{symbol}", self.connection)
# pd.DataFrame(data).dtypes.astype("string").to_sql(
# f"{symbol}___dtypes", self.connection
# )
# self.__close__()
#
# def load_raw(self, symbol):
# """Load Pandas object with datetime index from sqlite.
#
# If data is not present in in the database, return None.
# """
# try:
# self.__open__()
# dtypes = pd.read_sql_query(
# f"SELECT * FROM {symbol}___dtypes",
# self.connection,
# index_col="index",
# dtype={"index": "str", "0": "str"},
# )
# tmp = pd.read_sql_query(
# f"SELECT * FROM {symbol}",
# self.connection,
# index_col="index",
# parse_dates="index",
# dtype=dict(dtypes["0"]),
# )
# self.__close__()
# multiindex = []
# for col in tmp.columns:
# if col[:8] == "___level":
# multiindex.append(col)
# else:
# break
# if len(multiindex):
# multiindex = [tmp.index.name] + multiindex
# tmp = tmp.reset_index().set_index(multiindex)
# return tmp.iloc[:, 0] if tmp.shape[1] == 1 else tmp
# except pd.errors.DatabaseError:
# return None


class PickleStore(BaseDataStore):
Expand Down Expand Up @@ -463,7 +463,7 @@ def update_and_load(self, symbol):
return super().update_and_load(symbol)


class FredRate(FredBase, RateBase, SqliteDataStore):
class FredRate(FredBase, RateBase, PickleStore):
"""Load and store FRED rates like DFF."""

pass
Expand Down
20 changes: 10 additions & 10 deletions cvxportfolio/tests/test_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
Yfinance,
FredBase,
FredRate,
SqliteDataStore,
# SqliteDataStore,
TimeSeries,
PickleStore,
)
Expand Down Expand Up @@ -114,9 +114,9 @@ def test_yfinance(self):
self.assertTrue(np.allclose(
data1.loc[data.index[:-1]].Return, data.iloc[:-1].Return))

def test_sqlite3_store_series(self):
"""Test storing and retrieving of a Series with datetime index."""
self.base_test_series(SqliteDataStore, self.datadir)
# def test_sqlite3_store_series(self):
# """Test storing and retrieving of a Series with datetime index."""
# self.base_test_series(SqliteDataStore, self.datadir)

def test_local_store_series(self):
"""Test storing and retrieving of a Series with datetime index."""
Expand All @@ -126,9 +126,9 @@ def test_pickle_store_series(self):
"""Test storing and retrieving of a Series with datetime index."""
self.base_test_series(PickleStore, self.datadir)

def test_sqlite3_store_dataframe(self):
"""Test storing and retrieving of a DataFrame with datetime index."""
self.base_test_dataframe(SqliteDataStore, self.datadir)
# def test_sqlite3_store_dataframe(self):
# """Test storing and retrieving of a DataFrame with datetime index."""
# self.base_test_dataframe(SqliteDataStore, self.datadir)

def test_local_store_dataframe(self):
"""Test storing and retrieving of a DataFrame with datetime index."""
Expand All @@ -142,9 +142,9 @@ def test_local_store_multiindex(self):
"""Test storing and retrieving of a DataFrame with datetime index."""
self.base_test_multiindex(LocalDataStore, self.datadir)

def test_sqlite3_store_multiindex(self):
"""Test storing and retrieving of a DataFrame with datetime index."""
self.base_test_multiindex(SqliteDataStore, self.datadir)
# def test_sqlite3_store_multiindex(self):
# """Test storing and retrieving of a DataFrame with datetime index."""
# self.base_test_multiindex(SqliteDataStore, self.datadir)

def test_pickle_store_multiindex(self):
"""Test storing and retrieving of a DataFrame with datetime index."""
Expand Down

0 comments on commit 76aa86c

Please sign in to comment.