diff --git a/db/alpha/assets.sql b/db/alpha/assets.sql deleted file mode 100644 index 2f340ee..0000000 --- a/db/alpha/assets.sql +++ /dev/null @@ -1,14 +0,0 @@ - - --- DROP TABLE assets - -CREATE TABLE assets -( - pid int NOT NULL, - symbol varchar(20) NOT NULL, - name text NOT NULL, - asset_type varchar(20) NOT NULL, - ipo_date date NOT NULL, - delisting_date date NULL, - PRIMARY KEY (pid) -) \ No newline at end of file diff --git a/db/wrds/prices_wrds.sql b/db/wrds/prices_wrds.sql index a0c784a..edf786b 100644 --- a/db/wrds/prices_wrds.sql +++ b/db/wrds/prices_wrds.sql @@ -63,3 +63,4 @@ CREATE TABLE prices_wrds DisPERMNO varchar(20), -- int DisPERMCO varchar(20) -- int ) + diff --git a/db/wrds/prices_wrds_drop_columns.sql b/db/wrds/prices_wrds_drop_columns.sql new file mode 100644 index 0000000..161ca9f --- /dev/null +++ b/db/wrds/prices_wrds_drop_columns.sql @@ -0,0 +1,53 @@ + +--ALTER TABLE prices_wrds +--DROP COLUMN securitynm, +--DROP COLUMN permco; + +ALTER TABLE prices_wrds +DROP COLUMN yyyymmdd, +DROP COLUMN dlydelflg, +DROP COLUMN dlyprcflg, +DROP COLUMN dlycap, +DROP COLUMN dlycapflg, +DROP COLUMN dlyprevprc, +DROP COLUMN dlyprevprcflg, +DROP COLUMN dlyprevdt, +DROP COLUMN dlyprevcap, +DROP COLUMN dlyprevcapflg, +DROP COLUMN dlyretx, +DROP COLUMN dlyreti, +DROP COLUMN dlyretmissflg, +DROP COLUMN dlyretdurflg, +DROP COLUMN dlyorddivamt, +DROP COLUMN dlynonorddivamt, +DROP COLUMN dlyfacprc, +DROP COLUMN dlydistretflg, +DROP COLUMN dlyclose, +DROP COLUMN dlylow, +DROP COLUMN dlyhigh, +DROP COLUMN dlyopen, +DROP COLUMN dlynumtrd, +DROP COLUMN dlymmcnt, +DROP COLUMN dlyprcvol, +DROP COLUMN shrstartdt, +DROP COLUMN shrenddt, +DROP COLUMN shrsource, +DROP COLUMN shrfactype, +DROP COLUMN shradrflg, +DROP COLUMN disexdt, +DROP COLUMN disseqnbr, +DROP COLUMN disordinaryflg, +DROP COLUMN distype, +DROP COLUMN disfreqtype, +DROP COLUMN dispaymenttype, +DROP COLUMN disdetailtype, +DROP COLUMN distaxtype, +DROP COLUMN disorigcurtype, +DROP COLUMN disdivamt, +DROP COLUMN disfacpr, +DROP COLUMN disfacshr, +DROP COLUMN disdeclaredt, +DROP COLUMN disrecorddt, +DROP COLUMN dispaydt, +DROP COLUMN dispermno, +DROP COLUMN dispermco; \ No newline at end of file diff --git a/db/wrds/ratios_wrds.sql b/db/wrds/ratios_wrds.sql index 9d90782..0f3b89e 100644 --- a/db/wrds/ratios_wrds.sql +++ b/db/wrds/ratios_wrds.sql @@ -80,3 +80,4 @@ CREATE TABLE ratios_wrds TICKER varchar(20), cusip varchar(20) ) + diff --git a/esgtools/alpha/merge.py b/esgtools/alpha/merge.py new file mode 100644 index 0000000..a725270 --- /dev/null +++ b/esgtools/alpha/merge.py @@ -0,0 +1,70 @@ + + +from utils import sql_manager, utils, date_utils + +def merge_alpha_and_wrds_assets(sql_params=None): + + sql = sql_manager.ManagerSQL(sql_params) + + # Delete table + sql.query("drop table if exists assets") + + # Read query for creating assets table + query = """ +create table assets as +select +concat(case when w.ticker is null then coalesce(a.symbol, '0') else w.ticker end + , '-', coalesce(cast(permno as TEXT), '0')) as id +, coalesce(w.ticker, a.symbol) symbol +, w.permno +, a.name +, coalesce(a.exchange, w.primary_exch) exchange +, coalesce(a.asset_type, 'Stock') asset_type +, case + when coalesce(a.asset_type, 'Stock') = 'Stock' then coalesce(w.share_class, 'A') + else w.share_class + end share_class +, case + when ipo_date_proxy is not null and ipo_date is not null then + case when ipo_date_proxy < ipo_date then cast(ipo_date_proxy as date) + else ipo_date end + else cast(coalesce(ipo_date, ipo_date_proxy) as date) + end ipo_date +, cast(coalesce(delisting_date, delisting_date_proxy) as date) delisting_date +, case when a.symbol is not null then 1 else 0 end in_alpha +, case + when a.status is null and delisting_date_proxy is not null then 'Delisted' + else a.status + end status +, lud alpha_lud +, Now() lud +from assets_wrds w +full outer join ( + select * + from ( + select * + , row_number() over(partition by symbol order by coalesce(delisting_date, CURRENT_DATE) desc) rnk + from assets_alpha + where symbol not like '%-%' + order by symbol, delisting_date + ) a + where rnk = 1 +) a +on +w.delisting_date_proxy is NULL +and w.rnk=1 +and a.symbol = w.ticker + """ + + + # Create assets table + sql.query(query) + + + + + + + + + diff --git a/esgtools/alpha/table.py b/esgtools/alpha/table.py index 9778705..dc3e970 100644 --- a/esgtools/alpha/table.py +++ b/esgtools/alpha/table.py @@ -158,7 +158,7 @@ def update_all(self, date_input=None): # Download listing status data = self.get_api_data(date_input) - # Update assets_table + # Update assets table self.sql.clean_table(self.table_name) self.sql.upload_df_chunks(self.table_name, data) diff --git a/esgtools/get_assets.py b/esgtools/get_assets.py index 7043dc3..89763f0 100644 --- a/esgtools/get_assets.py +++ b/esgtools/get_assets.py @@ -9,7 +9,7 @@ def lambda_handler(event, context): - """Sample pure Lambda function + """Get asset symbols that need to be refreshed Parameters ---------- diff --git a/esgtools/update_assets.py b/esgtools/update_assets.py index 0017fc7..ca051de 100644 --- a/esgtools/update_assets.py +++ b/esgtools/update_assets.py @@ -5,7 +5,7 @@ from ast import literal_eval from utils import sql_manager, aws -from alpha import api, table +from alpha import api, table, merge def lambda_handler(event, context): @@ -41,6 +41,9 @@ def lambda_handler(event, context): "assets_alpha", [], alpha_scraper, sql_params=db_credentials, max_workers=os.cpu_count()) alpha_assets.update_all() + merge.merge_alpha_and_wrds_assets(sql_params=db_credentials) + + return { "statusCode": 200, "body": json.dumps({