Merge pull request #48 from nico-corthorn/wrds

Merge wrds and alpha assets

db/wrds/prices_wrds.sql

@@ -63,3 +63,4 @@ CREATE TABLE prices_wrds
 	DisPERMNO varchar(20), -- int
 	DisPERMCO varchar(20) -- int
 )
+

db/wrds/prices_wrds_drop_columns.sql

@@ -0,0 +1,53 @@
+
+--ALTER TABLE prices_wrds  
+--DROP COLUMN securitynm,
+--DROP COLUMN permco;
+
+ALTER TABLE prices_wrds
+DROP COLUMN yyyymmdd,
+DROP COLUMN dlydelflg,
+DROP COLUMN dlyprcflg,
+DROP COLUMN dlycap,
+DROP COLUMN dlycapflg,
+DROP COLUMN dlyprevprc,
+DROP COLUMN dlyprevprcflg,
+DROP COLUMN dlyprevdt,
+DROP COLUMN dlyprevcap,
+DROP COLUMN dlyprevcapflg,
+DROP COLUMN dlyretx,
+DROP COLUMN dlyreti,
+DROP COLUMN dlyretmissflg,
+DROP COLUMN dlyretdurflg,
+DROP COLUMN dlyorddivamt,
+DROP COLUMN dlynonorddivamt,
+DROP COLUMN dlyfacprc,
+DROP COLUMN dlydistretflg,
+DROP COLUMN dlyclose,
+DROP COLUMN dlylow,
+DROP COLUMN dlyhigh,
+DROP COLUMN dlyopen,
+DROP COLUMN dlynumtrd,
+DROP COLUMN dlymmcnt,
+DROP COLUMN dlyprcvol,
+DROP COLUMN shrstartdt,
+DROP COLUMN shrenddt,
+DROP COLUMN shrsource,
+DROP COLUMN shrfactype,
+DROP COLUMN shradrflg,
+DROP COLUMN disexdt,
+DROP COLUMN disseqnbr,
+DROP COLUMN disordinaryflg,
+DROP COLUMN distype,
+DROP COLUMN disfreqtype,
+DROP COLUMN dispaymenttype,
+DROP COLUMN disdetailtype,
+DROP COLUMN distaxtype,
+DROP COLUMN disorigcurtype,
+DROP COLUMN disdivamt,
+DROP COLUMN disfacpr,
+DROP COLUMN disfacshr,
+DROP COLUMN disdeclaredt,
+DROP COLUMN disrecorddt,
+DROP COLUMN dispaydt,
+DROP COLUMN dispermno,
+DROP COLUMN dispermco;

db/wrds/ratios_wrds.sql

@@ -80,3 +80,4 @@ CREATE TABLE ratios_wrds
 	TICKER varchar(20),
 	cusip varchar(20)
 )
+

esgtools/alpha/merge.py

@@ -0,0 +1,70 @@
+
+
+from utils import sql_manager, utils, date_utils
+
+def merge_alpha_and_wrds_assets(sql_params=None):
+
+    sql = sql_manager.ManagerSQL(sql_params)
+
+    # Delete table
+    sql.query("drop table if exists assets")
+
+    # Read query for creating assets table
+    query = """
+create table assets as
+select 
+concat(case when w.ticker is null then coalesce(a.symbol, '0') else w.ticker end
+	   , '-', coalesce(cast(permno as TEXT), '0')) as id
+, coalesce(w.ticker, a.symbol) symbol
+, w.permno
+, a.name
+, coalesce(a.exchange, w.primary_exch) exchange
+, coalesce(a.asset_type, 'Stock') asset_type
+, case 
+	when coalesce(a.asset_type, 'Stock') = 'Stock' then coalesce(w.share_class, 'A')
+	else w.share_class
+	end share_class
+, case 
+	when ipo_date_proxy is not null and ipo_date is not null then 
+		case when ipo_date_proxy < ipo_date then cast(ipo_date_proxy as date)
+		else ipo_date end
+	else cast(coalesce(ipo_date, ipo_date_proxy) as date) 
+	end ipo_date
+, cast(coalesce(delisting_date, delisting_date_proxy) as date) delisting_date
+, case when a.symbol is not null then 1 else 0 end in_alpha
+, case 
+	when a.status is null and delisting_date_proxy is not null then 'Delisted' 
+	else a.status
+  end status
+, lud alpha_lud
+, Now() lud
+from assets_wrds w
+full outer join (
+	select * 
+	from (
+		select *
+		, row_number() over(partition by symbol order by coalesce(delisting_date, CURRENT_DATE) desc) rnk
+		from assets_alpha
+		where symbol not like '%-%'
+		order by symbol, delisting_date
+	) a
+	where rnk = 1
+) a
+on 
+w.delisting_date_proxy is NULL
+and w.rnk=1
+and a.symbol = w.ticker    
+    """
+
+
+    # Create assets table
+    sql.query(query)
+
+
+
+
+
+
+
+
+

esgtools/alpha/table.py

@@ -158,7 +158,7 @@ def update_all(self, date_input=None):
         # Download listing status
         data = self.get_api_data(date_input)
 
-        # Update assets_table
+        # Update assets table
         self.sql.clean_table(self.table_name)
         self.sql.upload_df_chunks(self.table_name, data)
 

esgtools/get_assets.py

@@ -9,7 +9,7 @@
 
 
 def lambda_handler(event, context):
-    """Sample pure Lambda function
+    """Get asset symbols that need to be refreshed
 
     Parameters
     ----------

esgtools/update_assets.py

@@ -5,7 +5,7 @@
 from ast import literal_eval
 
 from utils import sql_manager, aws
-from alpha import api, table
+from alpha import api, table, merge
 
 
 def lambda_handler(event, context):
@@ -41,6 +41,9 @@ def lambda_handler(event, context):
             "assets_alpha", [], alpha_scraper, sql_params=db_credentials, max_workers=os.cpu_count())
     alpha_assets.update_all()
 
+    merge.merge_alpha_and_wrds_assets(sql_params=db_credentials)
+
+
     return {
         "statusCode": 200,
         "body": json.dumps({