Skip to content

Commit

Permalink
better control of 79 extracts
Browse files Browse the repository at this point in the history 
ref #3 & #4
  • Loading branch information
@wibeasley
wibeasley committed Oct 17, 2018
1 parent 692622f commit 09921d5
Show file tree
Hide file tree
Showing 6 changed files with 141,838 additions and 179,843 deletions.
307,975 changes: 135,801 additions & 172,174 deletions Base79/ImportDataSet.Designer.cs
View file

Large diffs are not rendered by default.

12,468 changes: 5,401 additions & 7,067 deletions Base79/ImportDataSet.xsd
View file

Large diffs are not rendered by default.

34 changes: 17 additions & 17 deletions Base79/ImportDataSet.xss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,24 @@
Changes to this file may cause incorrect behavior and will be lost if
the code is regenerated.
</autogenerated>-->
<DiagramLayout xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" ex:showrelationlabel="False" ViewPortX="-10" ViewPortY="46" xmlns:ex="urn:schemas-microsoft-com:xml-msdatasource-layout-extended" xmlns="urn:schemas-microsoft-com:xml-msdatasource-layout">
<DiagramLayout xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" ex:showrelationlabel="False" ViewPortX="38" ViewPortY="46" xmlns:ex="urn:schemas-microsoft-com:xml-msdatasource-layout-extended" xmlns="urn:schemas-microsoft-com:xml-msdatasource-layout">
<Shapes>
<Shape ID="DesignTable:tblLinks2004Gen2" ZOrder="16" X="526" Y="425" Height="172" Width="248" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="121" />
<Shape ID="DesignTable:tblGen1Outcomes" ZOrder="15" X="1051" Y="441" Height="305" Width="249" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2OutcomesHeight" ZOrder="14" X="1341" Y="444" Height="305" Width="289" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen1Explicit" ZOrder="13" X="849" Y="82" Height="305" Width="194" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen1Implicit" ZOrder="12" X="1053" Y="91" Height="305" Width="235" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2OutcomesWeight" ZOrder="11" X="1533" Y="418" Height="305" Width="291" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblLinks2004Gen1" ZOrder="10" X="291" Y="439" Height="248" Width="248" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="197" />
<Shape ID="DesignTable:tblGen2OutcomesMath" ZOrder="9" X="1559" Y="80" Height="305" Width="280" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen1GeocodeSanitized" ZOrder="8" X="48" Y="87" Height="305" Width="294" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblRosterGen1Assignment" ZOrder="7" X="124" Y="753" Height="305" Width="296" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2Links" ZOrder="6" X="48" Y="437" Height="305" Width="220" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen1Links" ZOrder="5" X="1317" Y="97" Height="305" Width="220" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2LinksFromGen1" ZOrder="4" X="573" Y="79" Height="305" Width="240" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2FatherFromGen1" ZOrder="3" X="369" Y="102" Height="305" Width="205" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2ImplicitFather" ZOrder="2" X="821" Y="529" Height="305" Width="218" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2FatherFromGen1Death" ZOrder="1" X="583" Y="653" Height="305" Width="221" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblLinks2004Gen2" ZOrder="10" X="540" Y="425" Height="172" Width="248" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="121" />
<Shape ID="DesignTable:tblGen1Explicit" ZOrder="16" X="849" Y="82" Height="305" Width="194" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblLinks2004Gen1" ZOrder="15" X="291" Y="439" Height="248" Width="248" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="197" />
<Shape ID="DesignTable:tblGen1GeocodeSanitized" ZOrder="14" X="48" Y="87" Height="305" Width="294" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblRosterGen1Assignment" ZOrder="13" X="124" Y="753" Height="305" Width="296" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2LinksFromGen1" ZOrder="12" X="573" Y="79" Height="305" Width="240" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2FatherFromGen1" ZOrder="11" X="369" Y="102" Height="305" Width="205" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2ImplicitFather" ZOrder="9" X="821" Y="529" Height="305" Width="218" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2FatherFromGen1Death" ZOrder="8" X="565" Y="656" Height="305" Width="233" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2Links" ZOrder="7" X="48" Y="427" Height="305" Width="220" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen1Links" ZOrder="6" X="1317" Y="102" Height="305" Width="220" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen1Outcomes" ZOrder="5" X="1057" Y="474" Height="305" Width="249" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen1Implicit" ZOrder="4" X="1059" Y="95" Height="305" Width="235" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2OutcomesHeight" ZOrder="3" X="1516" Y="449" Height="305" Width="289" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2OutcomesWeight" ZOrder="2" X="1639" Y="498" Height="305" Width="291" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
<Shape ID="DesignTable:tblGen2OutcomesMath" ZOrder="1" X="1737" Y="552" Height="305" Width="280" AdapterExpanded="true" DataTableExpanded="true" OldAdapterHeight="0" OldDataTableHeight="0" SplitterPosition="254" />
</Shapes>
<Connectors />
</DiagramLayout>
44 changes: 27 additions & 17 deletions dal/import-79-raw.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,25 +27,25 @@ directory_in <- "data-unshared/raw"
columns_to_drop <- c("A0002600", "Y2267000")

ds_extract <- tibble::tribble(
~schema , ~table_name , ~file_name
,"Extract" , "tblGen1Explicit" , "nlsy79-gen1/Gen1Explicit"
,"Extract" , "tblGen1Implicit" , "nlsy79-gen1/Gen1Implicit"
,"Extract" , "tblGen1Links" , "nlsy79-gen1/Gen1Links"
,"Extract" , "tblGen1Outcomes" , "nlsy79-gen1/Gen1Outcomes"
,"Extract" , "tblGen1GeocodeSanitized" , "nlsy79-gen1/Gen1GeocodeSanitized"
~schema , ~table_name , ~generation , ~file_name
,"Extract" , "tblGen1Explicit" , 1L , "nlsy79-gen1/Gen1Explicit"
,"Extract" , "tblGen1Implicit" , 1L , "nlsy79-gen1/Gen1Implicit"
,"Extract" , "tblGen1Links" , 1L , "nlsy79-gen1/Gen1Links"
,"Extract" , "tblGen1Outcomes" , 1L , "nlsy79-gen1/Gen1Outcomes"
,"Extract" , "tblGen1GeocodeSanitized" , 1L , "nlsy79-gen1/Gen1GeocodeSanitized"
# "Process.tblLURosterGen1" , "nlsy79-gen1/RosterGen1"
# tblGen1MzDzDistinction2010
#
,"Extract" , "tblGen2FatherFromGen1" , "nlsy79-gen2/Gen2FatherFromGen1"
,"Extract" , "tblGen2ImplicitFather" , "nlsy79-gen2/Gen2ImplicitFather"
,"Extract" , "tblGen2Links" , "nlsy79-gen2/Gen2Links"
,"Extract" , "tblGen2LinksFromGen1" , "nlsy79-gen2/Gen2LinksFromGen1"
,"Extract" , "tblGen2OutcomesHeight" , "nlsy79-gen2/Gen2OutcomesHeight"
,"Extract" , "tblGen2OutcomesMath" , "nlsy79-gen2/Gen2OutcomesMath"
,"Extract" , "tblGen2OutcomesWeight" , "nlsy79-gen2/Gen2OutcomesWeight"
,"Extract" , "tblGen2FatherFromGen1" , 2L , "nlsy79-gen2/Gen2FatherFromGen1"
,"Extract" , "tblGen2ImplicitFather" , 2L , "nlsy79-gen2/Gen2ImplicitFather"
,"Extract" , "tblGen2Links" , 2L , "nlsy79-gen2/Gen2Links"
,"Extract" , "tblGen2LinksFromGen1" , 2L , "nlsy79-gen2/Gen2LinksFromGen1"
,"Extract" , "tblGen2OutcomesHeight" , 2L , "nlsy79-gen2/Gen2OutcomesHeight"
,"Extract" , "tblGen2OutcomesMath" , 2L , "nlsy79-gen2/Gen2OutcomesMath"
,"Extract" , "tblGen2OutcomesWeight" , 2L , "nlsy79-gen2/Gen2OutcomesWeight"


,"Extract" , "tblGen2FatherFromGen1Death" , "nlsy79-gen2/Gen2FatherFromGen1Death"
,"Extract" , "tblGen2FatherFromGen1Death" , 2L , "nlsy79-gen2/Gen2FatherFromGen1Death"

# "Extract" , "tbl97Roster" , "nlsy97/97-roster.csv"
)
Expand All @@ -57,11 +57,16 @@ col_types_default <- readr::cols(
checkmate::assert_character(ds_extract$table_name , min.chars=10, any.missing=F, unique=T)
checkmate::assert_character(ds_extract$file_name , min.chars=10, any.missing=F, unique=T)

sql_template_primary_key <- "
sql_template_primary_key_gen1 <- "
ALTER TABLE {table_name_qualified} ADD CONSTRAINT
PK_{table_name} PRIMARY KEY CLUSTERED ( R0000100 )
WITH( STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
"
sql_template_primary_key_gen2 <- "
ALTER TABLE {table_name_qualified} ADD CONSTRAINT
PK_{table_name} PRIMARY KEY CLUSTERED ( C0000100 )
WITH( STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
"

# ---- load-data ---------------------------------------------------------------
start_time <- Sys.time()
Expand All @@ -79,7 +84,11 @@ ds_extract <- ds_extract %>%
sql_select = glue::glue("SELECT TOP(100) * FROM {table_name_qualified}"),
sql_truncate = glue::glue("TRUNCATE TABLE {table_name_qualified}"),
# sql_not_null = glue::glue(sql_template_not_null),
sql_primary_key = glue::glue(sql_template_primary_key)
sql_primary_key = dplyr::recode(
.data$generation,
glue::glue(sql_template_primary_key_gen1),
glue::glue(sql_template_primary_key_gen2)
)
)
testit::assert("All files should be found.", all(ds_extract$extract_exist))

Expand Down Expand Up @@ -179,7 +188,8 @@ for( i in seq_len(nrow(ds_extract)) ) { # i <- 1L
OuhscMunge::upload_sqls_rodbc(
d = d,
# d = d[1:100, ],
table_name = ds_extract$table_name_qualified[i] ,
schema_name = ds_extract$schema[i],
table_name = ds_extract$table_name[i],
dsn_name = "local-nlsy-links-79",
clear_table = F,
create_table = T
Expand Down
5 changes: 3 additions & 2 deletions dal/import-97-raw.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ DBI::dbGetInfo(channel_odbc)

channel_rodbc <- open_dsn_channel_rodbc(study)

for( i in seq_len(nrow(ds_extract)) ) { # i <- 1L
for( i in seq_len(nrow(ds_extract)) ) { # i <- 11L
# for( i in 1 ) { # i <- 1L
message(glue::glue("Uploading from `{ds_extract$path_zip[i]}` to `{ds_extract$table_name_qualified[i]}`."))

Expand Down Expand Up @@ -172,7 +172,8 @@ for( i in seq_len(nrow(ds_extract)) ) { # i <- 1L
OuhscMunge::upload_sqls_rodbc(
d = d,
# d = d[1:100, ],
table_name = ds_extract$table_name_qualified[i] ,
schema_name = ds_extract$schema[i],
table_name = ds_extract$table_name[i],
dsn_name = "local-nlsy-links-97",
clear_table = F,
create_table = T
Expand Down
Loading

0 comments on commit 09921d5

Please sign in to comment.