Skip to content
This repository has been archived by the owner on Aug 23, 2024. It is now read-only.

Add UTC loaded_at timestamp to iasWorld table pulls #10

Merged
merged 3 commits into from
Feb 7, 2024
Merged

Add UTC loaded_at timestamp to iasWorld table pulls #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e0586d1
Add UTC loaded_at timestamp to iasWorld table pulls
dfsnow Feb 6, 2024
52c6bc4
Update table schema definitions
dfsnow Feb 7, 2024
9169581
Add loaded_at timestamp to extract scripts
dfsnow Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/get-tables.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ for TABLE in ${JOB_TABLES}; do
--connect jdbc:oracle:thin:@//"$IPTS_HOSTNAME":"$IPTS_PORT"/"$IPTS_SERVICE_NAME" \
--username "$IPTS_USERNAME" \
--password-file file:///run/secrets/IPTS_PASSWORD \
--query "SELECT * FROM IASWORLD.${TABLE} WHERE \$CONDITIONS FETCH FIRST 1 ROWS ONLY" \
--query "SELECT t.*, SYS_EXTRACT_UTC(CURRENT_TIMESTAMP) AS loaded_at FROM IASWORLD.${TABLE} t WHERE \$CONDITIONS FETCH FIRST 1 ROWS ONLY" \
--hcatalog-database "$DB_NAME" \
--hcatalog-table "$TABLE" \
--drop-and-create-hcatalog-table \
Expand Down
6 changes: 4 additions & 2 deletions scripts/run-sqoop.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ for TABLE in ${JOB_TABLES}; do
NUM_MAPPERS=$(($(date +%Y) - 1998))
NUM_MAPPERS=$((NUM_MAPPERS < MAX_MAPPERS ? NUM_MAPPERS : MAX_MAPPERS))
BOUNDARY_QUERY="SELECT MIN(TAXYR), MAX(TAXYR) FROM IASWORLD.${TABLE}"
QUERY="SELECT * FROM IASWORLD.${TABLE}
QUERY="SELECT table.*, SYS_EXTRACT_UTC(CURRENT_TIMESTAMP) AS loaded_at
FROM IASWORLD.${TABLE} table
WHERE \$CONDITIONS"
else
# Make number of mappers roughly equal to number of tax years
Expand All @@ -58,7 +59,8 @@ for TABLE in ${JOB_TABLES}; do
NUM_MAPPERS=$((NUM_MAPPERS < MAX_MAPPERS ? NUM_MAPPERS : MAX_MAPPERS))
BOUNDARY_QUERY="SELECT MIN(TAXYR), MAX(TAXYR) FROM IASWORLD.${TABLE}
WHERE TAXYR ${QUERY_COND} ${QUERY_YEAR}"
QUERY="SELECT * FROM IASWORLD.${TABLE}
QUERY="SELECT table.*, SYS_EXTRACT_UTC(CURRENT_TIMESTAMP) AS loaded_at
FROM IASWORLD.${TABLE} table
WHERE TAXYR ${QUERY_COND} ${QUERY_YEAR}
AND \$CONDITIONS"
fi
Expand Down
3 changes: 2 additions & 1 deletion tables/AASYSJUR.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ CREATE TABLE `iasworld.aasysjur`(
`aprvalsum_ind` varchar(1),
`iasw_id` decimal(10,0),
`who` varchar(50),
`wen` string)
`wen` string,
`loaded_at` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
6 changes: 4 additions & 2 deletions tables/ADDN.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ CREATE TABLE `iasworld.addn`(
`eff_area` decimal(10,0),
`effageovr` decimal(4,0),
`stories` decimal(10,0),
`footprint` decimal(10,0),
`footprint` decimal(10,0),
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -166,7 +167,8 @@ CREATE TABLE `iasworld.addn_bucketed`(
`eff_area` decimal(10,0),
`effageovr` decimal(4,0),
`stories` decimal(10,0),
`footprint` decimal(10,0))
`footprint` decimal(10,0),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) SORTED BY (`seq`) INTO 10 BUCKETS
STORED AS PARQUET
Expand Down
3 changes: 2 additions & 1 deletion tables/ADDRINDX.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ CREATE TABLE `iasworld.addrindx`(
`xcoord` decimal(10,0),
`ycoord` decimal(10,0),
`strcd` varchar(10),
`strreloc` varchar(150))
`strreloc` varchar(150),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
6 changes: 4 additions & 2 deletions tables/APRVAL.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,8 @@ CREATE TABLE `iasworld.aprval`(
`mkttotval` decimal(10,0),
`ovrmraval` decimal(10,0),
`spssmraland` decimal(10,0),
`spssmrabldg` decimal(10,0),
`spssmrabldg` decimal(10,0),
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -386,7 +387,8 @@ CREATE TABLE `iasworld.aprval_bucketed`(
`mkttotval` decimal(10,0),
`ovrmraval` decimal(10,0),
`spssmraland` decimal(10,0),
`spssmrabldg` decimal(10,0))
`spssmrabldg` decimal(10,0),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) SORTED BY (`seq`) INTO 20 BUCKETS
STORED AS PARQUET
Expand Down
6 changes: 4 additions & 2 deletions tables/ASMT_ALL.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,8 @@ CREATE TABLE `iasworld.asmt_all`(
`ovrval60` decimal(10,0),
`val60` decimal(10,0),
`splitno` decimal(15,0),
`form` varchar(12),
`form` varchar(12),
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -644,7 +645,8 @@ CREATE TABLE `iasworld.asmt_all_bucketed`(
`ovrval60` decimal(10,0),
`val60` decimal(10,0),
`splitno` decimal(15,0),
`form` varchar(12))
`form` varchar(12),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) SORTED BY (`seq`) INTO 30 BUCKETS
STORED AS PARQUET
Expand Down
6 changes: 4 additions & 2 deletions tables/ASMT_HIST.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,8 @@ CREATE TABLE `iasworld.asmt_hist`(
`ovrval60` decimal(10,0),
`val60` decimal(10,0),
`splitno` decimal(15,0),
`form` varchar(12),
`form` varchar(12),
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -644,7 +645,8 @@ CREATE TABLE `iasworld.asmt_hist_bucketed`(
`ovrval60` decimal(10,0),
`val60` decimal(10,0),
`splitno` decimal(15,0),
`form` varchar(12))
`form` varchar(12),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) SORTED BY (`seq`) INTO 30 BUCKETS
STORED AS PARQUET
Expand Down
3 changes: 2 additions & 1 deletion tables/CNAME.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ CREATE TABLE `iasworld.cname`(
`addressguid` varchar(36),
`ssn2` varchar(9),
`cell_phone` varchar(25),
`business_phone` varchar(25))
`business_phone` varchar(25),
`loaded_at` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
6 changes: 4 additions & 2 deletions tables/COMDAT.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,8 @@ CREATE TABLE `iasworld.comdat`(
`external_propct` decimal(9,6),
`external_calc_rcnld` decimal(10,0),
`calc_meth` varchar(1),
`note3` string,
`note3` string,
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -304,7 +305,8 @@ CREATE TABLE `iasworld.comdat_bucketed`(
`external_propct` decimal(9,6),
`external_calc_rcnld` decimal(10,0),
`calc_meth` varchar(1),
`note3` string)
`note3` string,
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) SORTED BY (`seq`) INTO 10 BUCKETS
STORED AS PARQUET
Expand Down
3 changes: 2 additions & 1 deletion tables/COMFEAT.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ CREATE TABLE `iasworld.comfeat`(
`meas2_m` decimal(13,3),
`meas3_m` decimal(13,3),
`meas4_m` decimal(13,3),
`meas5_m` decimal(13,3))
`meas5_m` decimal(13,3),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
3 changes: 2 additions & 1 deletion tables/COMINTEXT.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,8 @@ CREATE TABLE `iasworld.comintext`(
`user25` varchar(250),
`numident` decimal(2,0),
`effageovr` decimal(4,0),
`upperflarea` decimal(9,0))
`upperflarea` decimal(9,0),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
3 changes: 2 additions & 1 deletion tables/COMNT.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ CREATE TABLE `iasworld.comnt`(
`trans_id` decimal(10,0),
`upd_status` varchar(1),
`caseno` varchar(15),
`comnt2` string)
`comnt2` string,
`loaded_at` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
3 changes: 2 additions & 1 deletion tables/COMNT3.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ CREATE TABLE `iasworld.comnt3`(
`upd_status` varchar(1),
`bldgno` varchar(10),
`unit` varchar(10),
`comnt2` string)
`comnt2` string,
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
6 changes: 4 additions & 2 deletions tables/CVLEG.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,8 @@ CREATE TABLE `iasworld.cvleg`(
`user57` varchar(80),
`user58` varchar(80),
`user59` varchar(80),
`user60` varchar(80),
`user60` varchar(80),
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -388,7 +389,8 @@ CREATE TABLE `iasworld.cvleg_bucketed`(
`user57` varchar(80),
`user58` varchar(80),
`user59` varchar(80),
`user60` varchar(80))
`user60` varchar(80),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) INTO 10 BUCKETS
STORED AS PARQUET
Expand Down
6 changes: 4 additions & 2 deletions tables/CVOWN.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,8 @@ CREATE TABLE `iasworld.cvown`(
`udate9` string,
`udate10` string,
`skip_addr_validation` varchar(1),
`user41` varchar(500),
`user41` varchar(500),
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -316,7 +317,8 @@ CREATE TABLE `iasworld.cvown_bucketed`(
`udate9` string,
`udate10` string,
`skip_addr_validation` varchar(1),
`user41` varchar(500))
`user41` varchar(500),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
6 changes: 4 additions & 2 deletions tables/CVTRAN.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,8 @@ CREATE TABLE `iasworld.cvtran`(
`userval17` decimal(10,0),
`userval18` decimal(10,0),
`userval19` decimal(10,0),
`userval20` decimal(10,0),
`userval20` decimal(10,0),
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -424,7 +425,8 @@ CREATE TABLE `iasworld.cvtran_bucketed`(
`userval17` decimal(10,0),
`userval18` decimal(10,0),
`userval19` decimal(10,0),
`userval20` decimal(10,0))
`userval20` decimal(10,0),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
3 changes: 2 additions & 1 deletion tables/DEDIT.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ CREATE TABLE `iasworld.dedit`(
`who` varchar(50),
`wen` string,
`longdesc` varchar(200),
`iasw_id` decimal(10,0))
`iasw_id` decimal(10,0),
`loaded_at` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
6 changes: 4 additions & 2 deletions tables/DWELDAT.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,8 @@ CREATE TABLE `iasworld.dweldat`(
`external_propct` decimal(9,6),
`external_calc_rcnld` decimal(10,0),
`calc_meth` varchar(1),
`note3` string,
`note3` string,
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -732,7 +733,8 @@ CREATE TABLE `iasworld.dweldat_bucketed`(
`external_propct` decimal(9,6),
`external_calc_rcnld` decimal(10,0),
`calc_meth` varchar(1),
`note3` string)
`note3` string,
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) SORTED BY (`seq`) INTO 10 BUCKETS
STORED AS PARQUET
Expand Down
3 changes: 2 additions & 1 deletion tables/ENTER.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ CREATE TABLE `iasworld.enter`(
`upd_status` varchar(1),
`seq` decimal(3,0),
`cur` varchar(1),
`deactivat` string)
`deactivat` string,
`loaded_at` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
3 changes: 2 additions & 1 deletion tables/EXADMN.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,8 @@ CREATE TABLE `iasworld.exadmn`(
`userval7` decimal(10,0),
`userval8` decimal(10,0),
`userval9` decimal(10,0),
`userval10` decimal(10,0))
`userval10` decimal(10,0),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
6 changes: 4 additions & 2 deletions tables/EXAPP.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,8 @@ CREATE TABLE `iasworld.exapp`(
`udate13` string,
`udate14` string,
`udate15` string,
`form` varchar(12),
`form` varchar(12),
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -838,7 +839,8 @@ CREATE TABLE `iasworld.exapp_bucketed`(
`udate13` string,
`udate14` string,
`udate15` string,
`form` varchar(12))
`form` varchar(12),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) SORTED BY (`seq`) INTO 20 BUCKETS
STORED AS PARQUET
Expand Down
3 changes: 2 additions & 1 deletion tables/EXCODE.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,8 @@ CREATE TABLE `iasworld.excode`(
`userval3` decimal(10,0),
`userval4` decimal(10,0),
`userval5` decimal(10,0),
`yrend` decimal(4,0))
`yrend` decimal(4,0),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
6 changes: 4 additions & 2 deletions tables/EXDET.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,8 @@ CREATE TABLE `iasworld.exdet`(
`yrbase` decimal(4,0),
`begdt` string,
`enddt` string,
`form` varchar(12),
`form` varchar(12),
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -404,7 +405,8 @@ CREATE TABLE `iasworld.exdet_bucketed`(
`yrbase` decimal(4,0),
`begdt` string,
`enddt` string,
`form` varchar(12))
`form` varchar(12),
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) SORTED BY (`seq`) INTO 20 BUCKETS
STORED AS PARQUET
Expand Down
3 changes: 2 additions & 1 deletion tables/HTAGNT.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ CREATE TABLE `iasworld.htagnt`(
`name2_suffix` varchar(20),
`name2_companyname` varchar(205),
`agent_type` decimal(10,0),
`iasw_id` decimal(10,0))
`iasw_id` decimal(10,0),
`loaded_at` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
3 changes: 2 additions & 1 deletion tables/HTDATES.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ CREATE TABLE `iasworld.htdates`(
`udate2` string,
`udate3` string,
`udate4` string,
`udate5` string)
`udate5` string,
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
STORED AS PARQUET
TBLPROPERTIES ('parquet.compression'='SNAPPY');
6 changes: 4 additions & 2 deletions tables/HTPAR.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,8 @@ CREATE TABLE `iasworld.htpar`(
`userval5` decimal(10,0),
`form` varchar(12),
`procname` varchar(10),
`procdate` string,
`procdate` string,
`loaded_at` string,
`taxyr` string)
STORED AS RCFILE;

Expand Down Expand Up @@ -640,7 +641,8 @@ CREATE TABLE `iasworld.htpar_bucketed`(
`userval5` decimal(10,0),
`form` varchar(12),
`procname` varchar(10),
`procdate` string)
`procdate` string,
`loaded_at` string)
PARTITIONED BY (`taxyr` string)
CLUSTERED BY (`parid`) SORTED BY (`seq`) INTO 15 BUCKETS
STORED AS PARQUET
Expand Down
Loading
Loading