Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
d8d1d0c
add huc12 handling
pauldzy May 12, 2025
5022e1e
fix for helper
pauldzy May 13, 2025
357bdb9
Merge branch 'owld_stubs' of https://github.com/USEPA/CIP-service int…
pauldzy May 13, 2025
1e10bb8
add upstream downstream attribute handling
pauldzy May 13, 2025
07b0ebb
Merge branch 'owld_stubs' of https://github.com/USEPA/CIP-service int…
pauldzy May 13, 2025
13cc9a6
add dumper script
pauldzy May 13, 2025
50ddfc3
refactor to better handle requests for full catchment
pauldzy Jun 1, 2025
d277bfe
add fix script for discussion
pauldzy Jun 4, 2025
155968a
tighten up tolerance for fabric creation
pauldzy Jun 16, 2025
0717207
add circle draw to demo
pauldzy Jun 19, 2025
5d3fb29
add more nhdplus fixes
pauldzy Jun 20, 2025
fe57d3d
add fix scripts
pauldzy Jun 25, 2025
cb1176a
add another fix file
pauldzy Jun 26, 2025
5d8c009
add final fixes
pauldzy Jun 28, 2025
5236932
add stubs for downloaders
pauldzy Jul 3, 2025
693033c
Merge branch 'owld_stubs' of https://github.com/USEPA/CIP-service int…
pauldzy Jul 3, 2025
4adade9
tidy up maintenance scripts
pauldzy Jul 4, 2025
85ee30c
Merge branch 'owld_stubs' of https://github.com/USEPA/CIP-service int…
pauldzy Jul 4, 2025
9c979fb
update nhdplus b set dumpers
pauldzy Jul 10, 2025
2dd0466
remove trailing spaces
pauldzy Jul 10, 2025
5b74079
Update README.md
pauldzy Jul 22, 2025
2b4b669
update support dumper
pauldzy Aug 3, 2025
db7b63f
Merge branch 'owld_stubs' of https://github.com/USEPA/CIP-service int…
pauldzy Aug 3, 2025
603a28a
update mv per sdwis changes
pauldzy Aug 4, 2025
e548d19
missed the commit
pauldzy Aug 4, 2025
c99e703
add border_status to catchment fabric
pauldzy Aug 19, 2025
4d18f69
add gis exporter vierws for selected temp and precip tables
pauldzy Aug 19, 2025
1134709
Merge branch 'owld_stubs' of https://github.com/USEPA/CIP-service int…
pauldzy Aug 19, 2025
e692202
default to HR
pauldzy Sep 23, 2025
c338d87
adjust topo dumpers
pauldzy Sep 23, 2025
f91e84f
Merge branch 'owld_stubs' of https://github.com/USEPA/CIP-service int…
pauldzy Sep 23, 2025
0766c6a
report long navs at hydroseq rather than nhdplusid
pauldzy Sep 23, 2025
cac7c42
Adjust vpu09 pruner notebook to accomodate newer data formats
pauldzy Sep 23, 2025
042d960
increment to newer docker containers
pauldzy Sep 23, 2025
fc5a75a
adjust mbr function in manifest
pauldzy Sep 23, 2025
31592d7
test with pg17
pauldzy Sep 23, 2025
7c8d46b
adjust functions for new PG17 MV security
pauldzy Sep 24, 2025
c151412
missed a reference
pauldzy Sep 24, 2025
70281b8
fix for small naming error in setup notebook
pauldzy Sep 24, 2025
d8db975
Add PGUSER env to make healthcheck pg_ready happy
pauldzy Sep 25, 2025
bbd37a2
mispelled filename
pauldzy Sep 25, 2025
8d280aa
fix typo
pauldzy Sep 25, 2025
fb0e3f0
adjust tap tests to add additional limit to us parameters
pauldzy Sep 25, 2025
f83582e
account for cases where the known region slips through as NULL
pauldzy Sep 25, 2025
e08d768
add more tests and fix edge case in raindrop indexing
pauldzy Sep 26, 2025
b84d220
rewrite randompoint to use ST_GeneratePoints for reliability and speed
pauldzy Sep 26, 2025
74f7f8a
adjust randomppnav to have more tolerance to find results in 32702
pauldzy Sep 26, 2025
41da706
bulk up test output for clarity
pauldzy Sep 26, 2025
025c067
performance tweaks for extended upstream with tribs
pauldzy Sep 26, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ Please review additional notes on [security](docs/security.md).
### Getting Help

* Open a GitHub issue.
* Email WATERS Support at WATERS_SUPPORT@epa.gov.
* Submit a [contact request](https://www.epa.gov/aboutepa/forms/contact-epas-office-water).

### Current Capabilities

Expand Down
2 changes: 1 addition & 1 deletion admin/helpers/dumpers/cipsrv_epageofab_h_beta_hr1.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ pg_dump.exe -h $env:POSTGRSQL_HOST -p $env:POSTGRSQL_PORT`
-t cipsrv_epageofab_h.alaskacatchment `
-t cipsrv_epageofab_h.grid0catchment `
-t cipsrv_epageofab_h.oceancatchment `
-f cipsrv_epageofab_h_beta_hr1_2.dmp
-f cipsrv_epageofab_h_beta_hr1_3.dmp
6 changes: 6 additions & 0 deletions admin/helpers/dumpers/cipsrv_nhdplus_h_beta.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pg_dump.exe -h $env:POSTGRSQL_HOST -p $env:POSTGRSQL_PORT`
-d $env:POSTGRSQL_DB -U cipsrv -Fc -v `
-t cipsrv_nhdplus_h.networknhdflowline `
-t cipsrv_nhdplus_h.nhdplusflow `
-t cipsrv_nhdplus_h.nonnetworknhdflowline `
-f cipsrv_nhdplus_h_beta_1.dmp
87 changes: 86 additions & 1 deletion admin/helpers/dumpers/cipsrv_nhdplus_h_betab.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,94 @@ pg_dump.exe -h $env:POSTGRSQL_HOST -p $env:POSTGRSQL_PORT`
-t cipsrv_nhdplus_h.nhdpluscatchmenttpp `
-t cipsrv_nhdplus_h.nhdpluscatchmentvpp `
-t cipsrv_nhdplus_h.nhdpluscatchmentxwalk `
-t cipsrv_nhdplus_h.nhdpluscumdivnlcd2011 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipma `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm01 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm02 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm03 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm04 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm05 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm06 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm07 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm08 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm09 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm10 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm11 `
-t cipsrv_nhdplus_h.nhdpluscumdivprecipmm12 `
-t cipsrv_nhdplus_h.nhdpluscumdivroma `
-t cipsrv_nhdplus_h.nhdpluscumdivtempma `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm01 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm02 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm03 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm04 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm05 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm06 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm07 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm08 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm09 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm10 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm11 `
-t cipsrv_nhdplus_h.nhdpluscumdivtempmm12 `
-t cipsrv_nhdplus_h.nhdpluscumtotnlcd2011 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipma `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm01 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm02 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm03 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm04 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm05 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm06 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm07 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm08 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm09 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm10 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm11 `
-t cipsrv_nhdplus_h.nhdpluscumtotprecipmm12 `
-t cipsrv_nhdplus_h.nhdpluscumtotroma `
-t cipsrv_nhdplus_h.nhdpluscumtottempma `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm01 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm02 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm03 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm04 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm05 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm06 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm07 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm08 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm09 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm10 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm11 `
-t cipsrv_nhdplus_h.nhdpluscumtottempmm12 `
-t cipsrv_nhdplus_h.nhdplusgage `
-t cipsrv_nhdplus_h.nhdplusincrlat `
-t cipsrv_nhdplus_h.nhdplusincrnlcd2011 `
-t cipsrv_nhdplus_h.nhdplusincrprecipma `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm01 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm02 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm03 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm04 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm05 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm06 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm07 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm08 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm09 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm10 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm11 `
-t cipsrv_nhdplus_h.nhdplusincrprecipmm12 `
-t cipsrv_nhdplus_h.nhdplusincrroma `
-t cipsrv_nhdplus_h.nhdplusincrtempma `
-t cipsrv_nhdplus_h.nhdplusincrtempmm01 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm02 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm03 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm04 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm05 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm06 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm07 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm08 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm09 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm10 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm11 `
-t cipsrv_nhdplus_h.nhdplusincrtempmm12 `
-t cipsrv_nhdplus_h.nhdplussink `
-t cipsrv_nhdplus_h.nhdpoint `
-t cipsrv_nhdplus_h.nhdwaterbody `
-t cipsrv_nhdplus_h.wbdhu12 `
-f cipsrv_nhdplus_h_betab_1.dmp
-f cipsrv_nhdplus_h_betab_2.dmp
17 changes: 4 additions & 13 deletions admin/helpers/dumpers/cipsrv_nhdplus_m_v21.txt
Original file line number Diff line number Diff line change
@@ -1,15 +1,6 @@
pg_dump.exe -h $env:POSTGRSQL_HOST -p $env:POSTGRSQL_PORT`
-d $env:POSTGRSQL_DB -U cipsrv -Fc -v `
-t cipsrv_nhdplus_m.nhdarea `
-t cipsrv_nhdplus_m.nhdline `
-t cipsrv_nhdplus_m.nhdpluscatchment `
-t cipsrv_nhdplus_m.nhdpluscatchmentfpp `
-t cipsrv_nhdplus_m.nhdpluscatchmenttpp `
-t cipsrv_nhdplus_m.nhdpluscatchmentvpp `
-t cipsrv_nhdplus_m.nhdpluscatchmentxwalk `
-t cipsrv_nhdplus_m.nhdplusgage `
-t cipsrv_nhdplus_m.nhdplussink `
-t cipsrv_nhdplus_m.nhdpoint `
-t cipsrv_nhdplus_m.nhdwaterbody `
-t cipsrv_nhdplus_m.wbdhu12 `
-f cipsrv_nhdplus_m_v21b_1.dmp
-t cipsrv_nhdplus_m.networknhdflowline `
-t cipsrv_nhdplus_m.nhdplusflow `
-t cipsrv_nhdplus_m.nonnetworknhdflowline `
-f cipsrv_nhdplus_m_v21_1.dmp
85 changes: 85 additions & 0 deletions admin/helpers/dumpers/cipsrv_nhdplus_m_v21b.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,92 @@ pg_dump.exe -h $env:POSTGRSQL_HOST -p $env:POSTGRSQL_PORT`
-t cipsrv_nhdplus_m.nhdpluscatchmenttpp `
-t cipsrv_nhdplus_m.nhdpluscatchmentvpp `
-t cipsrv_nhdplus_m.nhdpluscatchmentxwalk `
-t cipsrv_nhdplus_m.nhdpluscumdivnlcd2011 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipma `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm01 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm02 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm03 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm04 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm05 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm06 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm07 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm08 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm09 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm10 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm11 `
-t cipsrv_nhdplus_m.nhdpluscumdivprecipmm12 `
-t cipsrv_nhdplus_m.nhdpluscumdivroma `
-t cipsrv_nhdplus_m.nhdpluscumdivtempma `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm01 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm02 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm03 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm04 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm05 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm06 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm07 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm08 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm09 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm10 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm11 `
-t cipsrv_nhdplus_m.nhdpluscumdivtempmm12 `
-t cipsrv_nhdplus_m.nhdpluscumtotnlcd2011 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipma `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm01 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm02 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm03 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm04 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm05 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm06 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm07 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm08 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm09 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm10 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm11 `
-t cipsrv_nhdplus_m.nhdpluscumtotprecipmm12 `
-t cipsrv_nhdplus_m.nhdpluscumtotroma `
-t cipsrv_nhdplus_m.nhdpluscumtottempma `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm01 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm02 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm03 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm04 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm05 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm06 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm07 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm08 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm09 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm10 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm11 `
-t cipsrv_nhdplus_m.nhdpluscumtottempmm12 `
-t cipsrv_nhdplus_m.nhdplusgage `
-t cipsrv_nhdplus_m.nhdplusincrlat `
-t cipsrv_nhdplus_m.nhdplusincrnlcd2011 `
-t cipsrv_nhdplus_m.nhdplusincrprecipma `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm01 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm02 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm03 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm04 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm05 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm06 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm07 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm08 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm09 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm10 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm11 `
-t cipsrv_nhdplus_m.nhdplusincrprecipmm12 `
-t cipsrv_nhdplus_m.nhdplusincrroma `
-t cipsrv_nhdplus_m.nhdplusincrtempma `
-t cipsrv_nhdplus_m.nhdplusincrtempmm01 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm02 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm03 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm04 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm05 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm06 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm07 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm08 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm09 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm10 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm11 `
-t cipsrv_nhdplus_m.nhdplusincrtempmm12 `
-t cipsrv_nhdplus_m.nhdplussink `
-t cipsrv_nhdplus_m.nhdpoint `
-t cipsrv_nhdplus_m.nhdwaterbody `
Expand Down
10 changes: 10 additions & 0 deletions admin/helpers/dumpers/cipsrv_nhdplustopo_h.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
pg_dump.exe -h $env:POSTGRSQL_HOST -p $env:POSTGRSQL_PORT`
-d $env:POSTGRSQL_DB -U cipsrv -Fc -v `
-n cipsrv_nhdplustopo_h_catchment_fabric_3338 `
-n cipsrv_nhdplustopo_h_catchment_fabric_5070 `
-n cipsrv_nhdplustopo_h_catchment_fabric_26904 `
-n cipsrv_nhdplustopo_h_catchment_fabric_32161 `
-n cipsrv_nhdplustopo_h_catchment_fabric_32655 `
-n cipsrv_nhdplustopo_h_catchment_fabric_32702 `
-n cipsrv_nhdplustopo_h `
-f cipsrv_nhdplustopo_h_beta_1.dmp
10 changes: 10 additions & 0 deletions admin/helpers/dumpers/cipsrv_nhdplustopo_m.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
pg_dump.exe -h $env:POSTGRSQL_HOST -p $env:POSTGRSQL_PORT`
-d $env:POSTGRSQL_DB -U cipsrv -Fc -v `
-n cipsrv_nhdplustopo_m_catchment_fabric_3338 `
-n cipsrv_nhdplustopo_m_catchment_fabric_5070 `
-n cipsrv_nhdplustopo_m_catchment_fabric_26904 `
-n cipsrv_nhdplustopo_m_catchment_fabric_32161 `
-n cipsrv_nhdplustopo_m_catchment_fabric_32655 `
-n cipsrv_nhdplustopo_m_catchment_fabric_32702 `
-n cipsrv_nhdplustopo_m `
-f cipsrv_nhdplustopo_m_v21_1.dmp
7 changes: 7 additions & 0 deletions admin/helpers/dumpers/cipsrv_support.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ pg_dump.exe -h $env:POSTGRSQL_HOST -p $env:POSTGRSQL_PORT`
-t cipsrv_support.ecoregions_iii `
-t cipsrv_support.ecoregions_iv `
-t cipsrv_support.epa_regions_2023 `
-t cipsrv_support.outerwaters `
-t cipsrv_support.outerwaters_3338 `
-t cipsrv_support.outerwaters_5070 `
-t cipsrv_support.outerwaters_26904 `
-t cipsrv_support.outerwaters_32161 `
-t cipsrv_support.outerwaters_32655 `
-t cipsrv_support.outerwaters_32702 `
-t cipsrv_support.tiger_aiannha `
-t cipsrv_support.tiger_aiannha_3338 `
-t cipsrv_support.tiger_aiannha_5070 `
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ pg_dump.exe -h $env:POSTGRSQL_HOST -p $env:POSTGRSQL_PORT`
-t cipsrv_wbd.wbd_hu12_f3 `
-t cipsrv_wbd.wbd_hu12sp_f3 `
-t cipsrv_wbd.wbd_hu12_nphr `
-t cipsrv_wbd.wbd_hu12sp_np21 `
-t cipsrv_wbd.wbd_hu12sp_nphr `
-t cipsrv_wbd.wbd_hu12_np21 `
-t cipsrv_wbd.wbd_hu12sp_np21 `
-f cipsrv_wbd_1.dmp
90 changes: 90 additions & 0 deletions admin/helpers/loaders/nhdplus_usgs_r1.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
from osgeo import ogr;
import os,sys,psycopg2,requests,zipfile,shutil,subprocess,shlex;
from requests.adapters import HTTPAdapter;
from urllib3.util import Retry;

ogr.UseExceptions()

dbse = os.environ['POSTGRESQL_DB'];
host = os.environ['POSTGRESQL_HOST'];
port = os.environ['POSTGRESQL_PORT'];
user = 'cipsrv';
pasw = os.environ['POSTGRESQL_CIP_PASS'];

cs = "dbname=%s user=%s password=%s host=%s port=%s" % (
dbse
,user
,pasw
,host
,port
);

try:
conn = psycopg2.connect(cs);
except:
raise Exception("database connection error");

print("database is ready");

with open('nhdplus_usgs_r1.txt', 'r') as file:
for line in file:
if line[0] != '#':
(vpuid,url) = line.strip().split(',');

if os.path.exists('r1_vpu' + vpuid + '.zip'):
os.remove('r1_vpu' + vpuid + '.zip');

if os.path.exists('r1_vpu' + vpuid):
shutil.rmtree('r1_vpu' + vpuid);

r = requests.get(url);

try:
retry = Retry(
total=5,
backoff_factor=2,
status_forcelist=[429, 500, 502, 503, 504],
);

adapter = HTTPAdapter(max_retries=retry);

session = requests.Session();
session.mount('https://', adapter);
r = session.get(url, timeout=180);

except Exception as e:
print(e)


with open('r1_vpu' + vpuid + '.zip',"wb") as f:
f.write(r.content);

with zipfile.ZipFile('r1_vpu' + vpuid + '.zip','r') as zip_ref:
zip_ref.extractall('r1_vpu' + vpuid);

gdb = None;
for item in os.listdir('r1_vpu' + vpuid):

if '.gdb' in item:
gdb = item;

ogrgdb = ogr.Open(os.path.join('r1_vpu' + vpuid,gdb));
for item in ogrgdb:
lyr = item.GetName();

if 'NHDPlusIncr' in lyr:

cmd = 'ogr2ogr -f "PostgreSQL" PG:"' + cs + '" r1_vpu' + vpuid + os.sep + gdb + ' ' + lyr + ' '\
+ '-nln ' + 'cipsrv_upload.r1_vpu' + vpuid + '_' + lyr.lower();
cmd = shlex.split(cmd);

output, error = subprocess.Popen(
cmd
,shell = False
,universal_newlines = True
,stdout = subprocess.PIPE
,stderr = subprocess.PIPE
).communicate();

print('r1_vpu' + vpuid);

Loading