Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Conversation

@hillarymarler
Copy link
Collaborator

Updated so that "other" results remain when "clean = TRUE", also was able to make the code a little more concise.

updated so that "other" results remain when "clean = TRUE", also was able to make the code a little more concise
@cristinamullin
Copy link
Collaborator

cristinamullin commented Jun 13, 2025

It looks like this is still removing the others when clean = TRUE (example below). Could we add the OTHER category as a function input, and also the console note for that one "TADA_AnalysisDataFilter: Flagging other (noncategorized) results to exclude from assessments."? After thinking about it more, users may want to choose to keep or exclude other (noncategorized) results, but our default can be to keep them.

CODE TO REPRODUCE

query.params <- list(
  where = "assessmentunitidentifier IN ('CT6400-00-1-L5_01')",
  outFields = "*",
  f = "geojson"
)

url <- "https://gispub.epa.gov/arcgis/rest/services/OW/ATTAINS_Assessment/MapServer/2/query?"

poly.response <- httr::GET(url, query = query.params)

poly.geojson <- httr::content(poly.response, as = "text", encoding = "UTF-8")

poly.sf <- sf::st_read(poly.geojson, quiet = TRUE)

WQP_raw <- TADA_DataRetrieval(
  startDate = "null",
  endDate = "null",
  aoi_sf = poly.sf,
  countrycode = "null",
  countycode = "null",
  huc = "null",
  siteid = "null",
  siteType = "null",
  tribal_area_type = "null",
  tribe_name_parcel = "null",
  characteristicName = "null",
  characteristicType = "null",
  sampleMedia = "null",
  statecode = "null",
  organization = "null",
  project = "null",
  providers = "null",
  bBox = "null",
  maxrecs = 350000,
  ask = FALSE,
  applyautoclean = TRUE
)

rm(poly.response, poly.sf, query.params, poly.geojson, url)

WQP_flag <- TADA_AnalysisDataFilter(
  WQP_raw,
  clean = FALSE,
  surface_water = TRUE,
  ground_water = FALSE,
  sediment = FALSE
)

# Review unique flags
unique(WQP_flag$TADA.UseForAnalysis.Flag)

# Remove rows that are flagged as sediment or ground water, keep other and surface water (not working)
WQP_clean <- TADA_AnalysisDataFilter(
  WQP_flag,
  clean = TRUE,
  surface_water = TRUE,
  ground_water = FALSE,
  sediment = FALSE
)

CONSOLE

> query.params <- list(
+   where = "assessmentunitidentifier IN ('CT6400-00-1-L5_01')",
+   outFields = "*",
+   f = "geojson"
+ )
> 
> url <- "https://gispub.epa.gov/arcgis/rest/services/OW/ATTAINS_Assessment/MapServer/2/query?"
> 
> poly.response <- httr::GET(url, query = query.params)
> 
> poly.geojson <- httr::content(poly.response, as = "text", encoding = "UTF-8")
> 
> poly.sf <- sf::st_read(poly.geojson, quiet = TRUE)
> 
> WQP_raw <- TADA_DataRetrieval(
+   startDate = "null",
+   endDate = "null",
+   aoi_sf = poly.sf,
+   countrycode = "null",
+   countycode = "null",
+   huc = "null",
+   siteid = "null",
+   siteType = "null",
+   tribal_area_type = "null",
+   tribe_name_parcel = "null",
+   characteristicName = "null",
+   characteristicType = "null",
+   sampleMedia = "null",
+   statecode = "null",
+   organization = "null",
+   project = "null",
+   providers = "null",
+   bBox = "null",
+   maxrecs = 350000,
+   ask = FALSE,
+   applyautoclean = TRUE
+ )
Checking for available data. This may take a moment.
Transforming your data into a spatial object.
[1] "Downloading WQP query results. This may take some time depending upon the query size."
list()
[1] "Data successfully downloaded. Running TADA_AutoClean function."                        
[1] "TADA_Autoclean: creating TADA-specific columns."
[1] "TADA_Autoclean: harmonizing dissolved oxygen characterisic name to DISSOLVED OXYGEN SATURATION if unit is % or % SATURATN."
[1] "TADA_Autoclean: handling special characters and coverting TADA.ResultMeasureValue and TADA.DetectionQuantitationLimitMeasure.MeasureValue value fields to numeric."
[1] "TADA_Autoclean: converting TADA.LatitudeMeasure and TADA.LongitudeMeasure fields to numeric."
[1] "TADA_Autoclean: harmonizing synonymous unit names (m and meters) to m."
[1] "TADA_Autoclean: updating deprecated (i.e. retired) characteristic names."
[1] "No deprecated characteristic names found in dataset."
[1] "TADA_Autoclean: harmonizing result and depth units."
[1] "TADA_Autoclean: creating TADA.ComparableDataIdentifier field for use when generating visualizations and analyses."
[1] "NOTE: This version of the TADA package is designed to work with numeric data with media name: 'WATER'. TADA_AutoClean does not currently remove (filter) data with non-water media types. If desired, the user must make this specification on their own outside of package functions. Example: dplyr::filter(.data, TADA.ActivityMediaName == 'WATER')"
> rm(poly.response, poly.sf, query.params, poly.geojson, url)
> # Create table with count for each ActivityMediaName
> TADA_FieldValuesTable(
+   WQP_raw, 
+   field = "ActivityMediaName", 
+   characteristicName = "null"
+   )
  Value Count
1 Water   990
> WQP_flag <- TADA_AnalysisDataFilter(
+   WQP_raw,
+   clean = FALSE,
+   surface_water = TRUE,
+   ground_water = FALSE,
+   sediment = FALSE
+ )
[1] "TADA_AnalysisDataFilter: Identifying groundwater results."
[1] "TADA_AnalysisDataFilter: Flagging surface water results to include in assessments."
[1] "TADA_AnalysisDataFilter: Flagging groundwater results to exclude from assessments."
[1] "TADA_AnalysisDataFilter: Flagging sediment results to exclude from assessments."
[1] "TADA_AnalysisDataFilter: Returning all results with TADA.UseForAnalysis.Flag column indicating if result should be used for assessments."
> 
> # Review unique flags
> unique(WQP_flag$TADA.UseForAnalysis.Flag)
[1] "Yes - SURFACE WATER" "No - OTHER"         
> 
> # Remove rows that are flagged as sediment or ground water, keep other and surface water (not working)
> WQP_clean <- TADA_AnalysisDataFilter(
+   WQP_flag,
+   clean = TRUE,
+   surface_water = TRUE,
+   ground_water = FALSE,
+   sediment = FALSE
+ )
[1] "TADA_AnalysisDataFilter: Identifying groundwater results."
[1] "TADA_AnalysisDataFilter: Flagging surface water results to include in assessments."
[1] "TADA_AnalysisDataFilter: Flagging groundwater results to exclude from assessments."
[1] "TADA_AnalysisDataFilter: Flagging sediment results to exclude from assessments."
[1] "TADA_AnalysisDataFilter: Removing results flagged for exclusion from assessments."

@hillarymarler
Copy link
Collaborator Author

I updated the function to add an "other" argument, which the user can set to TRUE or FALSE (default is TRUE). I checked it with your example @cristinamullin and the default behavior is now to keep the "OTHER" samples, unless the user sets other = FALSE.

hillarymarler and others added 3 commits June 16, 2025 10:48
…e-if-tadauseforanalysisflag-no---na' of https://github.com/USEPA/EPATADA into 596-tada_analysisdatafilter-removes-rows-when-clean-true-if-tadauseforanalysisflag-no---na
@cristinamullin cristinamullin merged commit 64b7a7a into develop Jun 17, 2025
6 of 7 checks passed
@cristinamullin cristinamullin deleted the 596-tada_analysisdatafilter-removes-rows-when-clean-true-if-tadauseforanalysisflag-no---na branch June 17, 2025 15:26
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

TADA_AnalysisDataFilter removes rows when clean = TRUE if TADA.UseForAnalysis.Flag == "No - NA"

3 participants