Skip to content

Commit

Permalink
enh(get_eglv_*): progress
Browse files Browse the repository at this point in the history 
fixes #30
  • Loading branch information
@dimfalk
dimfalk committed Dec 27, 2024
1 parent bc0112e commit edd8a2e
Show file tree
Hide file tree
Showing 6 changed files with 58 additions and 37 deletions.
4 changes: 3 additions & 1 deletion DESCRIPTION
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Package: NRWgauges
Title: Web-scraping gauge (meta-)data from various providers in NRW, Germany
Version: 0.3.3
Version: 0.3.4
Date: 2024-12-27
Authors@R:
person("Dimitri", "Falk", , "[email protected]", role = c("aut", "cre"))
Expand All @@ -16,8 +16,10 @@ Imports:
dplyr,
httr,
jsonlite,
progress,
rvest,
sf,
stringi,
stringr,
tibble,
xts,
Expand Down
15 changes: 13 additions & 2 deletions R/get_eglv_measurements.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ get_eglv_measurements <- function(x = NULL,

# ----------------------------------------------------------------------------

# init object to be returned
xtslist <- list()

base_url <- "https://pegel.eglv.de/measurements/"

# set relevant parameter
Expand All @@ -48,12 +51,18 @@ get_eglv_measurements <- function(x = NULL,
par <- "Wasserstand"
}

# iterate over individual stations
# iterate over individual stations, initialize progress bar
ids <- x[["id"]]

n <- length(ids)

xtslist <- list()
pb <- progress::progress_bar$new(format = "(:spin) [:bar] :percent || Iteration: :current/:total || Elapsed time: :elapsedfull",
total = n,
complete = "#",
incomplete = "-",
current = ">",
clear = FALSE,
width = 100)

for (i in 1:n) {

Expand Down Expand Up @@ -106,6 +115,8 @@ get_eglv_measurements <- function(x = NULL,
xtslist[[i]] <- meas

Sys.sleep(0.5)

pb$tick()
}

names(xtslist) <- ids
Expand Down
19 changes: 15 additions & 4 deletions R/get_eglv_meta.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ get_eglv_meta <- function(x = NULL) {

# ----------------------------------------------------------------------------

base_url <- "https://pegel.eglv.de/Stammdaten/"

# init object to be returned
meta <- data.frame("id" = NA,
"name" = NA,
"operator" = NA,
Expand All @@ -41,11 +40,21 @@ get_eglv_meta <- function(x = NULL) {
"catchment_area" = NA,
"level_zero" = NA)

# iterate over individual stations
base_url <- "https://pegel.eglv.de/Stammdaten/"

# iterate over individual stations, initialize progress bar
ids <- x[["id"]]

n <- length(ids)

pb <- progress::progress_bar$new(format = "(:spin) [:bar] :percent || Iteration: :current/:total || Elapsed time: :elapsedfull",
total = n,
complete = "#",
incomplete = "-",
current = ">",
clear = FALSE,
width = 100)

for (i in 1:n) {

url <- paste0(base_url, ids[i], "/")
Expand All @@ -54,7 +63,7 @@ get_eglv_meta <- function(x = NULL) {
r_raw <- httr::GET(url)

# parse response: html to text
a <- rvest::read_html(r_raw) |>
a <- rvest::read_html(r_raw) |>
rvest::html_elements("li") |>
rvest::html_text()

Expand Down Expand Up @@ -84,6 +93,8 @@ get_eglv_meta <- function(x = NULL) {
}

Sys.sleep(0.5)

pb$tick()
}

meta_all
Expand Down
2 changes: 1 addition & 1 deletion README.Rmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ get_eglv_meta(gauge)

### Get available measurements for selected gauges

```{r}
```{r, warning = FALSE}
# fetch water level measurements
meas <- get_eglv_measurements(gauge)[[1]]
meas
Expand Down
55 changes: 26 additions & 29 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ and load the package via

``` r
library(NRWgauges)
#> 0.3.0
#> 0.3.4
```

## Basic example: EGLV
Expand Down Expand Up @@ -89,11 +89,11 @@ gauge

``` r
get_eglv_meta(gauge)
#> # A tibble: 1 × 9
#> id name waterbody municipality X Y river_km catchment_area
#> <chr> <chr> <chr> <chr> <dbl> <dbl> <dbl> <dbl>
#> 1 10119 Adenaueral… Emscher Gelsenkirch… 3.67e5 5.71e6 36.4 481.
#> # ℹ 1 more variable: level_zero <dbl>
#> # A tibble: 1 × 10
#> id name operator waterbody municipality X Y river_km
#> <chr> <chr> <chr> <chr> <chr> <dbl> <dbl> <dbl>
#> 1 10119 Adenauerallee EG Emscher Gelsenkirchen 367060. 5712147. 36.4
#> # ℹ 2 more variables: catchment_area <dbl>, level_zero <dbl>
```

### Get available measurements for selected gauges
Expand All @@ -102,31 +102,28 @@ get_eglv_meta(gauge)
# fetch water level measurements
meas <- get_eglv_measurements(gauge)[[1]]
meas
#> Warning: object timezone ('etc/GMT-1') is different from system timezone ('')
#> NOTE: set 'options(xts_check_TZ = FALSE)' to disable this warning
#> This note is displayed once per session
#> Wasserstand
#> 2024-10-29 12:20:00 75
#> 2024-10-29 12:25:00 75
#> 2024-10-29 12:30:00 75
#> 2024-10-29 12:35:00 75
#> 2024-10-29 12:40:00 75
#> 2024-10-29 12:45:00 75
#> 2024-10-29 12:50:00 75
#> 2024-10-29 12:55:00 75
#> 2024-10-29 13:00:00 75
#> 2024-10-29 13:05:00 75
#> 2024-10-29 20:25:00 86
#> 2024-10-29 20:30:00 86
#> 2024-10-29 20:35:00 86
#> 2024-10-29 20:40:00 86
#> 2024-10-29 20:45:00 86
#> 2024-10-29 20:50:00 86
#> 2024-10-29 20:55:00 86
#> 2024-10-29 21:00:00 86
#> 2024-10-29 21:05:00 86
#> 2024-10-29 21:10:00 86
#> ...
#> 2024-12-27 11:25:00 85
#> 2024-12-27 11:30:00 85
#> 2024-12-27 11:35:00 84
#> 2024-12-27 11:40:00 84
#> 2024-12-27 11:45:00 84
#> 2024-12-27 11:50:00 84
#> 2024-12-27 11:55:00 84
#> 2024-12-27 12:00:00 84
#> 2024-12-27 12:05:00 84
#> 2024-12-27 12:10:00 84
#> 2024-12-27 19:30:00 83
#> 2024-12-27 19:35:00 83
#> 2024-12-27 19:40:00 83
#> 2024-12-27 19:45:00 83
#> 2024-12-27 19:50:00 83
#> 2024-12-27 19:55:00 83
#> 2024-12-27 20:00:00 83
#> 2024-12-27 20:05:00 83
#> 2024-12-27 20:10:00 83
#> 2024-12-27 20:15:00 83

class(meas)
#> [1] "xts" "zoo"
Expand Down
Binary file modified man/figures/README-unnamed-chunk-5-1.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit edd8a2e

Please sign in to comment.