diff --git a/DESCRIPTION b/DESCRIPTION index ae097ff..1b1c10b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: restatapi Type: Package Title: Search and Retrieve Data from Eurostat Database -Date: 2023-10-23 -Version: 0.22.2 +Date: 2023-12-29 +Version: 0.22.3 Encoding: UTF-8 Authors@R: c(person("Mátyás", "Mészáros", email = "matyas.meszaros@ec.europa.eu", role = c("aut", "cre")), person("Sebastian", "Weinand", role = "ctb")) diff --git a/NEWS.md b/NEWS.md index 004e4d6..7ba16f7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# restatapi 0.22.3 + +- additional check in the get_eurostat_dsd() and get_eurostat_codelist() for failing writing data to disk because of failing network connection + # restatapi 0.22.2 - correcting the get_compressed_sdmx() function not closing connections diff --git a/R/extract_data.R b/R/extract_data.R index c2a915b..46d0730 100644 --- a/R/extract_data.R +++ b/R/extract_data.R @@ -32,7 +32,7 @@ #' extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE,check_toc=FALSE){ - if (getOption("restatapi_verbose",FALSE)) {message("extract_data - API version:",get("rav",envir=restatapi::.restatapi_env))} + # if (getOption("restatapi_verbose",FALSE)) {message("extract_data - API version:",get("rav",envir=restatapi::.restatapi_env))} rav<-get("rav",envir=restatapi::.restatapi_env) prefix<-switch(rav,"1"="generic:","2"="g:") if(bulk){ diff --git a/R/get_eurostat_codelist.R b/R/get_eurostat_codelist.R index a0a83ef..e9dec8d 100644 --- a/R/get_eurostat_codelist.R +++ b/R/get_eurostat_codelist.R @@ -72,13 +72,15 @@ get_eurostat_codelist <- function(id, tryCatch({utils::download.file(cls_endpoint,temp,dmethod)}, error = function(e) { message("get_eurostat_codelist - Error by the download of the codelist file:",'\n',paste(unlist(e),collapse="\n")) + cls_xml<-NULL }, warning = function(w) { message("get_eurostat_codelist - Warning by the download of the codelist file:",'\n',paste(unlist(w),collapse="\n")) tbc<-FALSE cls_xml<-NULL }) - if (file.size(temp)!=0 & tbc) { + if (is.na(file.size(temp))) {temp_size=0} else {temp_size=file.size(temp)} + if (temp_size!=0 & tbc) { message("Trying to extract the codelist from: ",temp) tryCatch({cls_xml<-xml2::read_xml(temp)}, error = function(e) { @@ -87,24 +89,27 @@ get_eurostat_codelist <- function(id, }, warning = function(w) { message("get_eurostat_codelist - There is warning by the extraction of the XML from the downloaded codelist file:",'\n',paste(unlist(w),collapse="\n")) + cls_xml<-NULL }) } else { cls_xml<-NULL } } else { tryCatch({utils::download.file(cls_endpoint,temp,dmethod,quiet=TRUE)}, - error = function(e) {message("There is an error by the download of the codelist file. Run the same command with verbose=TRUE option to get more info on the issue.") + error = function(e) {message("get_eurostat_codelist - There is an error by the download of the codelist file. Run the same command with verbose=TRUE option to get more info on the issue.") + cls_xml<-NULL }, - warning = function(w) {message("There is a warning by the download of the codelist file. Run the same command with verbose=TRUE option to get more info on the issue.") + warning = function(w) {message("get_eurostat_codelist - There is a warning by the download of the codelist file. Run the same command with verbose=TRUE option to get more info on the issue.") tbc<-FALSE cls_xml<-NULL }) - if (file.size(temp)!=0 & tbc) { + if (is.na(file.size(temp))) {temp_size=0} else {temp_size=file.size(temp)} + if (temp_size!=0 & tbc) { tryCatch({cls_xml<-xml2::read_xml(temp)}, - error = function(e) {message("There is an error by the reading of the downloaded codelist file. Run the same command with verbose=TRUE option to get more info on the issue.") + error = function(e) {message("get_eurostat_codelist - There is an error by the reading of the downloaded codelist file. Run the same command with verbose=TRUE option to get more info on the issue.") cls_xml<-NULL }, - warning = function(w) {message("There is a warning by the reading of the downloaded codelist file. Run the same command with verbose=TRUE option to get more info on the issue.") + warning = function(w) {message("get_eurostat_codelist - There is a warning by the reading of the downloaded codelist file. Run the same command with verbose=TRUE option to get more info on the issue.") cls_xml<-NULL }) } else { @@ -131,9 +136,9 @@ get_eurostat_codelist <- function(id, } } else { # cls<-NULL - # if (verbose) { - # message("get_eurostat_codelist - The cls_xml is NULL. Please check in a browser the url below. If it provides valid reponse you can try again to download the codelist.\n ",cls_endpoint) - # } + if (verbose) { + message("get_eurostat_codelist - The cls_xml is NULL. Please check in a browser the url below. If it provides valid response you can try again to download the codelist.\n ",cls_endpoint) + } } if (!is.null(cls)){ data.table::as.data.table(cls,stringsAsFactors=FALSE) diff --git a/R/get_eurostat_dsd.R b/R/get_eurostat_dsd.R index ea4fd35..96fea93 100644 --- a/R/get_eurostat_dsd.R +++ b/R/get_eurostat_dsd.R @@ -83,13 +83,16 @@ get_eurostat_dsd <- function(id, tryCatch({utils::download.file(dsd_endpoint,temp,dmethod)}, error = function(e) { message("get_eurostat_dsd - Error by the download of the DSD file:",'\n',paste(unlist(e),collapse="\n")) + dsd_xml<-NULL }, warning = function(w) { message("get_eurostat_dsd - Warning by the download of the DSD file:",'\n',paste(unlist(w),collapse="\n")) tbc<-FALSE dsd_xml<-NULL }) - if (file.size(temp)!=0 & tbc) { + message("\nget_eurostat_dsd - class(temp): ",class(temp)," - is.na(temp): ",is.na(temp)," - file.size(temp): ", file.size(temp)," - temp: ",temp) + if (is.na(file.size(temp))) {temp_size=0} else {temp_size=file.size(temp)} + if (temp_size!=0 & tbc) { message("Trying to extract the DSD from: ",temp) tryCatch({dsd_xml<-xml2::read_xml(temp)}, error = function(e) { @@ -98,24 +101,27 @@ get_eurostat_dsd <- function(id, }, warning = function(w) { message("get_eurostat_dsd - There is warning by the extraction of the XML from the downloaded DSD file:",'\n',paste(unlist(w),collapse="\n")) + dsd_xml<-NULL }) } else { dsd_xml<-NULL } } else { tryCatch({utils::download.file(dsd_endpoint,temp,dmethod,quiet=TRUE)}, - error = function(e) {message("There is an error by the download of the DSD file. Run the same command with verbose=TRUE option to get more info on the issue.") + error = function(e) {message("get_eurostat_dsd - There is an error by the download of the DSD file. Run the same command with verbose=TRUE option to get more info on the issue.") + dsd_xml<-NULL }, - warning = function(w) {message("There is a warning by the download of the DSD file. Run the same command with verbose=TRUE option to get more info on the issue.") + warning = function(w) {message("get_eurostat_dsd - There is a warning by the download of the DSD file. Run the same command with verbose=TRUE option to get more info on the issue.") tbc<-FALSE dsd_xml<-NULL }) - if (file.size(temp)!=0 & tbc) { + if (is.na(file.size(temp))) {temp_size=0} else {temp_size=file.size(temp)} + if (temp_size!=0 & tbc) { tryCatch({dsd_xml<-xml2::read_xml(temp)}, - error = function(e) {message("There is an error by the reading of the downloaded DSD file. Run the same command with verbose=TRUE option to get more info on the issue.") + error = function(e) {message("get_eurostat_dsd - There is an error by the reading of the downloaded DSD file. Run the same command with verbose=TRUE option to get more info on the issue.") dsd_xml<-NULL }, - warning = function(w) {message("There is an error by the reading of the downloaded DSD file. Run the same command with verbose=TRUE option to get more info on the issue.") + warning = function(w) {message("get_eurostat_dsd - There is an error by the reading of the downloaded DSD file. Run the same command with verbose=TRUE option to get more info on the issue.") dsd_xml<-NULL }) } else { @@ -162,14 +168,15 @@ get_eurostat_dsd <- function(id, tryCatch({utils::download.file(cc_endpoint,temp,dmethod)}, error = function(e) { message("get_eurostat_dsd - Error by the download of the CC file:",'\n',paste(unlist(e),collapse="\n")) - + cc_xml<-NULL }, warning = function(w) { message("get_eurostat_dsd - Warning by the download of the CC file:",'\n',paste(unlist(w),collapse="\n")) tbc<-FALSE cc_xml<-NULL }) - if (file.size(temp)!=0 & tbc) { + if (is.na(file.size(temp))) {temp_size=0} else {temp_size=file.size(temp)} + if (temp_size!=0 & tbc) { message("get_eurostat_dsd - Trying to extract the CC from: ",temp) tryCatch({cc_xml<-xml2::read_xml(temp)}, error = function(e) { @@ -178,6 +185,7 @@ get_eurostat_dsd <- function(id, }, warning = function(w) { message("get_eurostat_dsd - There is warning by the extraction of the XML from the downloaded CC file:",'\n',paste(unlist(w),collapse="\n")) + cc_xml<-NULL }) } else { cc_xml<-NULL @@ -192,7 +200,8 @@ get_eurostat_dsd <- function(id, tbc<-FALSE cc_xml<-NULL }) - if (file.size(temp)!=0 & tbc) { + if (is.na(file.size(temp))) {temp_size=0} else {temp_size=file.size(temp)} + if (temp_size!=0 & tbc) { tryCatch({cc_xml<-xml2::read_xml(temp)}, error = function(e) {message("There is an error by the reading of the downloaded CC file. Run the same command with verbose=TRUE option to get more info on the issue.") cc_xml<-NULL @@ -224,7 +233,7 @@ get_eurostat_dsd <- function(id, } else { # dsd<-NULL if (verbose) { - message("get_eurostat_dsd - The dsd_xml is NULL. Please check in a browser the url below. If it provides valid reponse you can try again to download the DSD.\n ",dsd_endpoint) + message("get_eurostat_dsd - The dsd_xml is NULL. Please check in a browser the url below. If it provides valid response you can try again to download the DSD.\n ",dsd_endpoint) } } }