From 1400619dcd6ba27944ee7e6fb6b983ac6fcea636 Mon Sep 17 00:00:00 2001 From: cmbTea <39584949+cmbTea@users.noreply.github.com> Date: Wed, 29 Jun 2022 16:49:39 +0200 Subject: [PATCH 1/2] fixin 252 and 254 --- requirements.txt | 1 + src/CovidCases.py | 12 +++ src/CovidFoliumMapGenerator.py | 9 ++- src/rest/app.py | 138 +++++++++++++++++++++++++++------ 4 files changed, 133 insertions(+), 27 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2ab54ba..603d6e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,4 @@ Pillow>=8.1.1 pycountry==22.3.5 folium==0.12.1.post1 geopandas==0.10.2 +aiofiles==0.8.0 diff --git a/src/CovidCases.py b/src/CovidCases.py index d3046c2..cb98d45 100644 --- a/src/CovidCases.py +++ b/src/CovidCases.py @@ -106,8 +106,10 @@ def __init__(self, df, filenameCache = '', cacheLevel = 0): # build a cache if wanted and keep it if (filenameCache != '' and cacheLevel > 0): self.__df = self.__build_cache(df, filenameCache, cacheLevel) + self.__cacheFilename = filenameCache else: self.__df = df + self.__cacheFilename = '' @staticmethod @@ -227,6 +229,16 @@ def __build_cache(self, df, filenameCache, cacheLevel = 0): print('building cache...done: ' + str(end - start) + 's') return dfCache + def get_cache_filename(self): + """ returns the name of the cache file after it has been build. the constructor had to been invoked so + that cached was generated + + Returns: + str: the name of the cache file after it has been build. When it has not been build it returns + an empty string + """ + return self.__cacheFilename + def __add_additional_attributes(self, dfSingleCountry): """Adds additional attributes to a dataframe of a single country. diff --git a/src/CovidFoliumMapGenerator.py b/src/CovidFoliumMapGenerator.py index f522a13..45bfd01 100644 --- a/src/CovidFoliumMapGenerator.py +++ b/src/CovidFoliumMapGenerator.py @@ -25,8 +25,13 @@ def main(): outputDir = os.path.join(absDirectory, './data/') print('Running on local jupyter server. Using ' + outputDir + ' as the data directory') except: - # we are running not in jupyter - outputDir = '../data' + # we are running not in jupyter + try: + # try to read the environment variable + outputDir = os.environ['COVID_DATA'] + except: + # fallback to the default + outputDir = '../data' print('Running locally. Using ' + outputDir + ' as the data directory') # print the start time diff --git a/src/rest/app.py b/src/rest/app.py index 4d065b1..ec1c08c 100644 --- a/src/rest/app.py +++ b/src/rest/app.py @@ -17,9 +17,23 @@ import requests from datetime import date from starlette.responses import StreamingResponse +from starlette.responses import FileResponse from fastapi import FastAPI, HTTPException from enum import Enum +class Maps(Enum): + """ Enumeration of available maps + """ + MapAfrica = 'MapAfrica' + MapAmerica = 'MapAmerica' + MapAsia = 'MapAsia' + MapEurope = 'MapEurope' + MapOceania = 'MapOceania' + MapWorld = 'MapWorld' + MapDEageAndGenderCounty = 'MapDEageAndGenderCounty' + MapDEageAndGenderState = 'MapDEageAndGenderState' + MapDEcounty = 'MapDEcounty' + MapDEState = 'MapDEstate' class Attributes(Enum): """ @@ -79,18 +93,35 @@ class DataSource(Enum): OWID = 'OWID' # Our World in data class Rest_API: + """ A class for the REST-API using FastAPI + ATTENTION + The REST-API uses a directory defined by the COVID_DATA environment variable as a directory to store data. + If you run the REST-API locally using "uvicorn app:app" you should ensure defining the variable on your + local machine: + MacOS + via .bash\_profile or using .zshrc (if you are using zsh), both files exist in your home folder to add + a line such as this: "export COVID_DATA=$HOME/ + Linux + via "sudo touch /etc/profile.d/covid-data.sh" and "sudo nano /etc/profile.d/covid-data.sh" to add a line + such as this: "export COVID_DATA=$HOME/ + If you use the REST-API in a Docker container then you have to declare the environment variable in your + "docker-compose.yml" file: + ... + environment: + COVID_DATA: "your_directory" + """ def generate_plot(self, geo_ids, wanted_attrib, data_source, log=False, last_n=-1, since_n=-1, bar=False): - """ - Generates a plot for given GeoIds and returns it in form of a byteIO stream - Parameters: - geo_ids: [String] -> countries that should be plotted - wanted_attrib: String -> the field you want to plot, e.g. Cases - data_source: DataSource -> Source of the data defined by the enum. Note! If vaccination data is selected and another + """ Generates a plot for given GeoIds and returns it in form of a byteIO stream + + Args: + geo_ids (String): countries that should be plotted + wanted_attrib (String): the field you want to plot, e.g. Cases + data_source (DataSource): Source of the data defined by the enum. Note! If vaccination data is selected and another source than OWID is selected this will implicitly switch! - log: bool -> should the plot be logarithmic - last_n: int -> plot the last n days, if not further specified all available data is plotted - since_n: int -> plot since the nth case, if not further specified all available data is plotted + log (bool, optional): should the plot be logarithmic + last_n (int, optional): plot the last n days, if not further specified all available data is plotted + since_n (int, optional): plot since the nth case, if not further specified all available data is plotted """ # vaccination data is only available with OWID if data_source != DataSource.OWID: @@ -107,13 +138,23 @@ def generate_plot(self, geo_ids, wanted_attrib, data_source, log=False, last_n=- data_source = DataSource.OWID case (_, _): pass """ - # load the cases - if data_source == DataSource.OWID: - csv_file = CovidCasesOWID.download_CSV_file() - self.covid_cases = CovidCasesOWID(csv_file) - if data_source == DataSource.WHO: - csv_file = CovidCasesWHO.download_CSV_file() - self.covid_cases = CovidCasesWHO(csv_file) + # load the data source + try: + prefix = os.environ['COVID_DATA'] + if data_source == DataSource.OWID: + csv_file = CovidCasesOWID.download_CSV_file(prefix) + self.covid_cases = CovidCasesOWID(csv_file) + if data_source == DataSource.WHO: + csv_file = CovidCasesWHO.download_CSV_file(prefix) + self.covid_cases = CovidCasesWHO(csv_file) + except: + print ('missing COVID_DATA environment variable') + if data_source == DataSource.OWID: + csv_file = CovidCasesOWID.download_CSV_file() + self.covid_cases = CovidCasesOWID(csv_file) + if data_source == DataSource.WHO: + csv_file = CovidCasesWHO.download_CSV_file() + self.covid_cases = CovidCasesWHO(csv_file) """ Alternatively in latest Python version match data_source: @@ -127,8 +168,7 @@ def generate_plot(self, geo_ids, wanted_attrib, data_source, log=False, last_n=- # try to collect the data for given geoIds, if a wrong geoId is passed, the operation will abort with a 400 # bad request error try: - df = self.covid_cases.get_data_by_geoid_list( - geo_ids, lastNdays=last_n, sinceNcases=since_n) + df = self.covid_cases.get_data_by_geoid_list(geo_ids, lastNdays=last_n, sinceNcases=since_n) except IndexError: raise HTTPException( status_code=400, detail="Couldn't load data") @@ -187,24 +227,26 @@ def generate_plot(self, geo_ids, wanted_attrib, data_source, log=False, last_n=- byte_io = io.BytesIO() plt.savefig(byte_io, dpi=fig.dpi) byte_io.seek(0) - return byte_io def setup_routes(self, app: FastAPI): - """ - Setup of the route. The url has to be in the form: - /api/data// + """ Setup of the route. The url has to be in the form: + /api/data// ?log=(True or False)[&lastN=X if you want to plot lastNdays][&sinceN=X if you want to plot since the Nth case] + + Args: + app (FastAPI): the API """ # setting up routes and implement methods @app.get('/api/data/{countries}/{wanted_attrib}') def get_data(countries: str, wanted_attrib: Attributes, dataSource: Optional[DataSource] = DataSource.WHO, sinceN: Optional[int] = None, lastN: Optional[int] = None, log: Optional[bool] = None, bar: Optional[bool] = None): - """ - Returns a png image of the plotted Attribute (see Attributes) for a list of Countries(comma seperated). The URL needs to be in the following form: + """ Returns a png image of the plotted Attribute (see Attributes) for a list of Countries(comma seperated). + The URL needs to be in the following form: **/api/data// ?log=(True or False)[&lastN=X if you want to plot lastNdays][&sinceN=X if you want to plot since the Nth case]** - SinceN and lastN plots the data starting from the given case or just the lastN days. Log is a boolean value that converts the y-scale to the logarithmic unit. + SinceN and lastN plots the data starting from the given case or just the lastN days. Log is a boolean value that + converts the y-scale to the logarithmic unit. """ # read the geoIds and plot the file countries = countries.upper() @@ -218,8 +260,54 @@ def get_data(countries: str, wanted_attrib: Attributes, dataSource: Optional[Dat # return the created stream as png image return StreamingResponse(file, media_type="image/png") + @app.get('/api/maps/{wanted_map}') + def get_map(wanted_map: Maps): + """ Returns a map in form of a interactive HTML document that can be shown in a browser + + Args: + wanted_map (Maps): Any of the support maps as a string + + Returns: + _type_: a HTML file + """ + # the prefix of the fileserver + try: + prefix = os.environ['COVID_DATA'] + except: + print ('missing COVID_DATA environment variable') + return 'Data directory not found' + # return the HTML file + return FileResponse(prefix + wanted_map.name + '.html') + + @app.get('/api/csv/') + def get_csv(): + """ Returns the cached CSV data until today + + Returns: + _type_: a CSV file + """ + # the prefix of the fileserver + try: + prefix = os.environ['COVID_DATA'] + except: + print ('missing COVID_DATA environment variable') + return 'Data directory not found' + # todays date + today = date.today() + # the prefix of the CSV file is Y-m-d + targetFilename = prefix + '/WHO-data-processed.csv' + if os.path.exists(targetFilename): + # return the file + return FileResponse(targetFilename) + else: + # return a hint + msg = 'Sorry, can not find ' + 'WHO-data-processed.csv' + '. Please try again later' + print (msg) + return msg # change matplotlib backend to agg (agg is not interactive and it can't be interactive) matplotlib.use('agg') +# create the REST API app = FastAPI() +# define the routes (functions) Rest_API().setup_routes(app) From 6ed420d5efeb79623bd4e1677131af64809dbb4e Mon Sep 17 00:00:00 2001 From: cmbTea <39584949+cmbTea@users.noreply.github.com> Date: Sat, 27 Aug 2022 18:41:09 +0200 Subject: [PATCH 2/2] Fix spelling error --- src/CovidCases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CovidCases.py b/src/CovidCases.py index cb98d45..2601a3f 100644 --- a/src/CovidCases.py +++ b/src/CovidCases.py @@ -230,8 +230,8 @@ def __build_cache(self, df, filenameCache, cacheLevel = 0): return dfCache def get_cache_filename(self): - """ returns the name of the cache file after it has been build. the constructor had to been invoked so - that cached was generated + """ returns the name of the cache file after it has been build. The constructor had to been invoked so + that the cache gets generated. Returns: str: the name of the cache file after it has been build. When it has not been build it returns