add chunks auto to open_mfdataset to avoid memory problems

EcoExtreML · Feb 2, 2024 · d3ffc6b · d3ffc6b
1 parent 177b897
commit d3ffc6b
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 4 deletions.
diff --git a/PyStemmusScope/global_data/cams_co2.py b/PyStemmusScope/global_data/cams_co2.py
@@ -60,7 +60,7 @@ def extract_cams_data(
     Returns:
         DataArray containing the CO2 concentration.
     """
-    ds = xr.open_mfdataset(files_cams)
+    ds = xr.open_mfdataset(files_cams, chunks='auto')
 
     check_cams_dataset(cams_data=ds, latlon=latlon, time_range=time_range)
 

diff --git a/PyStemmusScope/global_data/cci_landcover.py b/PyStemmusScope/global_data/cci_landcover.py
@@ -62,7 +62,7 @@ def extract_landcover_data(
     Returns:
         Dictionary containing IGBP and LCCS land cover classes.
     """
-    cci_dataset = xr.open_mfdataset(files_cci)
+    cci_dataset = xr.open_mfdataset(files_cci, chunks='auto')
 
     check_cci_dataset(cci_dataset, latlon, time_range)  # Assert spatial/temporal bounds
 

diff --git a/PyStemmusScope/global_data/copernicus_lai.py b/PyStemmusScope/global_data/copernicus_lai.py
@@ -60,7 +60,7 @@ def extract_lai_data(
     Returns:
         DataArray containing the LAI of the specified site for the given time range.
     """
-    ds = xr.open_mfdataset(files_lai)
+    ds = xr.open_mfdataset(files_lai, chunks='auto')
 
     check_lai_dataset(ds, latlon, time_range)
 

diff --git a/PyStemmusScope/global_data/era5.py b/PyStemmusScope/global_data/era5.py
@@ -124,7 +124,8 @@ def get_era5_dataset(
     """
     tol = RESOLUTION_ERA5 if name == "ERA5" else RESOLUTION_ERA5LAND
 
-    ds = xr.open_mfdataset(files)
+    ds = xr.open_mfdataset(files, chunks='auto')
+
     check_era5_dataset(ds, name, latlon, time_range)
 
     try: