From 51e6afd07f6073d2b16cd56a95a50bd84063b0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20Fern=C3=A1ndez?= Date: Wed, 23 Oct 2024 07:08:56 +0100 Subject: [PATCH] Install dash (#213) * Install dash * Force channel * Test moving to base * Move to base * Fix build * Unpin cdlib * Temporarily disable single-user * Merge all mamba installs * Make apt silent * Remove some generic stuff that may not be applicable to SBD * Try to fix installation issues * Bring back the image * Remove sobigdata image --- .github/workflows/d4science-images.yml | 2 +- d4science-base/Dockerfile | 31 ++++++++++++--- single-user-d4science/Dockerfile | 37 +++++------------- single-user-r-d4science/Dockerfile | 17 +------- single-user-sobigdata/Dockerfile | 54 -------------------------- 5 files changed, 38 insertions(+), 103 deletions(-) diff --git a/.github/workflows/d4science-images.yml b/.github/workflows/d4science-images.yml index a060fe21..d8e2149e 100644 --- a/.github/workflows/d4science-images.yml +++ b/.github/workflows/d4science-images.yml @@ -16,7 +16,7 @@ jobs: base: d4science-base/Dockerfile images: | "single-user-d4science/Dockerfile" - "single-user-sobigdata/Dockerfile" + # "single-user-sobigdata/Dockerfile" "single-user-3d-d4science/Dockerfile" "single-user-r-d4science/Dockerfile" "single-user-scientometrics-d4science/Dockerfile" diff --git a/d4science-base/Dockerfile b/d4science-base/Dockerfile index 59b88d96..b01d60a3 100644 --- a/d4science-base/Dockerfile +++ b/d4science-base/Dockerfile @@ -5,13 +5,13 @@ ARG julia_version="1.9.3" USER root -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ +RUN apt-get -qq update \ + && apt-get -qq install --no-install-recommends -y \ less \ graphviz \ vim \ openssh-client \ - && apt-get clean \ + && apt-get -qq clean \ && rm -rf /var/lib/apt/lists/* RUN curl https://raw.githubusercontent.com/kadwanev/retry/master/retry \ @@ -70,7 +70,7 @@ RUN julia -e 'import Pkg; Pkg.update()' && \ # ----------------------------------------------------------------------------- -RUN mamba install -y -q \ +RUN mamba install --channel conda-forge -y -q \ rdflib \ prov \ graphviz \ @@ -82,10 +82,29 @@ RUN mamba install -y -q \ cartopy \ ipyleaflet \ nbgitpuller \ + basemap \ + beakerx \ + xgboost \ + lightgbm \ + catboost \ + netcdf4 \ + aiohttp \ + graph-tool \ + scikit-learn \ + scikit-mobility \ + dask-jobqueue \ + fs \ + rasterio \ + contextily \ + nb_conda_kernels \ + nb_conda \ + pyophidia \ + # bring dash extension hopefully + dash \ + jupyter-server-proxy \ + && conda clean -tipy \ && conda clean --all RUN pip install --no-cache-dir shortid WORKDIR "${HOME}" - -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager jupyter-leaflet diff --git a/single-user-d4science/Dockerfile b/single-user-d4science/Dockerfile index cc275284..e97ebe15 100644 --- a/single-user-d4science/Dockerfile +++ b/single-user-d4science/Dockerfile @@ -4,38 +4,18 @@ FROM $BASE_IMAGE USER root -RUN apt-get update \ - && apt-get install --no-install-recommends -y \ +RUN apt-get -qq update \ + && apt-get -qq install --no-install-recommends -y \ nco \ cdo \ gdal-bin \ cmake \ libgdal-dev \ - && apt-get clean \ + && apt-get -qq clean \ && rm -rf /var/lib/apt/lists/* USER $NB_UID -RUN mamba install -y --quiet \ - basemap \ - beakerx \ - xgboost \ - lightgbm \ - catboost \ - netcdf4 \ - aiohttp \ - graph-tool \ - scikit-learn \ - scikit-mobility \ - dask-jobqueue \ - fs \ - rasterio \ - contextily \ - nb_conda_kernels \ - nb_conda \ - pyophidia \ - && conda clean -tipy - RUN conda create --quiet --yes -n java8 \ && mamba install -n java8 --yes openjdk=8 \ && conda clean -tipy @@ -43,7 +23,12 @@ RUN conda create --quiet --yes -n java8 \ ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -# Pinned networkx to 2.6.3 as 2.7 makes networksns fail +# Installing torch explictly to have only CPU version +RUN pip install --index-url https://download.pytorch.org/whl/cpu \ + torch torchvision torchaudio + + +# cdlib[C]==0.2.3 is too old for this env, so unpinning RUN pip install --no-cache-dir \ bicm \ rdp \ @@ -54,7 +39,7 @@ RUN pip install --no-cache-dir \ ndlib \ motuclient \ panel==0.9.* \ - cdlib[C]==0.2.3 \ + cdlib[C] \ cython \ node_conformity \ git+https://github.com/obidam/pyxpcm.git \ @@ -63,7 +48,6 @@ RUN pip install --no-cache-dir \ git+https://github.com/guglielmocola/RetweetCascade.git@f96319d0107473715104acceb2ff0925d35dd9e3 \ fast-forward-indexes \ NEMtropy \ - folium==0.11.0 \ geopy==2.0.0 \ hvplot==0.6.0 \ intake==0.6.0 \ @@ -72,7 +56,6 @@ RUN pip install --no-cache-dir \ seaborn==0.11.0 \ zarr==2.5.0 \ holoviews \ - networkx==2.6.3 \ networksns \ GDAL==$(gdal-config --version) #need to match the version of gdal installed diff --git a/single-user-r-d4science/Dockerfile b/single-user-r-d4science/Dockerfile index 2f4b2c03..e3c8b104 100644 --- a/single-user-r-d4science/Dockerfile +++ b/single-user-r-d4science/Dockerfile @@ -16,6 +16,8 @@ RUN curl https://raw.githubusercontent.com/kadwanev/retry/master/retry \ USER $NB_UID +RUN pip install --no-cache-dir shortid + RUN mamba install -y -q \ rdflib \ prov \ @@ -28,24 +30,12 @@ RUN mamba install -y -q \ cartopy \ ipyleaflet \ nbgitpuller \ - && conda clean --all - -RUN pip install --no-cache-dir shortid - -WORKDIR "${HOME}" - -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager jupyter-leaflet - -RUN mamba install -y --quiet \ r-plotly \ r-gdata \ r-rpostgres \ r-geojsonio \ r-rlang \ r-rcpp \ - && conda clean -tipy - -RUN mamba install -y --quiet \ r-vctrs \ r-pillar \ r-magrittr \ @@ -55,9 +45,6 @@ RUN mamba install -y --quiet \ r-dbi \ r-cpp11 \ r-bh \ - && conda clean -tipy - -RUN mamba install -y --quiet \ r-tibble \ r-hms \ r-jsonlite \ diff --git a/single-user-sobigdata/Dockerfile b/single-user-sobigdata/Dockerfile index 07b91f9e..42dafb01 100644 --- a/single-user-sobigdata/Dockerfile +++ b/single-user-sobigdata/Dockerfile @@ -26,60 +26,6 @@ USER $NB_UID ARG SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL=True -RUN mamba install -y --quiet \ - basemap \ - beakerx \ - xgboost \ - lightgbm \ - catboost \ - netcdf4 \ - aiohttp \ - graph-tool \ - scikit-learn \ - scikit-mobility \ - dask-jobqueue \ - fs \ - rasterio \ - contextily \ - nb_conda_kernels \ - nb_conda \ - && conda clean -tipy - -# Pinned networkx to 2.6.3 as 2.7 makes networksns fail -RUN pip install --no-cache-dir \ - bicm \ - rdp \ - sentinelhub \ - import_ipynb \ - nbresuse \ - ndlib \ - motuclient \ - panel==0.9.* \ - cdlib[C]==0.2.3 \ - cython \ - node_conformity \ - git+https://github.com/geopython/OWSLib.git \ - git+https://github.com/guglielmocola/RetweetCascade.git \ - fast-forward-indexes \ - NEMtropy \ - folium==0.11.0 \ - geopy==2.0.0 \ - hvplot==0.6.0 \ - intake==0.6.0 \ - intake-esm==2020.8.15 \ - hda \ - seaborn==0.11.0 \ - zarr==2.5.0 \ - holoviews \ - networkx==2.6.3 \ - networksns \ - eventclf \ - XAI-Library \ - convrewriting \ - git+https://github.com/guglielmocola/TwitterMonitorLib_v2.git \ - pygm \ - structify-net - # ------------------------------- # SoBigData ++ specific libraries # -------------------------------