From f88ffcead7b80976098b6e294d71ecdd04319bc6 Mon Sep 17 00:00:00 2001
From: YuviPanda <yuvipanda@gmail.com>
Date: Wed, 27 Oct 2021 11:42:17 +0530
Subject: [PATCH] Get some python packages from conda-forge

Cartopy and Shapely feel very difficult to install from PyPI,
as they have no binary wheels and require a specific version of
proj and geos from apt. But the R `sf` package also requires
a specific version of proj from apt, and this sort of puts us
in an impossible situation as cartopy 0.20.* requires a newer
version than sf.

So we start using conda-forge to install some of these base
scientific packages instead. This reduces the scope for a lot
of these conflicts, and lets us peacefully use apt just for R.
I'm trying to only have pip packages depend on conda, not
the other way around, so I've moved some of the 'base'
scientific python packages to be installed from conda.
Ideally, we should basically move everything.

Ref https://github.com/berkeley-dsep-infra/datahub/issues/2824
---
 .../datahub/images/default/environment.yml    | 37 +++++++++++++++++++
 .../datahub/images/default/requirements.txt   | 28 --------------
 2 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml
index 8000581e3..e68ffe22e 100644
--- a/deployments/datahub/images/default/environment.yml
+++ b/deployments/datahub/images/default/environment.yml
@@ -5,3 +5,40 @@ dependencies:
 
 # pymc3 needs this
 - mkl-service==2.4.*
+
+# Base scientific packages that other conda packages we install depend on
+# We don't want to have conda packages depend on pip packages if possible
+- numpy==1.21.*
+- matplotlib==3.4.*
+- scipy==1.7.*
+- ipympl==0.8.*
+- pandas==1.3.*
+- statsmodels==0.12.*
+- scikit-learn==0.24.*
+- seaborn==0.11.*
+- bokeh==2.3.*
+- decorator==5.0.*
+- networkx==2.6.*
+- spacy==3.1.*
+- nltk==3.6.*
+
+# data-x; DL
+- tensorflow==2.6.*
+- scikit-image==0.18.*
+
+# EPS88, data100
+# https://github.com/berkeley-dsep-infra/datahub/issues/1796
+# https://github.com/berkeley-dsep-infra/datahub/issues/2824
+- shapely==1.8.*
+- cartopy==0.20.*
+# data100, geog88
+# https://github.com/berkeley-dsep-infra/datahub/issues/2838
+- geopandas==0.10.*
+- geopy==2.2.*
+- pysal==2.5.*
+- rtree==0.9.*
+- netcdf4==1.5.*
+- mplleaflet==0.0.5
+
+# ls 88-3; neuro
+- pillow==8.3.*
\ No newline at end of file
diff --git a/deployments/datahub/images/default/requirements.txt b/deployments/datahub/images/default/requirements.txt
index d5063f4a7..eb9bfb4b4 100644
--- a/deployments/datahub/images/default/requirements.txt
+++ b/deployments/datahub/images/default/requirements.txt
@@ -1,10 +1,5 @@
 # data8; foundation
 datascience==0.17.0
-matplotlib==3.4.3
-ipympl==0.7.0
-pandas==1.3.2
-scipy==1.7.1
-statsmodels==0.12.2
 okpy==1.18.1
 folium==0.12.1
 #
@@ -16,28 +11,11 @@ jupyter-shiny-proxy==1.1
 # cogsci88;
 ggplot==0.11.5
 #
-# geog88; spring 2018
-mplleaflet==0.0.5
-# updating geopandas from 0.9.0 for issue #2838 (data100, fall 2021)
-geopandas==0.10.0
-geopy==2.2.0
-pysal==2.5.0
-rtree==0.9.7
-netCDF4==1.5.7
-#
 # cogsci131; spring 2018
 nose==1.3.7
 #
 # modules
-scikit-learn==0.24.2
-seaborn==0.11.2
-bokeh==2.3.3
-decorator==5.0.9
-networkx==2.6.2
-# pygame==1.9.6
-nltk==3.6.2
 beautifulsoup4==4.9.3
-spacy==3.1.1
 qgrid==1.3.1
 nb2pdf==0.6.2
 #
@@ -47,7 +25,6 @@ tqdm==4.62.1
 mne==0.23.0
 nibabel==3.2.1
 h5py==3.3.0
-pillow==8.3.1
 numexpr==2.7.3
 openpyxl==3.0.7
 nilearn==0.8.0
@@ -61,8 +38,6 @@ corner==2.2.1
 pymdptoolbox==4.0-b3
 #
 # data-x; DL
-tensorflow==2.6.0
-scikit-image==0.18.2
 tables==3.6.1
 opencv-python==4.5.3.56
 
@@ -88,8 +63,6 @@ gspread==4.0.1
 
 # eps 109; fall 2019
 ffmpeg-python==0.2.0
-# see https://github.com/berkeley-dsep-infra/datahub/issues/1796
-Cartopy==0.19.0.post1 --no-binary Cartopy
 
 # data 88e; spring 2021
 plotly==5.2.1
@@ -114,7 +87,6 @@ hdbscan==0.8.27
 
 # espm 125/bio 105; fall 2019
 # see https://github.com/berkeley-dsep-infra/datahub/issues/1796
-shapely==1.7.1 --no-binary shapely
 pyvcf==0.6.8
 bitarray==2.3.0
 nlmpy==1.0.1