Skip to content

Commit

Permalink
update thredds access notebook with magpie-auth active (reference htt…
Browse files Browse the repository at this point in the history 
…ps://github.com/bird-house/birdhouse-deploy/pull/107\#issuecomment-803167923)
  • Loading branch information
@fmigneault
fmigneault committed Mar 22, 2021
1 parent 0ace7bf commit 9c64b03
Showing 1 changed file with 152 additions and 97 deletions.
249 changes: 152 additions & 97 deletions docs/source/notebooks/pavics_thredds.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,42 @@
"# Accessing PAVICS THREDDS Server\n",
"\n",
"\n",
"The THREDDS data storing netCDF file on PAVICS has some public and private directories. Data from public directories can be accessed anonymously, while data from private directories require authentication. This notebook shows how to access public and private data on the THREDDS server. \n",
"The THREDDS data storing netCDF file on PAVICS has some public and private directories. Data from public directories can be accessed anonymously, while data from private directories require authentication. This notebook shows how to access public and private data on the THREDDS server.\n",
"\n",
"The PAVICS THREDDS server has a `testdata/` folder, in which we store test datasets to validate process requests. Within that directory is a `secure/` folder whose file access requires authentication (to be done). "
"The PAVICS THREDDS server has a `testdata/` folder, in which we store test datasets to validate process requests. Within that directory is a `secure/` folder whose file access requires authentication (to be done)."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false,
"jupyter": {
"outputs_hidden": false
},
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"THREDDS URL: https://host-140-10.rdext.crim.ca/twitcher/ows/proxy/thredds\n"
]
}
],
"source": [
"# NBVAL_IGNORE_OUTPUT\n",
"\n",
"# define some useful variables for following steps\n",
"import os\n",
"PAVICS_HOST = os.getenv(\"PAVICS_HOST\", \"pavics.ouranos.ca\")\n",
"THREDDS_URL = \"https://{}/twitcher/ows/proxy/thredds\".format(PAVICS_HOST)\n",
"\n",
"assert PAVICS_HOST != \"\", \"Invalid PAVICS HOST value.\"\n",
"print(\"THREDDS URL:\", THREDDS_URL)"
]
},
{
Expand All @@ -21,7 +54,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand All @@ -40,29 +73,14 @@
" lat_bnds (lat, bnds) float64 ...\n",
" lon_bnds (lon, bnds) float64 ...\n",
" ta (time, plev, lat, lon) float32 ...\n",
"Attributes:\n",
"Attributes: (12/28)\n",
" institution: MRI (Meteorological Research Institute, ...\n",
" institute_id: MRI\n",
" experiment_id: decadal1980\n",
" source: MRI-CGCM3 2011 atmosphere: GSMUV (gsmuv-...\n",
" model_id: MRI-CGCM3\n",
" forcing: GHG, SA, Oz, LU, Sl, Vl, BC, OC (GHG inc...\n",
" parent_experiment_id: N/A\n",
" parent_experiment_rip: N/A\n",
" branch_time: 0.0\n",
" contact: Seiji Yukimoto (yukimoto@mri-jma.go.jp)\n",
" history: Output from /sharex3/cmip5/decadal1980/r...\n",
" references: Model described by Yukimoto et al. (Tech...\n",
" initialization_method: 1\n",
" physics_version: 1\n",
" tracking_id: ce91e727-5f22-44fc-b24d-5bb53393ac69\n",
" product: output\n",
" experiment: 10- or 30-year run initialized in year 1980\n",
" frequency: mon\n",
" creation_date: 2011-08-12T05:05:34Z\n",
" Conventions: CF-1.4\n",
" project_id: CMIP5\n",
" table_id: Table Amon (26 July 2011) 976b7fd1d9e1be...\n",
" ... ...\n",
" title: MRI-CGCM3 model output prepared for CMIP...\n",
" parent_experiment: N/A\n",
" modeling_realm: atmos\n",
Expand All @@ -84,29 +102,14 @@
" lat_bnds (lat, bnds) float64 ...\n",
" lon_bnds (lon, bnds) float64 ...\n",
" ta (time, plev, lat, lon) float32 ...\n",
"Attributes:\n",
"Attributes: (12/28)\n",
" institution: MRI (Meteorological Research Institute, ...\n",
" institute_id: MRI\n",
" experiment_id: decadal1980\n",
" source: MRI-CGCM3 2011 atmosphere: GSMUV (gsmuv-...\n",
" model_id: MRI-CGCM3\n",
" forcing: GHG, SA, Oz, LU, Sl, Vl, BC, OC (GHG inc...\n",
" parent_experiment_id: N/A\n",
" parent_experiment_rip: N/A\n",
" branch_time: 0.0\n",
" contact: Seiji Yukimoto (yukimoto@mri-jma.go.jp)\n",
" history: Output from /sharex3/cmip5/decadal1980/r...\n",
" references: Model described by Yukimoto et al. (Tech...\n",
" initialization_method: 1\n",
" physics_version: 1\n",
" tracking_id: ce91e727-5f22-44fc-b24d-5bb53393ac69\n",
" product: output\n",
" experiment: 10- or 30-year run initialized in year 1980\n",
" frequency: mon\n",
" creation_date: 2011-08-12T05:05:34Z\n",
" Conventions: CF-1.4\n",
" project_id: CMIP5\n",
" table_id: Table Amon (26 July 2011) 976b7fd1d9e1be...\n",
" ... ...\n",
" title: MRI-CGCM3 model output prepared for CMIP...\n",
" parent_experiment: N/A\n",
" modeling_realm: atmos\n",
Expand All @@ -115,7 +118,7 @@
" DODS_EXTRA.Unlimited_Dimension: time"
]
},
"execution_count": 1,
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -124,41 +127,116 @@
"import xarray as xr\n",
"xr.set_options(display_style=\"text\") # comment out for html style, text style simpler for automated testing\n",
"\n",
"url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc\"\n",
"ds = xr.open_dataset(url)\n",
"PUBLIC_URL = f\"{THREDDS_URL}/dodsC/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc\"\n",
"ds = xr.open_dataset(PUBLIC_URL)\n",
"ds"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now let's do the same with a secured link. "
"Now let's do the same with a secured link."
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 7,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Unauthorized was raised as expected.\n"
]
}
],
"source": [
"secured_url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc\"\n",
"from webob.exc import HTTPError\n",
"\n",
"# This should fail but doesn't at the moment. \n",
"ds = xr.open_dataset(secured_url, decode_cf=False)"
"SECURED_URL = f\"{THREDDS_URL}/dodsC/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc\"\n",
"try:\n",
" ds = xr.open_dataset(SECURED_URL, decode_cf=False)\n",
"except HTTPError as exc: # FIXME: should be HTTPUnauthorized, but failing operation raises 500 with 'message' Unauthorized\n",
" http_error = exc\n",
"else:\n",
" raise RuntimeError(\"Expected unauthorized response, but dataset open operation did not raise!\")\n",
"assert \"401 Unauthorized\" in str(http_error)\n",
"print(\"Unauthorized was raised as expected.\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To open a secured link, we need to open a session. We've created a `authtest` user to facilitate testing. "
"To open a secured link, we need to open a session with `Authentication`.\n",
"Using wrong `Authentication` credentials will not work. They will raise immediately when failing login procedure.\n",
"Using valid credentials will instead succeed login, but will raise a forbidden response when attempting to retrieve\n",
"the data. Either way, user must be logged in and have appropriate access to fulfill `Authorization` requirements\n",
"of the resource.\n",
"\n",
"Let's see the result when credentials are invalid."
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Access with invalid credentials was not permitted as expected.\n"
]
}
],
"source": [
"import requests\n",
"from requests_magpie import MagpieAuth, MagpieAuthenticationError\n",
"\n",
"BAD_USR = \"an-invalid-user\"\n",
"BAD_PWD = \"or-bad-password\"\n",
"\n",
"try:\n",
" with requests.session() as session:\n",
" session.auth = MagpieAuth(f\"https://{PAVICS_HOST}/magpie\", BAD_USR, BAD_PWD)\n",
" xr.open_dataset(SECURED_URL, decode_cf=False) # Attributes are problematic with this file.\n",
"except (HTTPError, MagpieAuthenticationError) as exc: # error depends on what raises (unauthorized, forbidden, login failure)\n",
" print(\"Access with invalid credentials was not permitted as expected.\")\n",
"else:\n",
" raise RuntimeError(\"Expected authentication failure response, but login operation did not raise!\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"As we can see, the server identified that credentials were provided, but they were incorrect and could not log in.\n",
"Similar result would happen if login succeeded, but user was forbidden access due to insufficient permissions.\n",
"\n",
"We've created an `authtest` user in advance that has access to the `secure` contents to facilitate testing.\n",
"\n",
"Let's use it now to obtain the secured resource.\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false,
"jupyter": {
"outputs_hidden": false
},
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [
{
"data": {
Expand All @@ -176,29 +254,14 @@
" lon_bnds (lon, bnds) float64 ...\n",
" latitude_longitude |S128 ...\n",
" tasmax (time, lat, lon) float32 ...\n",
"Attributes:\n",
"Attributes: (12/28)\n",
" institution: Max Planck Institute for Meteorology\n",
" institute_id: MPI-M\n",
" experiment_id: rcp45\n",
" source: MPI-ESM-MR 2011; URL: http://svn.zmaw.de...\n",
" model_id: MPI-ESM-MR\n",
" forcing: GHG,Oz,SD,Sl,Vl,LU\n",
" parent_experiment_id: historical\n",
" parent_experiment_rip: r1i1p1\n",
" branch_time: 56978.0\n",
" contact: cmip5-mpi-esm@dkrz.de\n",
" history: Model raw output postprocessing with mod...\n",
" references: ECHAM6: n/a; JSBACH: Raddatz et al., 200...\n",
" initialization_method: 1\n",
" physics_version: 1\n",
" tracking_id: e7e78b40-458b-48bd-963a-aa377a89f90b\n",
" product: output\n",
" experiment: RCP4.5\n",
" frequency: mon\n",
" creation_date: 2011-10-11T11:27:30Z\n",
" Conventions: CF-1.4\n",
" project_id: CMIP5\n",
" table_id: Table Amon (27 April 2011) a5a1c518f52ae...\n",
" ... ...\n",
" title: MPI-ESM-MR model output prepared for CMI...\n",
" parent_experiment: historical\n",
" modeling_realm: atmos\n",
Expand All @@ -220,29 +283,14 @@
" lon_bnds (lon, bnds) float64 ...\n",
" latitude_longitude |S128 ...\n",
" tasmax (time, lat, lon) float32 ...\n",
"Attributes:\n",
"Attributes: (12/28)\n",
" institution: Max Planck Institute for Meteorology\n",
" institute_id: MPI-M\n",
" experiment_id: rcp45\n",
" source: MPI-ESM-MR 2011; URL: http://svn.zmaw.de...\n",
" model_id: MPI-ESM-MR\n",
" forcing: GHG,Oz,SD,Sl,Vl,LU\n",
" parent_experiment_id: historical\n",
" parent_experiment_rip: r1i1p1\n",
" branch_time: 56978.0\n",
" contact: cmip5-mpi-esm@dkrz.de\n",
" history: Model raw output postprocessing with mod...\n",
" references: ECHAM6: n/a; JSBACH: Raddatz et al., 200...\n",
" initialization_method: 1\n",
" physics_version: 1\n",
" tracking_id: e7e78b40-458b-48bd-963a-aa377a89f90b\n",
" product: output\n",
" experiment: RCP4.5\n",
" frequency: mon\n",
" creation_date: 2011-10-11T11:27:30Z\n",
" Conventions: CF-1.4\n",
" project_id: CMIP5\n",
" table_id: Table Amon (27 April 2011) a5a1c518f52ae...\n",
" ... ...\n",
" title: MPI-ESM-MR model output prepared for CMI...\n",
" parent_experiment: historical\n",
" modeling_realm: atmos\n",
Expand All @@ -251,27 +299,34 @@
" DODS_EXTRA.Unlimited_Dimension: time"
]
},
"execution_count": 3,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import requests\n",
"from requests_magpie import MagpieAuth\n",
"\n",
"secured_url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc\"\n",
"auth = MagpieAuth(\"https://pavics.ouranos.ca/magpie\", \"authtest\", \"authtest1234\")\n",
"AUTH_USR = os.getenv(\"TEST_MAGPIE_ADMIN_USERNAME\", \"authtest\")\n",
"AUTH_PWD = os.getenv(\"TEST_MAGPIE_ADMIN_PASSWORD\", \"authtest1234\")\n",
"\n",
"# Open session\n",
"session = requests.Session()\n",
"session.auth = auth\n",
"\n",
"# Open a Pydap data store and pass it to xarray\n",
"store = xr.backends.PydapDataStore.open(secured_url, session=session)\n",
"ds = xr.open_dataset(store, decode_cf=False) # Attributes are problematic with this file. \n",
"with requests.Session() as session:\n",
" session.auth = MagpieAuth(f\"https://{PAVICS_HOST}/magpie\", AUTH_USR, AUTH_PWD)\n",
" # Open a PyDAP data store and pass it to xarray\n",
" store = xr.backends.PydapDataStore.open(SECURED_URL, session=session)\n",
" ds = xr.open_dataset(store, decode_cf=False) # Attributes are problematic with this file.\n",
"ds"
]
},
{
"cell_type": "markdown",
"metadata": {
"pycharm": {
"name": "#%% md\n"
}
},
"source": [
"Successful listing of the above data means the user was granted access for this reference.\n"
]
}
],
"metadata": {
Expand All @@ -290,9 +345,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
"version": "3.7.9"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
"nbformat_minor": 4
}

0 comments on commit 9c64b03

Please sign in to comment.