Db mods (#73)

* Converted over surveyors to observers * Moved utm_zone to integer * change connection string to utc default * change default timezone for testing * Fixes about half of the examples for #72 * Updated the docs to better reflect the database modification and splitting of the repos Co-authored-by: Micah Sandusky <micc5725@gmail.com>
SnowEx · Jul 6, 2022 · b2ab520 · b2ab520
1 parent 9726413
commit b2ab520
Show file tree

Hide file tree

Showing 17 changed files with 189 additions and 329 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -25,8 +25,8 @@ jobs:
           POSTGRES_PASSWORD: db_builder
           POSTGRES_USER: builder
           POSTGRES_DB: test
-          TZ: 'MST'
-          PGTZ: 'MST'
+          TZ: 'UTC'
+          PGTZ: 'UTC'
           POSTGIS_GDAL_ENABLED_DRIVERS: 'ENABLE_ALL'
           POSTGIS_ENABLE_OUTDB_RASTERS: 'True'
         ports:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,8 +10,8 @@ services:
       POSTGRES_DB: 'test'
       POSTGIS_GDAL_ENABLED_DRIVERS: 'ENABLE_ALL'
       POSTGIS_ENABLE_OUTDB_RASTERS: 'True'
-      TZ: 'MST'
-      PGTZ: 'MST'
+      TZ: 'UTC'
+      PGTZ: 'UTC'
     ports:
       - 5432:5432
     healthcheck:

diff --git a/docs/database_structure.rst b/docs/database_structure.rst
@@ -53,7 +53,7 @@ This table has the following columns:
 * northing - Northing coordinate projected in UTM in meters
 * site_id - Unique identifier to pit location
 * site_name - Name describing the general survey area ( e.g. Grand Mesa)
-* surveyors - Name of the people who acquired the data
+* observers - Name of the people who acquired the data
 * time - Time (MST) that the data was collected
 * time_updated - Time this entry was last modified
 * type - Name of the data collected
@@ -97,6 +97,7 @@ This table contains the following columns:
 * depth - Depth in centimeters in the snowpack that the data was collected
 * easting - UTM projected coordinate in the east direction in meters
 * elevation - Elevation at the site or acquisition in meters
+* flags - data that was flagged typically just pits
 * geom - GIS software friendly version of the coordinates of where the data was collected in UTM.
 * id - Unique identifier that is automatically assigned when uploaded
 * instrument - Name of the instrument used to collect the data
@@ -108,7 +109,7 @@ This table contains the following columns:
 * sample_c - 1 of potentially three samples that could have been taken for this measurement, e.g. density
 * site_id - Unique identifier to pit location
 * site_name - Name describing the general survey area ( e.g. Grand Mesa)
-* surveyors - Names of the people performing the acquisition
+* observers - Names of the people performing the acquisition
 * time - Time (MST) at the beginning of acquisition
 * time_created - Time this entry was uploaded
 * time_updated - Time this entry was last modified
@@ -146,7 +147,7 @@ This table contains the following columns:
 * raster - Raster data in Well Known Binary Format (WKB) best generated using `raster2psql` in the command line
 * site_id - Unique identifier to pit location
 * site_name - Name describing the general survey area ( e.g. Grand Mesa)
-* surveyors - Names of the people or organization that acquired the data
+* observers - Names of the people or organization that acquired the data
 * time_created - Time this entry was uploaded
 * time_updated - Time this entry was last modified
 * type - Name of the data collected

diff --git a/docs/gallery/compare_SSA_instruments_example.ipynb b/docs/gallery/compare_SSA_instruments_example.ipynb
diff --git a/docs/gallery/compare_UAVSAR_to_depths_example.ipynb b/docs/gallery/compare_UAVSAR_to_depths_example.ipynb
diff --git a/docs/gallery/get_spiral_example.ipynb b/docs/gallery/get_spiral_example.ipynb
diff --git a/docs/gallery/getting_started_example.ipynb b/docs/gallery/getting_started_example.ipynb
@@ -37,14 +37,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "5N15, 2S20, 2N13, 7S50, 7S23, 8N33, 9N30, 6S53, 6S32, 8N35, 9N39, 5N10, 8S41, 2N49, 6N17, TLSFL2A, 5N11, 2S45, 1C14, 8C26, 2C2, 2C9, 8S28, 9C28, 2C4, 1C7, 1C1, 4C30, 2C6, 6C24, 2N12, 8C22, 1C8, 2C3, 6C10, GML, 9C23, 8C31, 8C32, 5C27, 1S17, 6N16, 3N26, 1C1, 1S2, 2S16, 1N1, 9N28, 2S7, 2N8, 2S10, 8C18, 1N20, 8S18, 8C29, 6S22, 9C16, 8C11, 8N9, 8C36, 1S13, 6S15, 8C35, 2S4, 2S9, 8N54, 6S34, 5S49, 8N37, 9N47, 3S52, 8N52, 5S42, 9N42, 5S29, 8N51, 2S46, 2S27, 6S44, 7N40, 3S33, 9N43, 9N56, 6C37, 3S5, 6S19, 2S3, 1S1, 2S25, 5S43, 2S48, 2S36, 5S24, 9S51, 8S30, 3S47, FL1B, 5N24, 6S26, 2C12, 5N32, 2C13, 1C5, 9N29, 4N27, 8N34, 2C33, 9C17, 5C21, 8C25, 5S31, 2S35, 9C19, 6N36, 5C20, 6N31, 8N38, 7C15, 5N41, 9N44, 5N50, 3S14, 1S8, 5S21, 9S39, 2S37, 6N46, 8N45, 2N48, 6C34, 2S11, 3S38, 9S40, 1N3, 2N21, 2N14, 2S6, 1N7, 1N23, 1N5, 1S12, FL2A, 5N10, 2N4, 3N22, 5N19, 4N2, 7N57, 1N6, 3N53, 6N18, 8N55, 9N59, 8N25, 8N58\n",
+      "8C18, 1C1, Open 6, 2S4, 2S7, 6N36, Skyway Open, 6S22, 2S6, 9S40, 9N30, 1N1, 9N47, 1N3, 9N29, 8N45, 9N39, 8C35, 1C5, 6N31, 3S33, 5N24, 3N53, 1C7, 7N40, SNB 2, 8N55, County Line Open, 6N18, 5C20, 5N10, 8N52, 2N21, 1N5, 6S32, GML, 1S1, 2C3, 8S41, JPL 1, Forest 14, 6S53, C1, 8C11, Open, 2C13, Open Flat, Joe Wright, 8N54, 7S23, 6N16, 8C31, Open 2, 5S42, Mesa West Open, 8N34, Upper, 6C37, 5S49, 9S39, 1S17, 2S35, 1C8, BA Flux Tower, 8N9, FL2A, 5S31, 8N38, 6S26, Caples Lake, Open 4, 2S46, 8S28, 8C36, 5N15, 2C33, 6N46, 3S14, 6S15, 8N35, Skyway Tree, TLSFL2A, 2N13, 3N26, 1S8, Saddle, Banner Open, 3S47, 3S52, 4N2, 2S9, 9S51, Trench 13, 6C24, Panorama Dome, 5C27, Senator Beck, 2S25, Swamp Angel, FL1B, 9N42, 1N6, JPL 2, 2S11, 2N8, 9N59, 1N7, 8C25, 3S5, 8N58, 9C28, 2S10, 2S45, 5C21, 5S24, 7S50, 2N49, 8C22, Forest 13, 2N14, 9C17, 5N19, 2C9, 5N50, 2N4, Mores Creek Summit, LDP Tree, 1C14, 2C2, CUES, SNB 1, 8S18, Michigan River, 7C15, Irwin Barn, 2S20, 1S12, 6S44, 2S48, 9C19, 9N43, 9N56, 9N44, 8S30, 8C26, 7N57, 3S38, 9C16, 5N11, 6S34, 4N27, Forest 12, 5S21, 2C12, 2N12, 9C23, 1S2, 3N22, 2N48, 2S3, 5S29, 8N25, 2C4, Forest North, 2C6, Forest, 2S37, 2S16, HQ Met Station, 1S13, Alta Collins, 4C30, County Line Tree, Bogus Upper, 2S36, 6S19, 8N37, Forest Flat, Atwater, LDP Open, 9N28, 1N23, Gothic, 6N17, 8N51, 1N20, Forest South, 5N41, 8C32, 5S43, 8N33, 5N32, 6C10, Tower 4, Banner Snotel, 2S27, Mesa West Trees, 6C34, Aspen, 8C29\n",
       "\n"
      ]
     }
@@ -54,7 +54,7 @@
     "conn = engine.connect()\n",
     "\n",
     "# Form a typical SQL query and use python to populate the table name\n",
-    "qry = \"SELECT site_id FROM sites\"\n",
+    "qry = \"SELECT DISTINCT site_id FROM sites\"\n",
     "\n",
     "# Then we execute the sql command and collect the results\n",
     "results = conn.execute(qry)\n",
@@ -82,14 +82,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "5N15, 2S20, 2N13, 7S50, 7S23, 8N33, 9N30, 6S53, 6S32, 8N35, 9N39, 5N10, 8S41, 2N49, 6N17, TLSFL2A, 5N11, 2S45, 1C14, 8C26, 2C2, 2C9, 8S28, 9C28, 2C4, 1C7, 1C1, 4C30, 2C6, 6C24, 2N12, 8C22, 1C8, 2C3, 6C10, GML, 9C23, 8C31, 8C32, 5C27, 1S17, 6N16, 3N26, 1C1, 1S2, 2S16, 1N1, 9N28, 2S7, 2N8, 2S10, 8C18, 1N20, 8S18, 8C29, 6S22, 9C16, 8C11, 8N9, 8C36, 1S13, 6S15, 8C35, 2S4, 2S9, 8N54, 6S34, 5S49, 8N37, 9N47, 3S52, 8N52, 5S42, 9N42, 5S29, 8N51, 2S46, 2S27, 6S44, 7N40, 3S33, 9N43, 9N56, 6C37, 3S5, 6S19, 2S3, 1S1, 2S25, 5S43, 2S48, 2S36, 5S24, 9S51, 8S30, 3S47, FL1B, 5N24, 6S26, 2C12, 5N32, 2C13, 1C5, 9N29, 4N27, 8N34, 2C33, 9C17, 5C21, 8C25, 5S31, 2S35, 9C19, 6N36, 5C20, 6N31, 8N38, 7C15, 5N41, 9N44, 5N50, 3S14, 1S8, 5S21, 9S39, 2S37, 6N46, 8N45, 2N48, 6C34, 2S11, 3S38, 9S40, 1N3, 2N21, 2N14, 2S6, 1N7, 1N23, 1N5, 1S12, FL2A, 5N10, 2N4, 3N22, 5N19, 4N2, 7N57, 1N6, 3N53, 6N18, 8N55, 9N59, 8N25, 8N58\n",
+      "8C18, 1C1, Open 6, 2S4, 2S7, 6N36, Skyway Open, 6S22, 2S6, 9S40, 9N30, 1N1, 9N47, 1N3, 9N29, 8N45, 9N39, 8C35, 1C5, 6N31, 3S33, 5N24, 3N53, 1C7, 7N40, SNB 2, 8N55, County Line Open, 6N18, 5C20, 5N10, 8N52, 2N21, 1N5, 6S32, GML, 1S1, 2C3, 8S41, JPL 1, Forest 14, 6S53, C1, 8C11, Open, 2C13, Open Flat, Joe Wright, 8N54, 7S23, 6N16, 8C31, Open 2, 5S42, Mesa West Open, 8N34, Upper, 6C37, 5S49, 9S39, 1S17, 2S35, 1C8, BA Flux Tower, 8N9, FL2A, 5S31, 8N38, 6S26, Caples Lake, Open 4, 2S46, 8S28, 8C36, 5N15, 2C33, 6N46, 3S14, 6S15, 8N35, Skyway Tree, TLSFL2A, 2N13, 3N26, 1S8, Saddle, Banner Open, 3S47, 3S52, 4N2, 2S9, 9S51, Trench 13, 6C24, Panorama Dome, 5C27, Senator Beck, 2S25, Swamp Angel, FL1B, 9N42, 1N6, JPL 2, 2S11, 2N8, 9N59, 1N7, 8C25, 3S5, 8N58, 9C28, 2S10, 2S45, 5C21, 5S24, 7S50, 2N49, 8C22, Forest 13, 2N14, 9C17, 5N19, 2C9, 5N50, 2N4, Mores Creek Summit, LDP Tree, 1C14, 2C2, CUES, SNB 1, 8S18, Michigan River, 7C15, Irwin Barn, 2S20, 1S12, 6S44, 2S48, 9C19, 9N43, 9N56, 9N44, 8S30, 8C26, 7N57, 3S38, 9C16, 5N11, 6S34, 4N27, Forest 12, 5S21, 2C12, 2N12, 9C23, 1S2, 3N22, 2N48, 2S3, 5S29, 8N25, 2C4, Forest North, 2C6, Forest, 2S37, 2S16, HQ Met Station, 1S13, Alta Collins, 4C30, County Line Tree, Bogus Upper, 2S36, 6S19, 8N37, Forest Flat, Atwater, LDP Open, 9N28, 1N23, Gothic, 6N17, 8N51, 1N20, Forest South, 5N41, 8C32, 5S43, 8N33, 5N32, 6C10, Tower 4, Banner Snotel, 2S27, Mesa West Trees, 6C34, Aspen, 8C29\n",
       "\n"
      ]
     }
@@ -99,7 +99,7 @@
     "from  snowexsql.data import SiteData, PointData, LayerData, ImageData\n",
     "\n",
     "# Form the query to receive all the site_id from the sites table\n",
-    "qry = session.query(SiteData.site_id)\n",
+    "qry = session.query(SiteData.site_id).distinct()\n",
     "\n",
     "# Execute the query and collect the result\n",
     "results = qry.all()\n",
@@ -113,25 +113,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Close your session to avoid hanging transactions\n",
     "session.close()"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },

diff --git a/docs/gallery/graupel_pits_example.ipynb b/docs/gallery/graupel_pits_example.ipynb
diff --git a/docs/gallery/graupel_smp_example.ipynb b/docs/gallery/graupel_smp_example.ipynb