open_data/arcgis_pro_notebook_sample.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import arcpy\n",
    "import json\n",
    "\n",
    "from arcpy import metadata as md\n",
    "S_NUMBER = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "arcpy.env.workspace = os.path.join(\n",
    "    \"C:\\\\\", \"Users\", S_NUMBER, \n",
    "    \"Documents\", \"ArcGIS\"\n",
    ")\n",
    "working_dir = arcpy.env.workspace\n",
    "working_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "directory = arcpy.GetInstallInfo(\"desktop\")[\"InstallDir\"] \n",
    "directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Set datasets to update...match to `update_vars.RUN_ME`\n",
    "in_features = [\n",
    "    'ca_hq_transit_areas',\n",
    "    'ca_hq_transit_stops',\n",
    "    'ca_transit_routes',\n",
    "    'ca_transit_stops',\n",
    "    'speeds_by_stop_segments',\n",
    "    'speeds_by_route_time_of_day'\n",
    "]\n",
    "\n",
    "staging_location = 'staging.gdb'\n",
    "out_location = 'open_data.gdb'\n",
    "\n",
    "def feature_class_in_gdb_path(my_gdb, file_name):\n",
    "    return os.path.join(my_gdb, file_name)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unzip zipped shapefiles, download metadata.json into local path\n",
    "\n",
    "### Set FGDC field defs for each dataset and export XML (do once when new dataset added)\n",
    "\n",
    "Only the FGDC standard keeps fields.\n",
    "See if we can use this and combine it with our ISO 19139 standard later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in json with all the changes we need for each layer\n",
    "with open(f\"{working_dir}\\metadata.json\") as f:\n",
    "    meta_dict = json.load(f)\n",
    "\n",
    "    \n",
    "def update_metadata_class(this_feature_class, meta_dict_for_dataset: dict):\n",
    "    \"\"\"\n",
    "    Update the elements in the arcpy.metadata class.\n",
    "    \"\"\"\n",
    "    # Now update metadata class elements that are available\n",
    "    source_metadata = md.Metadata(this_feature_class)\n",
    "\n",
    "    source_metadata.title = meta_dict_for_dataset[\"dataset_name\"]\n",
    "    source_metadata.tags = meta_dict_for_dataset[\"theme_keywords\"]\n",
    "    source_metadata.summary = meta_dict_for_dataset[\"summary_purpose\"]\n",
    "    source_metadata.description = meta_dict_for_dataset[\"description\"]\n",
    "    source_metadata.accessConstraints = meta_dict_for_dataset[\"public_access\"]\n",
    "    source_metadata.save()\n",
    "    \n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def export_fgdc_metadata(one_feature_class):\n",
    "    \"\"\"\n",
    "    Export XML as FGDC format, \n",
    "    that's the only one that keeps field names and definitions\n",
    "    available.\n",
    "    \"\"\"\n",
    "    this_feature_class = feature_class_in_gdb_path(\n",
    "        staging_location, \n",
    "        one_feature_class\n",
    "    )\n",
    "    \n",
    "    subset_meta_dict = meta_dict[one_feature_class]\n",
    "\n",
    "    update_metadata_class(this_feature_class, subset_meta_dict)\n",
    "    \n",
    "    source_metadata = md.Metadata(this_feature_class)\n",
    "    \n",
    "    # Export metadata XML in FGDC   \n",
    "    meta_output = os.path.join(working_dir, \n",
    "                               f\"./{one_feature_class}_fgdc.xml\")\n",
    "            \n",
    "    TRANSLATOR = \"FGDC_CSDGM\"     \n",
    "\n",
    "    source_metadata.exportMetadata(\n",
    "        outputPath = meta_output, \n",
    "        metadata_export_option = TRANSLATOR\n",
    "    )\n",
    "    print(f\"Exported FGDC XML for {one_feature_class}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Do field data dictionary updates in Jupyter Hub\n",
    "### Use shapefile and write it to a file gdb layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Clean up last run (if applicable)\n",
    "for f in in_features:\n",
    "    feature_path = f\"{working_dir}\\{f}.xml\"\n",
    "    if os.path.exists(feature_path):\n",
    "        os.remove(feature_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def shp_to_feature_class(file_name: str):\n",
    "    \"\"\"\n",
    "    From shapefile (directory of files), unpack those\n",
    "    and write it to our staging gdb as a feature class.\n",
    "    \"\"\"\n",
    "    # construct the filename, which is takes form of routes_assembled/routes_assembled.shp\n",
    "    shp_file_name = f\"{os.path.join(file_name, f'{file_name}.shp')}\"\n",
    "    \n",
    "    this_feature_class = os.path.join(staging_location, file_name)\n",
    "    \n",
    "    if arcpy.Exists(this_feature_class): \n",
    "        arcpy.management.Delete(this_feature_class)\n",
    "\n",
    "    # Execute FeatureClassToGeodatabase\n",
    "    arcpy.FeatureClassToGeodatabase_conversion(\n",
    "        shp_file_name, \n",
    "        staging_location\n",
    "    )\n",
    "    \n",
    "    # Print field names, just in case it needs renaming\n",
    "    # get a list of fields for each feature class\n",
    "    field_list = arcpy.ListFields(this_feature_class)  \n",
    "    \n",
    "    print(this_feature_class)\n",
    "    for field in field_list: \n",
    "        print(field.name)\n",
    "    \n",
    "    return\n",
    "\n",
    "\n",
    "for f in in_features:\n",
    "    shp_to_feature_class(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def rename_columns_with_dict(this_feature_class, rename_dict: dict):\n",
    "    \"\"\"\n",
    "    Get a list of fields for each feature class and use a dict to rename.\n",
    "    To change field names, must use AlterField_management, \n",
    "    because changing it in XML won't carry through when you sync\n",
    "    \"\"\"\n",
    "    field_list = arcpy.ListFields(this_feature_class)  \n",
    "\n",
    "    for field in field_list: \n",
    "        if field.name in rename_dict: \n",
    "            arcpy.AlterField_management(\n",
    "                this_feature_class, \n",
    "                field.name, \n",
    "                rename_dict[field.name], # new_field_name\n",
    "                rename_dict[field.name] # new_field_alias\n",
    "            ) \n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def update_feature_class_with_json(one_feature_class, meta_json_dict: dict):\n",
    "    \"\"\"\n",
    "    Update a single feature class.\n",
    "    Rename columns, apply FGDC metadata fields \n",
    "    template, and update metadata class attributes\n",
    "    that can be accessed through the arcpy.metadata class.\n",
    "    \"\"\"\n",
    "    this_feature_class = feature_class_in_gdb_path(\n",
    "        staging_location, \n",
    "        one_feature_class\n",
    "    )\n",
    "        \n",
    "    subset_meta_dict = meta_json_dict[one_feature_class]\n",
    "        \n",
    "    if \"rename_cols\" in subset_meta_dict.keys():  \n",
    "        rename_dict = subset_meta_dict[\"rename_cols\"]\n",
    "\n",
    "        rename_columns_with_dict(this_feature_class, rename_dict)\n",
    "    \n",
    "    # Check that renaming is done\n",
    "    print(this_feature_class)\n",
    "    check_fields = arcpy.ListFields(this_feature_class)\n",
    "    for field in check_fields:\n",
    "        print(field.name)\n",
    "    \n",
    "    # Sync with FGDC metadata \n",
    "    # (this is on the one_feature_class, which sits outside of staging/)\n",
    "    #import_fgdc_metadata_and_sync(one_feature_class)\n",
    "    \n",
    "    # Now update the rest of the metadata elements\n",
    "    update_metadata_class(this_feature_class, subset_meta_dict)\n",
    "\n",
    "    return\n",
    "\n",
    "    \n",
    "for f in in_features:\n",
    "    update_feature_class_with_json(f, meta_dict)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "# if there are updates to data_dictionary.yml, this needs to be run\n",
    "# so fields reflect new definitions.\n",
    "for f in in_features:\n",
    "    export_fgdc_metadata(f)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in in_features:\n",
    "    this_feature_class = feature_class_in_gdb_path(staging_location, f)\n",
    "\n",
    "    # Original metadata\n",
    "    # Migrating to Pro: https://pro.arcgis.com/en/pro-app/latest/arcpy/metadata/migrating-from-arcmap-to-arcgis-pro.htm\n",
    "\n",
    "    source_metadata = md.Metadata(this_feature_class)\n",
    "    # Export metadata XML    \n",
    "    meta_output = os.path.join(working_dir, f\"{f}.xml\")\n",
    "            \n",
    "    # In ArcGIS Pro, instead of FGDC for Desktop, use ISO 19139 GML 3.2\n",
    "    # https://sv03tmcpo.ct.dot.ca.gov/portal/apps/sites/#/geep/pages/open-data-request\n",
    "    TRANSLATOR = \"ISO19139_GML32\"     \n",
    "    \n",
    "    source_metadata.exportMetadata(\n",
    "        outputPath = meta_output, \n",
    "        metadata_export_option = TRANSLATOR\n",
    "    )\n",
    "    \n",
    "    print(f\"successful export: {f}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update XML in JupyterHub\n",
    "\n",
    "Run `python metadata_update_pro.py`\n",
    "\n",
    "### Import FGDC metadata for each dataset manually\n",
    "The button to Metadata > Import > type of metadata set to FGDC does something different than the `metadata.importMetadata` feature, which doesn't do it. Manually doing the import for the fgdb metadata works for each dataset only.\n",
    "\n",
    "Do this FGDC metadata first to get the field descriptions populated. If we do this second, certain items in the metadata will get overwritten and set to blank.\n",
    "\n",
    "Somewhere once FGDC applied first, it erases the tags we included. Sad.\n",
    "\n",
    "### With new XML, finish up workflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write layers to open_data gdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Write layers to open_data (with the overwritten and updated XML already)\n",
    "def write_feature_class_to_open_data(\n",
    "    one_feature_class,\n",
    "    staging_gdb = staging_location, \n",
    "    output_gdb = out_location, \n",
    "):\n",
    "    \"\"\"\n",
    "    Move the feature class from the staging gdb to the output gdb.\n",
    "    Delete the feature class in the output gdb because\n",
    "    we don't want _1 appended to the end\n",
    "    \"\"\"\n",
    "    staging_feature_class = feature_class_in_gdb_path(\n",
    "        staging_gdb, \n",
    "        one_feature_class\n",
    "    )\n",
    "    out_feature_class = feature_class_in_gdb_path(\n",
    "        output_gdb, \n",
    "        one_feature_class\n",
    "    )\n",
    "    \n",
    "    if arcpy.Exists(out_feature_class): \n",
    "        arcpy.management.Delete(out_feature_class)\n",
    "\n",
    "    # Copy over the feature class from staging.gdb to open_data.gdb\n",
    "    arcpy.conversion.FeatureClassToFeatureClass(\n",
    "        staging_feature_class, \n",
    "        output_gdb, \n",
    "        one_feature_class\n",
    "    )\n",
    "    \n",
    "    arcpy.conversion.FeatureClassToFeatureClass(\n",
    "        staging_feature_class, \n",
    "        output_gdb, \n",
    "        one_feature_class\n",
    "    )\n",
    "    \n",
    "    return\n",
    "    \n",
    "\n",
    "for f in in_features:\n",
    "    write_feature_class_to_open_data(f)\n",
    "    print(f\"in open_data.gdb: {f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exit and restart ArcPro to clear locks on layers in overwriting\n",
    "\n",
    "If we don't exit, the layer will be locked because it shows we're already using it (staging to open_data), and it will prevent writing from open_data to the enterprise gdb.\n",
    "\n",
    "License Select must be set to `Advanced` for this to work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ENTERPRISE_DATABASE = \"Database Connections/HQrail(edit)@sv03tmcsqlprd1.sde\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in in_features:\n",
    "    out_feature_class = feature_class_in_gdb_path(out_location, f)\n",
    "    \n",
    "    arcpy.FeatureClassToFeatureClass_conversion(\n",
    "        in_features = out_feature_class,\n",
    "        out_path = ENTERPRISE_DATABASE,\n",
    "        out_name = f\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}