ADT: Add Biomarker/Pathology GX validation suites

.gitignore

@@ -139,3 +139,5 @@ test_staging_dir/
 
 # dev config file
 dev_config.yaml
+
+.vscode/

CONTRIBUTING.md

@@ -181,6 +181,24 @@ This repository is currently home to three custom expectations that were created
 
 These expectations are defined in the `/great_expectations/gx/plugins/expectations` folder. To add more custom expectations, follow the instructions [here](https://docs.greatexpectations.io/docs/guides/expectations/custom_expectations_lp).
 
+#### Nested Columns
+
+If the transform includes nested columns (example: `druggability` column in `gene_info` tranform), the following must be included:
+1. The nested column name must be specified in the config under `gx_nested_columns`.
+```
+gx_nested_columns:
+   - nested_column_name
+```
+2. When creating the validator object in the gx_suite_definitions notebook, the nested column(s) must be included in the `nested_columns` list.
+```
+df = pd.read_json(data_file)
+nested_columns = ['nested_column_name']
+df = GreatExpectationsRunner.convert_nested_columns_to_json(df, nested_columns)
+validator = context.sources.pandas_default.read_dataframe(df)
+validator.expectation_suite_name = "suite_name"
+```
+3. A JSON file containing the expected schema must be added: `src/agoradatatools/great_expectations/gx/json_schemas/transform_name/column_name.json`. Use the [JSON schema tool](https://jsonschema.net/app/schemas/0) to create the schema template for your nested column.
+
 ### DockerHub
 
 Rather than using GitHub actions to build and push Docker images to DockerHub, the Docker images are automatically built in DockerHub. This requires the `sagebiodockerhub` GitHub user to be an Admin of this repo. You can view the docker build [here](https://hub.docker.com/r/sagebionetworks/agora-data-tools).

gx_suite_definitions/biomarkers.ipynb

@@ -0,0 +1,239 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import synapseclient\n",
+    "\n",
+    "import great_expectations as gx\n",
+    "import pandas as pd\n",
+    "import json\n",
+    "\n",
+    "context = gx.get_context(project_root_dir='../src/agoradatatools/great_expectations')\n",
+    "\n",
+    "from agoradatatools.gx import GreatExpectationsRunner\n",
+    "from expectations.expect_column_values_to_have_list_length import ExpectColumnValuesToHaveListLength\n",
+    "from expectations.expect_column_values_to_have_list_members import ExpectColumnValuesToHaveListMembers\n",
+    "from expectations.expect_column_values_to_have_list_members_of_type import ExpectColumnValuesToHaveListMembersOfType"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create Expectation Suite for Biomarkers Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "syn = synapseclient.Synapse()\n",
+    "syn.login()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "biomarkers_data_file = syn.get(\"syn63540269\").path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Validator Object on Data File"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_json(biomarkers_data_file)\n",
+    "nested_columns = ['points']\n",
+    "df = GreatExpectationsRunner.convert_nested_columns_to_json(df, nested_columns)\n",
+    "validator = context.sources.pandas_default.read_dataframe(df)\n",
+    "validator.expectation_suite_name = \"biomarkers\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Add Expectations to Validator Object For Each Column"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# points\n",
+    "validator.expect_column_values_to_be_of_type(\"points\", \"str\")\n",
+    "with open(\"../src/agoradatatools/great_expectations/gx/json_schemas/immunohisto/points.json\", \"r\") as file:\n",
+    "    points_schema = json.load(file)\n",
+    "validator.expect_column_values_to_match_json_schema(\"points\", json_schema=points_schema)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# model\n",
+    "validator.expect_column_values_to_be_of_type(\"model\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"model\")\n",
+    "# allows all alphanumeric characters, underscores, periods, and dashes\n",
+    "validator.expect_column_values_to_match_regex(\"type\", \"^[A-Za-z0-9\\s\\*_.-]+$\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# type\n",
+    "validator.expect_column_values_to_be_of_type(\"type\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"type\")\n",
+    "# allows all alphanumeric characters, underscores, periods, and dashes\n",
+    "validator.expect_column_values_to_match_regex(\"type\", \"^[A-Za-z0-9\\s_.-]+$\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# units\n",
+    "validator.expect_column_values_to_be_of_type(\"units\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"units\")\n",
+    "# allows all alphanumeric characters, underscores, periods, and dashes\n",
+    "validator.expect_column_values_to_match_regex(\"units\", \"^[A-Za-z0-9\\/\\s\\*_.-]+$\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# age_death\n",
+    "validator.expect_column_values_to_be_of_type(\"age_death\", \"int\")\n",
+    "validator.expect_column_values_to_not_be_null(\"age_death\")\n",
+    "validator.expect_column_values_to_be_between(\"age_death\", strict_min_value=0, max_value=100)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# tissue\n",
+    "validator.expect_column_values_to_be_of_type(\"tissue\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"tissue\")\n",
+    "# allows all alphanumeric characters, underscores, periods, and dashes\n",
+    "validator.expect_column_values_to_match_regex(\"tissue\", \"^[A-Za-z0-9\\/\\s\\*_.-]+$\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# unique entries ExpectSelectColumnValuesToBeUniqueWithinRecord\n",
+    "validator.expect_select_column_values_to_be_unique_within_record(column_list=[\"model\", \"type\", \"age_death\", \"tissue\", \"units\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "validator.save_expectation_suite(discard_failed_expectations=False)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Checkpoint and View Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "checkpoint = context.add_or_update_checkpoint(\n",
+    "    name=\"agora-test-checkpoint\",\n",
+    "    validator=validator,\n",
+    ")\n",
+    "checkpoint_result = checkpoint.run()\n",
+    "context.view_validation_result(checkpoint_result)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build Data Docs - Click on Expectation Suite to View All Expectations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "context.build_data_docs()\n",
+    "context.open_data_docs()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}