From 9f8366d687272a8c357bc28d164177c8ddfc94ad Mon Sep 17 00:00:00 2001 From: Sandeep Puthanveetil Satheesan Date: Fri, 1 Nov 2024 16:36:28 -0500 Subject: [PATCH 1/7] When calculating total benefit payment, in CRP, include only Total CRP sub program. This changes the Title-2 total benefit amount value and the overall total benefit amount value. --- app/controllers/pdl.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/pdl.py b/app/controllers/pdl.py index f6f4ddf..d56dfff 100644 --- a/app/controllers/pdl.py +++ b/app/controllers/pdl.py @@ -766,7 +766,7 @@ def generate_allprograms_response(start_year, end_year): SUM(CASE WHEN t.name = 'Title I: Commodities' AND p.year = {year} THEN p.payment ELSE 0 END) AS "Title I {year}", -- Title II: Conservation - SUM(CASE WHEN t.name = 'Title II: Conservation' AND p.year = {year} THEN p.payment ELSE 0 END) + SUM(CASE WHEN t.name = 'Title II: Conservation' AND p.year = {year} AND (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN p.payment ELSE 0 END) AS "Title II {year}", -- Title IV: SNAP SUM(CASE WHEN t.name = 'Title IV: Nutrition' AND p.year = {year} THEN p.payment ELSE 0 END) @@ -775,7 +775,7 @@ def generate_allprograms_response(start_year, end_year): SUM(CASE WHEN t.name = 'Title IX: Crop Insurance' AND p.year = {year} THEN p.net_farmer_benefit_amount ELSE 0 END) AS "Crop Insurance {year}", -- All Programs Total for {year} - SUM(CASE WHEN p.year = {year} THEN COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0) + SUM(CASE WHEN p.year = {year} AND (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) ELSE 0 END) AS "{year} All Programs Total" """) @@ -785,7 +785,7 @@ def generate_allprograms_response(start_year, end_year): SUM(CASE WHEN t.name = 'Title I: Commodities' AND p.year BETWEEN {start_year} AND {end_year} THEN p.payment ELSE 0 END) AS "Title I Total", -- Corrected total for Title II - SUM(CASE WHEN t.name = 'Title II: Conservation' AND p.year BETWEEN {start_year} AND {end_year} + SUM(CASE WHEN t.name = 'Title II: Conservation' AND p.year BETWEEN {start_year} AND {end_year} AND (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN p.payment ELSE 0 END) AS "Title II Total", -- Corrected total for SNAP SUM(CASE WHEN t.name = 'Title IV: Nutrition' AND p.year BETWEEN {start_year} AND {end_year} @@ -794,7 +794,7 @@ def generate_allprograms_response(start_year, end_year): SUM(CASE WHEN t.name = 'Title IX: Crop Insurance' AND p.year BETWEEN {start_year} AND {end_year} THEN p.net_farmer_benefit_amount ELSE 0 END) AS "Crop Insurance Total", -- 18-22 All Programs Total - SUM(COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) AS "18-22 All Programs Total" + SUM(CASE WHEN (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) ELSE 0 END) AS "18-22 All Programs Total" """ # Combine the query of non-dynamic part @@ -895,13 +895,13 @@ def generate_allprograms_response(start_year, end_year): formatted_result[f"{year} All Programs Total"] = value total_row[f"{year} All Programs Total"] += value - # Add the final total for all programs between the years - formatted_result["18-22 All Programs Total"] = result_dict.get("18-22 All Programs Total", 0) - total_row["18-22 All Programs Total"] += formatted_result["18-22 All Programs Total"] - # Append the formatted result to the state data state_data.append(formatted_result) + # Add the final total for all programs between the years + for year in range(start_year, end_year + 1): + total_row["18-22 All Programs Total"] += total_row[f"{year} All Programs Total"] + # Append the total row at the end of state data state_data.append(total_row) From 3489292df9583a8484b0dba7825344aeb898980a Mon Sep 17 00:00:00 2001 From: Sandeep Puthanveetil Satheesan Date: Fri, 1 Nov 2024 16:36:53 -0500 Subject: [PATCH 2/7] Update CHANGELOG. --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d44f58..3d40c01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.18.0] - 2024-10-31 +## [0.19.0] - 2024-11-03 + ### Changed - Landing page values to use data from the database. [#258](https://github.com/policy-design-lab/pdl-api/issues/258) +### Fixed +- Title-II total payment calculation and overall benefits payment amounts calculation in the GET /pdl/allprograms endpoint. [#268](https://github.com/policy-design-lab/pdl-api/issues/268) + ## [0.18.0] - 2024-10-07 ### Changed From 200b6a4f33fce6a048b59c37cee9fb8ca7c03781 Mon Sep 17 00:00:00 2001 From: Sandeep Puthanveetil Satheesan Date: Fri, 1 Nov 2024 16:48:06 -0500 Subject: [PATCH 3/7] Update README with detail on total payment amount calculation. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5565acc..801deec 100644 --- a/README.md +++ b/README.md @@ -44,4 +44,7 @@ Now you can view the API at http://localhost/pdl/ For swagger doc, go to http://localhost/ui - +## Short note on total payment calculation +When calculating the total dollar amounts payment calculation, the following details are considered in the GET /pdl/allprograms endpoint: +1. In CRP, the "Total CRP" sub program contains the total value of all CRP payments. Just for the purpose of calculating the total values, the other sub programs and sub-sub programs in that can be ignored. +2. In Crop Insurance, the column `net_farmer_benefits_amount` contain the amount value that should be used for the total payment calculation. The `payment` column value is NULL for that program. From ac75582b0a337ec29da3fe29794db7256eb3786a Mon Sep 17 00:00:00 2001 From: Sandeep Puthanveetil Satheesan Date: Sun, 3 Nov 2024 22:10:48 -0600 Subject: [PATCH 4/7] Add total for year range to state JSON object. --- app/controllers/pdl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/pdl.py b/app/controllers/pdl.py index d56dfff..5c8e0b7 100644 --- a/app/controllers/pdl.py +++ b/app/controllers/pdl.py @@ -895,6 +895,9 @@ def generate_allprograms_response(start_year, end_year): formatted_result[f"{year} All Programs Total"] = value total_row[f"{year} All Programs Total"] += value + # Add the final total for all programs between the years + formatted_result["18-22 All Programs Total"] = result_dict.get("18-22 All Programs Total", 0) + # Append the formatted result to the state data state_data.append(formatted_result) From 7cd12a7fb93b26f7d9169bb2c5915e2d84863c8f Mon Sep 17 00:00:00 2001 From: Sandeep Puthanveetil Satheesan Date: Mon, 4 Nov 2024 09:24:42 -0600 Subject: [PATCH 5/7] Use start and end years to calculate key and minor changes to comments. --- app/controllers/pdl.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/app/controllers/pdl.py b/app/controllers/pdl.py index 5c8e0b7..69f647c 100644 --- a/app/controllers/pdl.py +++ b/app/controllers/pdl.py @@ -781,20 +781,20 @@ def generate_allprograms_response(start_year, end_year): # Add the totals for each program and the overall total totals = f""" - -- Corrected total for Title I + -- Title I Total SUM(CASE WHEN t.name = 'Title I: Commodities' AND p.year BETWEEN {start_year} AND {end_year} THEN p.payment ELSE 0 END) AS "Title I Total", - -- Corrected total for Title II + -- Title II Total SUM(CASE WHEN t.name = 'Title II: Conservation' AND p.year BETWEEN {start_year} AND {end_year} AND (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN p.payment ELSE 0 END) AS "Title II Total", - -- Corrected total for SNAP + -- SNAP Total SUM(CASE WHEN t.name = 'Title IV: Nutrition' AND p.year BETWEEN {start_year} AND {end_year} THEN p.payment ELSE 0 END) AS "SNAP Total", - -- Corrected total for Crop Insurance + -- Crop Insurance Total SUM(CASE WHEN t.name = 'Title IX: Crop Insurance' AND p.year BETWEEN {start_year} AND {end_year} THEN p.net_farmer_benefit_amount ELSE 0 END) AS "Crop Insurance Total", - -- 18-22 All Programs Total - SUM(CASE WHEN (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) ELSE 0 END) AS "18-22 All Programs Total" + -- start_year-end_year All Programs Total + SUM(CASE WHEN (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) ELSE 0 END) AS "{str(start_year)[-2:]}-{str(end_year)[-2:]} All Programs Total" """ # Combine the query of non-dynamic part @@ -819,6 +819,7 @@ def generate_allprograms_response(start_year, end_year): state_data = [] total_row = {"State": "Total"} + start_to_end_years_total_key = str(start_year)[-2:] + "-" + str(end_year)[-2:] + " All Programs Total" # Initialize total_row with zero values for all keys in the desired order for year in range(start_year, end_year + 1): @@ -839,7 +840,7 @@ def generate_allprograms_response(start_year, end_year): for year in range(start_year, end_year + 1): total_row[f"{year} All Programs Total"] = 0 - total_row["18-22 All Programs Total"] = 0 + total_row[start_to_end_years_total_key] = 0 for row in result: result_dict = dict(zip(columns, row)) # Using zip() to match columns with their values @@ -896,14 +897,14 @@ def generate_allprograms_response(start_year, end_year): total_row[f"{year} All Programs Total"] += value # Add the final total for all programs between the years - formatted_result["18-22 All Programs Total"] = result_dict.get("18-22 All Programs Total", 0) + formatted_result[start_to_end_years_total_key] = result_dict.get(start_to_end_years_total_key, 0) # Append the formatted result to the state data state_data.append(formatted_result) # Add the final total for all programs between the years for year in range(start_year, end_year + 1): - total_row["18-22 All Programs Total"] += total_row[f"{year} All Programs Total"] + total_row[start_to_end_years_total_key] += total_row[f"{year} All Programs Total"] # Append the total row at the end of state data state_data.append(total_row) @@ -987,7 +988,7 @@ def generate_summary_response(start_year, end_year): except Exception as e: # Log any errors that occur - logger.error(f"Error occurred: {str(e)}") + logging.error(f"Error occurred: {str(e)}") return {"error": str(e)} finally: From 4812d9f01130c0ff4d1592a37b385036b9a4b2ab Mon Sep 17 00:00:00 2001 From: Sandeep Puthanveetil Satheesan Date: Mon, 4 Nov 2024 11:25:50 -0600 Subject: [PATCH 6/7] Fix SQL query for total payment calculation by including Title-1 sub programs. --- app/controllers/pdl.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/app/controllers/pdl.py b/app/controllers/pdl.py index 69f647c..a09c819 100644 --- a/app/controllers/pdl.py +++ b/app/controllers/pdl.py @@ -775,7 +775,7 @@ def generate_allprograms_response(start_year, end_year): SUM(CASE WHEN t.name = 'Title IX: Crop Insurance' AND p.year = {year} THEN p.net_farmer_benefit_amount ELSE 0 END) AS "Crop Insurance {year}", -- All Programs Total for {year} - SUM(CASE WHEN p.year = {year} AND (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) + SUM(CASE WHEN p.year = {year} AND (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP' or pdl.sub_programs.name = 'Agriculture Risk Coverage County Option (ARC-CO)' OR pdl.sub_programs.name = 'Agriculture Risk Coverage Individual Coverage (ARC-IC)') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) ELSE 0 END) AS "{year} All Programs Total" """) @@ -794,7 +794,7 @@ def generate_allprograms_response(start_year, end_year): SUM(CASE WHEN t.name = 'Title IX: Crop Insurance' AND p.year BETWEEN {start_year} AND {end_year} THEN p.net_farmer_benefit_amount ELSE 0 END) AS "Crop Insurance Total", -- start_year-end_year All Programs Total - SUM(CASE WHEN (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) ELSE 0 END) AS "{str(start_year)[-2:]}-{str(end_year)[-2:]} All Programs Total" + SUM(CASE WHEN (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP' or pdl.sub_programs.name = 'Agriculture Risk Coverage County Option (ARC-CO)' OR pdl.sub_programs.name = 'Agriculture Risk Coverage Individual Coverage (ARC-IC)') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) ELSE 0 END) AS "{str(start_year)[-2:]}-{str(end_year)[-2:]} All Programs Total" """ # Combine the query of non-dynamic part @@ -936,7 +936,13 @@ def generate_summary_response(start_year, end_year): JOIN pdl.titles t ON p.title_id = t.id WHERE - p.year BETWEEN {start_year} AND {end_year} + p.year BETWEEN {start_year} AND {end_year} + AND p.sub_program_id IN + (SELECT id FROM pdl.sub_programs + WHERE pdl.sub_programs.name = 'Total CRP' + OR pdl.sub_programs.name = 'Agriculture Risk Coverage County Option (ARC-CO)' + OR pdl.sub_programs.name = 'Agriculture Risk Coverage Individual Coverage (ARC-IC)') + OR p.sub_program_id is NULL GROUP BY t.name, p.state_code, p.year ORDER BY From 9bd665a229c6e4df620b2ec43bd5a17b6bd93de0 Mon Sep 17 00:00:00 2001 From: Sandeep Puthanveetil Satheesan Date: Mon, 4 Nov 2024 11:32:05 -0600 Subject: [PATCH 7/7] Simplify the SQL queries. --- app/controllers/pdl.py | 52 ++++++++++++------------------------------ 1 file changed, 15 insertions(+), 37 deletions(-) diff --git a/app/controllers/pdl.py b/app/controllers/pdl.py index a09c819..c1385a4 100644 --- a/app/controllers/pdl.py +++ b/app/controllers/pdl.py @@ -762,39 +762,20 @@ def generate_allprograms_response(start_year, end_year): for year in range(start_year, end_year + 1): year_columns.append(f""" - -- Title I: Commodities - SUM(CASE WHEN t.name = 'Title I: Commodities' AND p.year = {year} THEN p.payment ELSE 0 END) - AS "Title I {year}", - -- Title II: Conservation - SUM(CASE WHEN t.name = 'Title II: Conservation' AND p.year = {year} AND (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) THEN p.payment ELSE 0 END) - AS "Title II {year}", - -- Title IV: SNAP - SUM(CASE WHEN t.name = 'Title IV: Nutrition' AND p.year = {year} THEN p.payment ELSE 0 END) - AS "SNAP {year}", - -- Title IX: Crop Insurance - SUM(CASE WHEN t.name = 'Title IX: Crop Insurance' AND p.year = {year} - THEN p.net_farmer_benefit_amount ELSE 0 END) AS "Crop Insurance {year}", - -- All Programs Total for {year} - SUM(CASE WHEN p.year = {year} AND (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP' or pdl.sub_programs.name = 'Agriculture Risk Coverage County Option (ARC-CO)' OR pdl.sub_programs.name = 'Agriculture Risk Coverage Individual Coverage (ARC-IC)') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) - ELSE 0 END) AS "{year} All Programs Total" + SUM(CASE WHEN t.name = 'Title I: Commodities' AND p.year = {year} THEN p.payment ELSE 0 END) AS "Title I {year}", + SUM(CASE WHEN t.name = 'Title II: Conservation' AND p.year = {year} AND (sub_program_id IN (SELECT id FROM pdl.sub_programs WHERE pdl.sub_programs.name = 'Total CRP') OR sub_program_id IS NULL) THEN p.payment ELSE 0 END) AS "Title II {year}", + SUM(CASE WHEN t.name = 'Title IV: Nutrition' AND p.year = {year} THEN p.payment ELSE 0 END) AS "SNAP {year}", + SUM(CASE WHEN t.name = 'Title IX: Crop Insurance' AND p.year = {year} THEN p.net_farmer_benefit_amount ELSE 0 END) AS "Crop Insurance {year}", + SUM(CASE WHEN p.year = {year} AND (sub_program_id IN (SELECT id FROM pdl.sub_programs WHERE pdl.sub_programs.name IN ('Total CRP', 'Agriculture Risk Coverage County Option (ARC-CO)', 'Agriculture Risk Coverage Individual Coverage (ARC-IC)')) OR sub_program_id IS NULL) THEN COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0) ELSE 0 END) AS "{year} All Programs Total" """) # Add the totals for each program and the overall total totals = f""" - -- Title I Total - SUM(CASE WHEN t.name = 'Title I: Commodities' AND p.year BETWEEN {start_year} AND {end_year} - THEN p.payment ELSE 0 END) AS "Title I Total", - -- Title II Total - SUM(CASE WHEN t.name = 'Title II: Conservation' AND p.year BETWEEN {start_year} AND {end_year} AND (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP') OR sub_program_id is NULL) - THEN p.payment ELSE 0 END) AS "Title II Total", - -- SNAP Total - SUM(CASE WHEN t.name = 'Title IV: Nutrition' AND p.year BETWEEN {start_year} AND {end_year} - THEN p.payment ELSE 0 END) AS "SNAP Total", - -- Crop Insurance Total - SUM(CASE WHEN t.name = 'Title IX: Crop Insurance' AND p.year BETWEEN {start_year} AND {end_year} - THEN p.net_farmer_benefit_amount ELSE 0 END) AS "Crop Insurance Total", - -- start_year-end_year All Programs Total - SUM(CASE WHEN (sub_program_id in (SELECT id from pdl.sub_programs where pdl.sub_programs.name = 'Total CRP' or pdl.sub_programs.name = 'Agriculture Risk Coverage County Option (ARC-CO)' OR pdl.sub_programs.name = 'Agriculture Risk Coverage Individual Coverage (ARC-IC)') OR sub_program_id is NULL) THEN (COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0)) ELSE 0 END) AS "{str(start_year)[-2:]}-{str(end_year)[-2:]} All Programs Total" + SUM(CASE WHEN t.name = 'Title I: Commodities' AND p.year BETWEEN {start_year} AND {end_year} THEN p.payment ELSE 0 END) AS "Title I Total", + SUM(CASE WHEN t.name = 'Title II: Conservation' AND p.year BETWEEN {start_year} AND {end_year} AND (sub_program_id IN (SELECT id FROM pdl.sub_programs WHERE pdl.sub_programs.name = 'Total CRP') OR sub_program_id IS NULL) THEN p.payment ELSE 0 END) AS "Title II Total", + SUM(CASE WHEN t.name = 'Title IV: Nutrition' AND p.year BETWEEN {start_year} AND {end_year} THEN p.payment ELSE 0 END) AS "SNAP Total", + SUM(CASE WHEN t.name = 'Title IX: Crop Insurance' AND p.year BETWEEN {start_year} AND {end_year} THEN p.net_farmer_benefit_amount ELSE 0 END) AS "Crop Insurance Total", + SUM(CASE WHEN (sub_program_id IN (SELECT id FROM pdl.sub_programs WHERE pdl.sub_programs.name IN ('Total CRP', 'Agriculture Risk Coverage County Option (ARC-CO)', 'Agriculture Risk Coverage Individual Coverage (ARC-IC)')) OR sub_program_id IS NULL) THEN COALESCE(p.payment, 0) + COALESCE(p.net_farmer_benefit_amount, 0) ELSE 0 END) AS "{str(start_year)[-2:]}-{str(end_year)[-2:]} All Programs Total" """ # Combine the query of non-dynamic part @@ -935,14 +916,11 @@ def generate_summary_response(start_year, end_year): pdl.payments p JOIN pdl.titles t ON p.title_id = t.id + LEFT JOIN + pdl.sub_programs sp ON p.sub_program_id = sp.id WHERE - p.year BETWEEN {start_year} AND {end_year} - AND p.sub_program_id IN - (SELECT id FROM pdl.sub_programs - WHERE pdl.sub_programs.name = 'Total CRP' - OR pdl.sub_programs.name = 'Agriculture Risk Coverage County Option (ARC-CO)' - OR pdl.sub_programs.name = 'Agriculture Risk Coverage Individual Coverage (ARC-IC)') - OR p.sub_program_id is NULL + p.year BETWEEN {start_year} AND {end_year} + AND (sp.name IN ('Total CRP', 'Agriculture Risk Coverage County Option (ARC-CO)', 'Agriculture Risk Coverage Individual Coverage (ARC-IC)') OR p.sub_program_id IS NULL) GROUP BY t.name, p.state_code, p.year ORDER BY @@ -950,7 +928,7 @@ def generate_summary_response(start_year, end_year): WHEN t.name = 'Title I: Commodities' THEN 1 WHEN t.name = 'Title II: Conservation' THEN 2 WHEN t.name = 'Title IX: Crop Insurance' THEN 3 - WHEN t.name = 'Title IV: Nutrition' THEN 4 -- SNAP comes last + WHEN t.name = 'Title IV: Nutrition' THEN 4 END, p.year, p.state_code;