From f139545eb93891348140d95bb3cadf1751741e69 Mon Sep 17 00:00:00 2001 From: Pranav Kakaraparti Date: Mon, 24 Jul 2023 18:18:44 -0500 Subject: [PATCH 1/7] feat: migration to add tree_id column --- .../20230724231226-add-tree-id-column.js | 53 +++++++++++++++++++ ...20230724231226-add-tree-id-column-down.sql | 1 + .../20230724231226-add-tree-id-column-up.sql | 1 + 3 files changed, 55 insertions(+) create mode 100644 database/migrations/20230724231226-add-tree-id-column.js create mode 100644 database/migrations/sqls/20230724231226-add-tree-id-column-down.sql create mode 100644 database/migrations/sqls/20230724231226-add-tree-id-column-up.sql diff --git a/database/migrations/20230724231226-add-tree-id-column.js b/database/migrations/20230724231226-add-tree-id-column.js new file mode 100644 index 0000000..f6d1310 --- /dev/null +++ b/database/migrations/20230724231226-add-tree-id-column.js @@ -0,0 +1,53 @@ +'use strict'; + +var dbm; +var type; +var seed; +var fs = require('fs'); +var path = require('path'); +var Promise; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function(options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; + Promise = options.Promise; +}; + +exports.up = function(db) { + var filePath = path.join(__dirname, 'sqls', '20230724231226-add-tree-id-column-up.sql'); + return new Promise( function( resolve, reject ) { + fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){ + if (err) return reject(err); + console.log('received data: ' + data); + + resolve(data); + }); + }) + .then(function(data) { + return db.runSql(data); + }); +}; + +exports.down = function(db) { + var filePath = path.join(__dirname, 'sqls', '20230724231226-add-tree-id-column-down.sql'); + return new Promise( function( resolve, reject ) { + fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){ + if (err) return reject(err); + console.log('received data: ' + data); + + resolve(data); + }); + }) + .then(function(data) { + return db.runSql(data); + }); +}; + +exports._meta = { + "version": 1 +}; diff --git a/database/migrations/sqls/20230724231226-add-tree-id-column-down.sql b/database/migrations/sqls/20230724231226-add-tree-id-column-down.sql new file mode 100644 index 0000000..5b28223 --- /dev/null +++ b/database/migrations/sqls/20230724231226-add-tree-id-column-down.sql @@ -0,0 +1 @@ +ALTER TABLE capture_denormalized DROP COLUMN tree_id; \ No newline at end of file diff --git a/database/migrations/sqls/20230724231226-add-tree-id-column-up.sql b/database/migrations/sqls/20230724231226-add-tree-id-column-up.sql new file mode 100644 index 0000000..4cb7e57 --- /dev/null +++ b/database/migrations/sqls/20230724231226-add-tree-id-column-up.sql @@ -0,0 +1 @@ +ALTER TABLE capture_denormalized ADD COLUMN tree_id uuid; \ No newline at end of file From 31f4c77049809759e3a61fb24f1a89371e014712 Mon Sep 17 00:00:00 2001 From: Pranav Kakaraparti Date: Mon, 24 Jul 2023 20:24:54 -0500 Subject: [PATCH 2/7] feat: repository methods for matched_captures --- server/repositories/CaptureRepository.js | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/server/repositories/CaptureRepository.js b/server/repositories/CaptureRepository.js index 28ae075..f40f81e 100644 --- a/server/repositories/CaptureRepository.js +++ b/server/repositories/CaptureRepository.js @@ -363,6 +363,24 @@ class CaptureRepository extends BaseRepository { .limit(options.limit) .offset(options.offset); + + // total number of matched captures + const totalMatchedCapturesQuery = knex(this._tableName) + .count() + .whereNotNull('tree_id') + .where((builder) => whereBuilder({ ...filter, approved: true }, builder)); + + //top matched captures (by organization) + const topMatchedCapturesQuery = knex(this._tableName) + .select(knex.raw('planting_organization_name, count(*) as count')) + .whereNotNull('tree_id') + .where((builder) => whereBuilder({ ...filter, approved: true }, builder)) + .groupBy('planting_organization_uuid', 'planting_organization_name') + .orderBy('count', 'desc') + .limit(options.limit) + .offset(options.offset); + + if (filter?.card_title) { const { card_title } = filter; @@ -402,6 +420,11 @@ class CaptureRepository extends BaseRepository { const approvalRates = await approvalRateQuery.cache(); return { approvalRates }; } + case 'matched_captures': { + console.log('MATCHED CAPTURES IS CALLED') + const topMatchedCaptures = await topMatchedCapturesQuery.cache(); + return { topMatchedCaptures }; + } default: break; @@ -429,6 +452,8 @@ class CaptureRepository extends BaseRepository { const topCatchment = await topCatchmentQuery.cache(); const genderCount = await genderCountQuery.cache(); const approvalRates = await approvalRateQuery.cache(); + const totalMatchedCaptures = await totalMatchedCapturesQuery.cache(); + const topMatchedCaptures = await topMatchedCapturesQuery.cache(); return { totalGrowers: +totalGrowers[0].totalPlanters, @@ -449,6 +474,8 @@ class CaptureRepository extends BaseRepository { topCatchment, genderCount, approvalRates, + totalMatchedCaptures: +totalMatchedCaptures[0].count, + topMatchedCaptures, }; } } From dbad7554d71a05c2c7e829374a70947d9ef08782 Mon Sep 17 00:00:00 2001 From: Pranav Kakaraparti Date: Mon, 24 Jul 2023 20:26:20 -0500 Subject: [PATCH 3/7] feat: matched_captures reporting card --- server/models/Capture.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/models/Capture.js b/server/models/Capture.js index 0cd7f42..763111a 100644 --- a/server/models/Capture.js +++ b/server/models/Capture.js @@ -94,6 +94,8 @@ class Capture { topCatchment = [], genderCount = [], approvalRates, + totalMatchedCaptures = undefined, + topMatchedCaptures = [], }) { const planters = { total: totalGrowers, @@ -170,11 +172,22 @@ class Capture { }; }); + const matched_captures = { + total: totalMatchedCaptures, + matched_captures: topMatchedCaptures.map(({ planting_organization_name, count }) => { + return { + name: planting_organization_name, + number: count, + } + }) + } + return { planters, species, captures, unverified_captures, + matched_captures, top_planters, trees_per_planters, last_updated_at, @@ -203,6 +216,8 @@ class Capture { topCatchment, genderCount, approvalRates, + totalMatchedCaptures, + topMatchedCaptures, } = await this._captureRepository.getStatistics(filter); return this.constructor.generateFormattedResponse({ @@ -223,6 +238,8 @@ class Capture { topCatchment, genderCount, approvalRates, + totalMatchedCaptures, + topMatchedCaptures, }); } From 4c2e0e08a9a756c4b18c3d46d6186241e0d03b12 Mon Sep 17 00:00:00 2001 From: Pranav Kakaraparti Date: Mon, 24 Jul 2023 20:26:58 -0500 Subject: [PATCH 4/7] feat: matched_captures schema validation update --- server/handlers/captureHandler.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/handlers/captureHandler.js b/server/handlers/captureHandler.js index 4e28097..c6573d1 100644 --- a/server/handlers/captureHandler.js +++ b/server/handlers/captureHandler.js @@ -65,6 +65,7 @@ const captureStatisticsGetCardQuerySchema = Joi.object({ 'species', 'captures', 'unverified_captures', + 'matched_captures', 'top_planters', 'trees_per_planters', 'catchments', From 97e3d5813b69882252aa208226de5a5061df66e0 Mon Sep 17 00:00:00 2001 From: Pranav Kakaraparti Date: Tue, 25 Jul 2023 03:44:25 -0500 Subject: [PATCH 5/7] feat: update capture_denormalized.spec.js --- __tests__/integration/capture_denormalized.spec.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/__tests__/integration/capture_denormalized.spec.js b/__tests__/integration/capture_denormalized.spec.js index e7015b9..d618006 100644 --- a/__tests__/integration/capture_denormalized.spec.js +++ b/__tests__/integration/capture_denormalized.spec.js @@ -364,6 +364,7 @@ describe('capture_denormalized', () => { 'species', 'captures', 'unverified_captures', + 'matched_captures', 'top_planters', 'trees_per_planters', 'last_updated_at', @@ -381,6 +382,7 @@ describe('capture_denormalized', () => { 'total', 'unverified_captures', ]); + expect(res.body.matched_captures).to.have.keys(['total', 'matched_captures']); expect(res.body.top_planters).to.have.keys([ 'average', 'top_planters', @@ -396,6 +398,7 @@ describe('capture_denormalized', () => { checkObjectProperties( res.body.unverified_captures.unverified_captures, ); + checkObjectProperties(res.body.matched_captures.matched_captures); checkObjectProperties(res.body.top_planters.top_planters); checkObjectProperties(res.body.trees_per_planters.trees_per_planters); checkObjectProperties(res.body.catchments.catchments); @@ -426,7 +429,7 @@ describe('capture_denormalized', () => { .end(function (err, res) { if (err) return done(err); expect(res.body.message).to.eql( - '"card_title" must be one of [planters, species, captures, unverified_captures, top_planters, trees_per_planters, catchments, gender_details, approval_rates]', + '"card_title" must be one of [planters, species, captures, unverified_captures, matched_captures, top_planters, trees_per_planters, catchments, gender_details, approval_rates]', ); return done(); }); From 63d70c6cb7385e805e297192344485c7361287f8 Mon Sep 17 00:00:00 2001 From: Pranav Kakaraparti Date: Tue, 25 Jul 2023 04:02:46 -0500 Subject: [PATCH 6/7] fix: eslint errors --- server/repositories/CaptureRepository.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/server/repositories/CaptureRepository.js b/server/repositories/CaptureRepository.js index f40f81e..289ee8e 100644 --- a/server/repositories/CaptureRepository.js +++ b/server/repositories/CaptureRepository.js @@ -2,8 +2,8 @@ const BaseRepository = require('./BaseRepository'); class CaptureRepository extends BaseRepository { constructor(session) { - super('capture_denormalized', session); - this._tableName = 'capture_denormalized'; + super('reporting.capture_denormalized', session); + this._tableName = 'reporting.capture_denormalized'; this._session = session; } @@ -55,6 +55,7 @@ class CaptureRepository extends BaseRepository { const whereBuilder = function (object, builder) { const result = builder; const filterObject = { ...object }; + console.log('FILTEROBJ', filterObject) delete filterObject.card_title; if (filterObject.capture_created_start_date) { result.where( @@ -131,7 +132,7 @@ class CaptureRepository extends BaseRepository { 'planter_last_name', 'planter_identifier', ) - .from('capture_denormalized') + .from('reporting.capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .as('planters'); }); @@ -147,7 +148,7 @@ class CaptureRepository extends BaseRepository { 'planting_organization_name', 'planting_organization_uuid', ) - .from('capture_denormalized') + .from('reporting.capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .groupBy( 'planting_organization_uuid', @@ -177,7 +178,7 @@ class CaptureRepository extends BaseRepository { .avg('totalPlanters') .from(function () { this.count('* as totalPlanters') - .from('capture_denormalized') + .from('reporting.capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .groupBy( 'planter_first_name', @@ -236,7 +237,7 @@ class CaptureRepository extends BaseRepository { `count(*) as count, planting_organization_name, planting_organization_uuid`, ), ) - .from('capture_denormalized') + .from('reporting.capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .groupBy( 'planting_organization_uuid', @@ -265,7 +266,7 @@ class CaptureRepository extends BaseRepository { `count(*) as count, planting_organization_name, planting_organization_uuid`, ), ) - .from('capture_denormalized') + .from('reporting.capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .groupBy( 'planting_organization_uuid', @@ -287,7 +288,7 @@ class CaptureRepository extends BaseRepository { .avg('totalCatchment') .from(function () { this.count('* as totalCatchment') - .from('capture_denormalized') + .from('reporting.capture_denormalized') .where((builder) => cachmentWhereBuilder(filter, builder)) .groupBy('catchment') .as('catchments'); @@ -312,7 +313,7 @@ class CaptureRepository extends BaseRepository { 'gender', ) .where((builder) => whereBuilder(filter, builder)) - .from('capture_denormalized') + .from('reporting.capture_denormalized') .as('planters'); }) .groupBy('gender') @@ -370,7 +371,7 @@ class CaptureRepository extends BaseRepository { .whereNotNull('tree_id') .where((builder) => whereBuilder({ ...filter, approved: true }, builder)); - //top matched captures (by organization) + // top matched captures (by organization) const topMatchedCapturesQuery = knex(this._tableName) .select(knex.raw('planting_organization_name, count(*) as count')) .whereNotNull('tree_id') @@ -403,6 +404,7 @@ class CaptureRepository extends BaseRepository { await topUnverifiedCapturesQuery.cache(); return { topUnverifiedCaptures }; } + case 'top_planters': { const topPlanters = await topPlantersQuery.cache(); return { topPlanters }; @@ -421,7 +423,6 @@ class CaptureRepository extends BaseRepository { return { approvalRates }; } case 'matched_captures': { - console.log('MATCHED CAPTURES IS CALLED') const topMatchedCaptures = await topMatchedCapturesQuery.cache(); return { topMatchedCaptures }; } From f184299e1ccf36ea29dc84aa3798d69251a88115 Mon Sep 17 00:00:00 2001 From: Pranav Kakaraparti Date: Tue, 25 Jul 2023 17:31:17 -0500 Subject: [PATCH 7/7] fix: removed table name prefix --- server/repositories/CaptureRepository.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/server/repositories/CaptureRepository.js b/server/repositories/CaptureRepository.js index 289ee8e..6a18dbd 100644 --- a/server/repositories/CaptureRepository.js +++ b/server/repositories/CaptureRepository.js @@ -2,8 +2,8 @@ const BaseRepository = require('./BaseRepository'); class CaptureRepository extends BaseRepository { constructor(session) { - super('reporting.capture_denormalized', session); - this._tableName = 'reporting.capture_denormalized'; + super('capture_denormalized', session); + this._tableName = 'capture_denormalized'; this._session = session; } @@ -55,7 +55,6 @@ class CaptureRepository extends BaseRepository { const whereBuilder = function (object, builder) { const result = builder; const filterObject = { ...object }; - console.log('FILTEROBJ', filterObject) delete filterObject.card_title; if (filterObject.capture_created_start_date) { result.where( @@ -132,7 +131,7 @@ class CaptureRepository extends BaseRepository { 'planter_last_name', 'planter_identifier', ) - .from('reporting.capture_denormalized') + .from('capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .as('planters'); }); @@ -148,7 +147,7 @@ class CaptureRepository extends BaseRepository { 'planting_organization_name', 'planting_organization_uuid', ) - .from('reporting.capture_denormalized') + .from('capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .groupBy( 'planting_organization_uuid', @@ -178,7 +177,7 @@ class CaptureRepository extends BaseRepository { .avg('totalPlanters') .from(function () { this.count('* as totalPlanters') - .from('reporting.capture_denormalized') + .from('capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .groupBy( 'planter_first_name', @@ -237,7 +236,7 @@ class CaptureRepository extends BaseRepository { `count(*) as count, planting_organization_name, planting_organization_uuid`, ), ) - .from('reporting.capture_denormalized') + .from('capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .groupBy( 'planting_organization_uuid', @@ -266,7 +265,7 @@ class CaptureRepository extends BaseRepository { `count(*) as count, planting_organization_name, planting_organization_uuid`, ), ) - .from('reporting.capture_denormalized') + .from('capture_denormalized') .where((builder) => whereBuilder(filter, builder)) .groupBy( 'planting_organization_uuid', @@ -288,7 +287,7 @@ class CaptureRepository extends BaseRepository { .avg('totalCatchment') .from(function () { this.count('* as totalCatchment') - .from('reporting.capture_denormalized') + .from('capture_denormalized') .where((builder) => cachmentWhereBuilder(filter, builder)) .groupBy('catchment') .as('catchments'); @@ -313,7 +312,7 @@ class CaptureRepository extends BaseRepository { 'gender', ) .where((builder) => whereBuilder(filter, builder)) - .from('reporting.capture_denormalized') + .from('capture_denormalized') .as('planters'); }) .groupBy('gender')