From 3ad8ff312007ef8ca237b9c16cefe9b2ba139e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Tue, 16 Nov 2021 22:30:26 +0000 Subject: [PATCH 01/10] add totalClaimed, totalRaised, totalBeneficiaries and totalManagers in ubiCommunityDailyState --- .../z1603886582-create-communityDailyState.js | 18 ++ .../migrations/z1636472208-update-triggers.js | 71 +++++ ...636638661-update-ubiCommunityDailyState.js | 35 +++ ...636638954-update-ubiCommunityDailyState.js | 64 +++++ ...637070248-update-ubiCommunityDailyState.js | 23 ++ src/database/migrations/zz-create-triggers.js | 118 ++++---- src/database/models/associations/community.ts | 7 + .../models/ubi/communityDailyState.ts | 22 ++ src/interfaces/ubi/ubiCommunityDailyState.ts | 8 + src/services/ubi/community.ts | 270 +++++++++++++++++- src/worker/jobs/cron/community.ts | 128 ++++++++- .../cron/calcuateCommunitiesMetrics.test.ts | 40 +++ 12 files changed, 729 insertions(+), 75 deletions(-) create mode 100644 src/database/migrations/z1636472208-update-triggers.js create mode 100644 src/database/migrations/z1636638661-update-ubiCommunityDailyState.js create mode 100644 src/database/migrations/z1636638954-update-ubiCommunityDailyState.js create mode 100644 src/database/migrations/z1637070248-update-ubiCommunityDailyState.js diff --git a/src/database/migrations/z1603886582-create-communityDailyState.js b/src/database/migrations/z1603886582-create-communityDailyState.js index 097154916..523fd393b 100644 --- a/src/database/migrations/z1603886582-create-communityDailyState.js +++ b/src/database/migrations/z1603886582-create-communityDailyState.js @@ -70,6 +70,24 @@ module.exports = { type: Sequelize.DATEONLY, allowNull: false, }, + totalClaimed: { + // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 + type: Sequelize.DECIMAL(27), // max 999,999,999 - plus 18 decimals + defaultValue: 0, + }, + totalRaised: { + // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 + type: Sequelize.DECIMAL(27), // max 999,999,999 - plus 18 decimals + defaultValue: 0, + }, + totalBeneficiaries: { + type: Sequelize.INTEGER, + defaultValue: 0, + }, + totalManagers: { + type: Sequelize.INTEGER, + defaultValue: 0, + }, }); }, down: (queryInterface) => { diff --git a/src/database/migrations/z1636472208-update-triggers.js b/src/database/migrations/z1636472208-update-triggers.js new file mode 100644 index 000000000..608b92f10 --- /dev/null +++ b/src/database/migrations/z1636472208-update-triggers.js @@ -0,0 +1,71 @@ +'use strict'; + +// eslint-disable-next-line no-undef +module.exports = { + async up(queryInterface, Sequelize) { + if (process.env.NODE_ENV === 'test') { + return; + } + + await queryInterface.sequelize.query(` + CREATE OR REPLACE FUNCTION update_inflow_community_states() + RETURNS TRIGGER AS $$ + declare + -- state_raised numeric(29); + -- state_daily_raised numeric(29); + n_backer bigint; + community_id integer; + BEGIN + SELECT id INTO community_id FROM community where "contractAddress"=NEW."contractAddress"; + + IF community_id is null THEN + return new; + end if; + -- if this address never donated, it's a new backer + SELECT count(*) INTO n_backer FROM inflow WHERE "from" = NEW."from" AND "contractAddress"=NEW."contractAddress"; + IF n_backer = 0 THEN + UPDATE ubi_community_state SET backers = backers + 1 WHERE "communityId"=community_id; + end if; + -- update total raised + -- SELECT SUM(raised + NEW.amount) INTO state_raised FROM ubi_community_state WHERE "communityId"=community_id; + -- UPDATE ubi_community_state SET raised = state_raised WHERE "communityId"=community_id; + -- SELECT SUM(raised + NEW.amount) INTO state_daily_raised FROM ubi_community_daily_state WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); + -- UPDATE ubi_community_daily_state SET raised = state_daily_raised WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); + return NEW; + END; +$$ LANGUAGE plpgsql;`); + + await queryInterface.sequelize.query(` + CREATE OR REPLACE FUNCTION update_claim_states() + RETURNS TRIGGER AS $$ + declare + -- state_claimed numeric(29); + -- state_daily_claimed numeric(29); + beneficiary_claimed numeric(22); + beneficiary_last_claim_at timestamp with time zone; + community_public_id uuid; + BEGIN + SELECT "publicId" INTO community_public_id FROM community where id=NEW."communityId"; + -- update claims + UPDATE ubi_community_state SET claims = claims + 1 WHERE "communityId"=NEW."communityId"; + -- UPDATE ubi_community_daily_state SET claims = claims + 1 WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); + -- update beneficiary table as well + SELECT "lastClaimAt" INTO beneficiary_last_claim_at FROM beneficiary WHERE "communityId"=community_public_id AND address=NEW.address; + UPDATE beneficiary SET claims = claims + 1, "penultimateClaimAt"=beneficiary_last_claim_at, "lastClaimAt"=NEW."txAt" WHERE "communityId"=community_public_id AND address=NEW.address; + SELECT SUM(claimed + NEW.amount) INTO beneficiary_claimed FROM beneficiary WHERE "communityId"=community_public_id AND address=NEW.address; + UPDATE beneficiary SET claimed = beneficiary_claimed WHERE "communityId"=community_public_id AND address=NEW.address; + -- update total claimed + -- SELECT SUM(claimed + NEW.amount) INTO state_claimed FROM ubi_community_state WHERE "communityId"=NEW."communityId"; + -- UPDATE ubi_community_state SET claimed = state_claimed WHERE "communityId"=NEW."communityId"; + -- SELECT SUM(claimed + NEW.amount) INTO state_daily_claimed FROM ubi_community_daily_state WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); + -- UPDATE ubi_community_daily_state SET claimed = state_daily_claimed WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); + return NEW; + END; +$$ LANGUAGE plpgsql;`); + + await queryInterface.sequelize.query(`DROP TRIGGER IF EXISTS update_managers_community_state ON manager`); + await queryInterface.sequelize.query(`DROP TRIGGER IF EXISTS update_beneficiaries_community_states ON beneficiary`); + }, + + down(queryInterface, Sequelize) {}, +}; \ No newline at end of file diff --git a/src/database/migrations/z1636638661-update-ubiCommunityDailyState.js b/src/database/migrations/z1636638661-update-ubiCommunityDailyState.js new file mode 100644 index 000000000..4434e7e67 --- /dev/null +++ b/src/database/migrations/z1636638661-update-ubiCommunityDailyState.js @@ -0,0 +1,35 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + if (process.env.NODE_ENV === 'test') { + return; + } + + await queryInterface.addColumn('ubi_community_daily_state', 'totalClaimed', { + type: Sequelize.DECIMAL(29), + allowNull: false, + defaultValue: 0 + }); + + await queryInterface.addColumn('ubi_community_daily_state', 'totalRaised', { + type: Sequelize.DECIMAL(29), + allowNull: false, + defaultValue: 0 + }); + + await queryInterface.addColumn('ubi_community_daily_state', 'totalBeneficiaries', { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: 0 + }); + + await queryInterface.addColumn('ubi_community_daily_state', 'totalManagers', { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: 0 + }); + }, + + down(queryInterface, Sequelize) {}, +}; diff --git a/src/database/migrations/z1636638954-update-ubiCommunityDailyState.js b/src/database/migrations/z1636638954-update-ubiCommunityDailyState.js new file mode 100644 index 000000000..62f4b8edc --- /dev/null +++ b/src/database/migrations/z1636638954-update-ubiCommunityDailyState.js @@ -0,0 +1,64 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + if (process.env.NODE_ENV === 'test') { + return; + } + + const communities = (await queryInterface.sequelize.query(` + select "communityId" + from ubi_community_daily_state + group by "communityId"; + `, { raw: true }))[0]; + + // update the first day of each community + for(let i = 0; i < communities.length; i++) { + //update totalClaimed, totalRaised, totalBeneficiaries + await queryInterface.sequelize.query(` + update ubi_community_daily_state + set "totalClaimed" = claimed, + "totalRaised" = raised, + "totalBeneficiaries" = beneficiaries + where date = (select date + from ubi_community_daily_state + where "communityId" = ${communities[i].communityId} + order by date + limit 1 + ) and "communityId" = ${communities[i].communityId}; + `); + } + + // update all others by day + const startDate = new Date('2020-09-22'); + startDate.setUTCHours(0, 0, 0, 0); + + const today = new Date(); + today.setUTCHours(0, 0, 0, 0); + + while (startDate <= today) { + const query = ` + update ubi_community_daily_state + set "totalClaimed" = previous."totalClaimed" + today.claimed, + "totalRaised" = previous."totalRaised" + today.raised, + "totalBeneficiaries" = previous."totalBeneficiaries" + today.beneficiaries + from ( + select distinct on ("communityId") "communityId", "totalClaimed", "totalRaised", "totalBeneficiaries" + from ubi_community_daily_state + where date < '${startDate.toISOString().split('T')[0]}' + order by "communityId", date DESC + ) as previous, + (select claimed, raised, beneficiaries, "communityId" from ubi_community_daily_state where date = '${startDate.toISOString().split('T')[0]}') as today + where ubi_community_daily_state."communityId" = previous."communityId" + and ubi_community_daily_state."communityId" = today."communityId" + and ubi_community_daily_state.date = '${startDate.toISOString().split('T')[0]}'; + `; + + await queryInterface.sequelize.query(query); + + startDate.setDate(startDate.getDate() + 1); + } + }, + + down(queryInterface, Sequelize) {}, +}; diff --git a/src/database/migrations/z1637070248-update-ubiCommunityDailyState.js b/src/database/migrations/z1637070248-update-ubiCommunityDailyState.js new file mode 100644 index 000000000..cf24d99a4 --- /dev/null +++ b/src/database/migrations/z1637070248-update-ubiCommunityDailyState.js @@ -0,0 +1,23 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + if (process.env.NODE_ENV === 'test') { + return; + } + + const lastState = (await queryInterface.sequelize.query( + `select date from ubi_community_daily_state order by date DESC limit 1`, + { raw: true, type: Sequelize.QueryTypes.SELECT } + ))[0]; + + await queryInterface.sequelize.query(` + update ubi_community_daily_state + set "totalManagers" = state.managers + from (select managers, "communityId" from ubi_community_state) as state + where date = '${lastState.date}' and ubi_community_daily_state."communityId" = state."communityId"; + `); + }, + + down(queryInterface, Sequelize) {}, +}; diff --git a/src/database/migrations/zz-create-triggers.js b/src/database/migrations/zz-create-triggers.js index a814f40bf..85b0be494 100644 --- a/src/database/migrations/zz-create-triggers.js +++ b/src/database/migrations/zz-create-triggers.js @@ -6,39 +6,38 @@ module.exports = { // use datagrip for better understanding + highlight // create trigger to update total beneficiaries ina given community, by day and since ever - await queryInterface.sequelize.query(` - CREATE OR REPLACE FUNCTION update_beneficiaries_community_states() - RETURNS TRIGGER AS $$ - declare - community_id integer; - BEGIN - SELECT id INTO community_id FROM community where "publicId"=NEW."communityId"; - IF (TG_OP = 'INSERT') THEN -- INSERT operations (beneficiary being added from community) - -- update overall state - UPDATE ubi_community_state SET beneficiaries = beneficiaries + 1 WHERE "communityId"=community_id; - -- update daily state - -- UPDATE ubi_community_daily_state SET beneficiaries = beneficiaries + 1 WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); - ELSEIF (OLD.active IS TRUE AND NEW.active IS FALSE) THEN -- beneficiary being removed from community - -- update overall state - UPDATE ubi_community_state SET beneficiaries = beneficiaries - 1, "removedBeneficiaries" = "removedBeneficiaries" + 1 WHERE "communityId"=community_id; - -- update daily state - -- UPDATE ubi_community_daily_state SET beneficiaries = beneficiaries - 1 WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); - END IF; - RETURN NEW; - END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER update_beneficiaries_community_states -BEFORE INSERT OR UPDATE -ON beneficiary -FOR EACH ROW -EXECUTE PROCEDURE update_beneficiaries_community_states();`); +// await queryInterface.sequelize.query(` +// CREATE OR REPLACE FUNCTION update_beneficiaries_community_states() +// RETURNS TRIGGER AS $$ +// declare +// community_id integer; +// BEGIN +// SELECT id INTO community_id FROM community where "publicId"=NEW."communityId"; +// IF (TG_OP = 'INSERT') THEN -- INSERT operations (beneficiary being added from community) +// -- update overall state +// UPDATE ubi_community_state SET beneficiaries = beneficiaries + 1 WHERE "communityId"=community_id; +// -- update daily state +// -- UPDATE ubi_community_daily_state SET beneficiaries = beneficiaries + 1 WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); +// ELSEIF (OLD.active IS TRUE AND NEW.active IS FALSE) THEN -- beneficiary being removed from community +// -- update overall state +// UPDATE ubi_community_state SET beneficiaries = beneficiaries - 1, "removedBeneficiaries" = "removedBeneficiaries" + 1 WHERE "communityId"=community_id; +// -- update daily state +// -- UPDATE ubi_community_daily_state SET beneficiaries = beneficiaries - 1 WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); +// END IF; +// RETURN NEW; +// END; +// $$ LANGUAGE plpgsql; +// CREATE TRIGGER update_beneficiaries_community_states +// BEFORE INSERT OR UPDATE +// ON beneficiary +// FOR EACH ROW +// EXECUTE PROCEDURE update_beneficiaries_community_states();`); await queryInterface.sequelize.query(` CREATE OR REPLACE FUNCTION update_claim_states() RETURNS TRIGGER AS $$ declare - state_claimed numeric(29); + -- state_claimed numeric(29); -- state_daily_claimed numeric(29); beneficiary_claimed numeric(22); beneficiary_last_claim_at timestamp with time zone; @@ -54,50 +53,48 @@ EXECUTE PROCEDURE update_beneficiaries_community_states();`); SELECT SUM(claimed + NEW.amount) INTO beneficiary_claimed FROM beneficiary WHERE "communityId"=community_public_id AND address=NEW.address; UPDATE beneficiary SET claimed = beneficiary_claimed WHERE "communityId"=community_public_id AND address=NEW.address; -- update total claimed - SELECT SUM(claimed + NEW.amount) INTO state_claimed FROM ubi_community_state WHERE "communityId"=NEW."communityId"; - UPDATE ubi_community_state SET claimed = state_claimed WHERE "communityId"=NEW."communityId"; + -- SELECT SUM(claimed + NEW.amount) INTO state_claimed FROM ubi_community_state WHERE "communityId"=NEW."communityId"; + -- UPDATE ubi_community_state SET claimed = state_claimed WHERE "communityId"=NEW."communityId"; -- SELECT SUM(claimed + NEW.amount) INTO state_daily_claimed FROM ubi_community_daily_state WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); -- UPDATE ubi_community_daily_state SET claimed = state_daily_claimed WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); return NEW; END; $$ LANGUAGE plpgsql; - CREATE TRIGGER update_claim_states BEFORE INSERT ON ubi_claim FOR EACH ROW EXECUTE PROCEDURE update_claim_states();`); - await queryInterface.sequelize.query(` - CREATE OR REPLACE FUNCTION update_managers_community_state() - RETURNS TRIGGER AS -$$ -declare - community_id integer; -BEGIN - SELECT id INTO community_id FROM community where "publicId" = NEW."communityId"; - IF (TG_OP = 'INSERT') THEN -- INSERT operations - -- update overall state - UPDATE ubi_community_state SET managers = managers + 1 WHERE "communityId" = community_id; - ELSEIF (OLD.active IS TRUE AND NEW.active IS FALSE) THEN -- manager being removed from community - -- update overall state - UPDATE ubi_community_state SET managers = managers - 1 WHERE "communityId" = community_id; - END IF; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER update_managers_community_state -BEFORE INSERT OR UPDATE -ON manager -FOR EACH ROW -EXECUTE PROCEDURE update_managers_community_state();`); +// await queryInterface.sequelize.query(` +// CREATE OR REPLACE FUNCTION update_managers_community_state() +// RETURNS TRIGGER AS +// $$ +// declare +// community_id integer; +// BEGIN +// SELECT id INTO community_id FROM community where "publicId" = NEW."communityId"; +// IF (TG_OP = 'INSERT') THEN -- INSERT operations +// -- update overall state +// UPDATE ubi_community_state SET managers = managers + 1 WHERE "communityId" = community_id; +// ELSEIF (OLD.active IS TRUE AND NEW.active IS FALSE) THEN -- manager being removed from community +// -- update overall state +// UPDATE ubi_community_state SET managers = managers - 1 WHERE "communityId" = community_id; +// END IF; +// RETURN NEW; +// END; +// $$ LANGUAGE plpgsql; +// CREATE TRIGGER update_managers_community_state +// BEFORE INSERT OR UPDATE +// ON manager +// FOR EACH ROW +// EXECUTE PROCEDURE update_managers_community_state();`); await queryInterface.sequelize.query(` CREATE OR REPLACE FUNCTION update_inflow_community_states() RETURNS TRIGGER AS $$ declare - state_raised numeric(29); + -- state_raised numeric(29); -- state_daily_raised numeric(29); n_backer bigint; community_id integer; @@ -107,21 +104,19 @@ EXECUTE PROCEDURE update_managers_community_state();`); IF community_id is null THEN return new; end if; - -- if this address never donated, it's a new backer SELECT count(*) INTO n_backer FROM inflow WHERE "from" = NEW."from" AND "contractAddress"=NEW."contractAddress"; IF n_backer = 0 THEN UPDATE ubi_community_state SET backers = backers + 1 WHERE "communityId"=community_id; end if; -- update total raised - SELECT SUM(raised + NEW.amount) INTO state_raised FROM ubi_community_state WHERE "communityId"=community_id; - UPDATE ubi_community_state SET raised = state_raised WHERE "communityId"=community_id; + -- SELECT SUM(raised + NEW.amount) INTO state_raised FROM ubi_community_state WHERE "communityId"=community_id; + -- UPDATE ubi_community_state SET raised = state_raised WHERE "communityId"=community_id; -- SELECT SUM(raised + NEW.amount) INTO state_daily_raised FROM ubi_community_daily_state WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); -- UPDATE ubi_community_daily_state SET raised = state_daily_raised WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); return NEW; END; $$ LANGUAGE plpgsql; - CREATE TRIGGER update_inflow_community_states BEFORE INSERT ON inflow @@ -141,7 +136,6 @@ EXECUTE PROCEDURE update_inflow_community_states();`); END IF; END; $$ LANGUAGE plpgsql; - CREATE TRIGGER update_loves_stories BEFORE INSERT OR DELETE ON story_user_engagement @@ -150,4 +144,4 @@ EXECUTE PROCEDURE update_loves_stories();`); }, down(queryInterface, Sequelize) {}, -}; +}; \ No newline at end of file diff --git a/src/database/models/associations/community.ts b/src/database/models/associations/community.ts index 1df743df8..07dffa4dd 100644 --- a/src/database/models/associations/community.ts +++ b/src/database/models/associations/community.ts @@ -62,6 +62,13 @@ export function communityAssociation(sequelize: Sequelize) { as: 'beneficiaries', }); + // used to query from the community with incude + sequelize.models.Community.hasMany(sequelize.models.Manager, { + foreignKey: 'communityId', + sourceKey: 'publicId', + as: 'managers', + }); + // used to query from the promoter with incude sequelize.models.UbiPromoterModel.belongsToMany( sequelize.models.Community, diff --git a/src/database/models/ubi/communityDailyState.ts b/src/database/models/ubi/communityDailyState.ts index 6ba3d31e7..11ec2587f 100644 --- a/src/database/models/ubi/communityDailyState.ts +++ b/src/database/models/ubi/communityDailyState.ts @@ -22,6 +22,10 @@ export class UbiCommunityDailyStateModel extends Model< public reachOut!: number; public fundingRate!: number; public date!: Date; + public totalClaimed!: string; + public totalRaised!: string; + public totalBeneficiaries!: number; + public totalManagers!: number; } export function initializeUbiCommunityDailyState(sequelize: Sequelize): void { @@ -93,6 +97,24 @@ export function initializeUbiCommunityDailyState(sequelize: Sequelize): void { type: DataTypes.DATEONLY, allowNull: false, }, + totalClaimed: { + // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 + type: DataTypes.DECIMAL(29), // max 99,999,999,999 - plus 18 decimals + defaultValue: 0, + }, + totalRaised: { + // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 + type: DataTypes.DECIMAL(29), // max 99,999,999,999 - plus 18 decimals + defaultValue: 0, + }, + totalBeneficiaries: { + type: DataTypes.INTEGER, + defaultValue: 0, + }, + totalManagers: { + type: DataTypes.INTEGER, + defaultValue: 0, + }, }, { tableName: 'ubi_community_daily_state', diff --git a/src/interfaces/ubi/ubiCommunityDailyState.ts b/src/interfaces/ubi/ubiCommunityDailyState.ts index daf375df7..8e6816ff0 100644 --- a/src/interfaces/ubi/ubiCommunityDailyState.ts +++ b/src/interfaces/ubi/ubiCommunityDailyState.ts @@ -13,6 +13,10 @@ export interface UbiCommunityDailyState { reachOut: number; fundingRate: number; date: Date; + totalClaimed: string; + totalRaised: string; + totalBeneficiaries: number; + totalManagers: number; } export interface UbiCommunityDailyStateCreation { communityId: number; @@ -28,4 +32,8 @@ export interface UbiCommunityDailyStateCreation { reachOut: number; fundingRate: number; date: Date; + totalClaimed: string; + totalRaised: string; + totalBeneficiaries: number; + totalManagers: number; } diff --git a/src/services/ubi/community.ts b/src/services/ubi/community.ts index d1284a18c..1cb602521 100644 --- a/src/services/ubi/community.ts +++ b/src/services/ubi/community.ts @@ -17,6 +17,7 @@ import { ManagerAttributes } from '@models/ubi/manager'; import { BaseError } from '@utils/baseError'; import { fetchData } from '@utils/dataFetching'; import { notifyManagerAdded } from '@utils/util'; +import BigNumber from 'bignumber.js'; import { ethers } from 'ethers'; import { Op, @@ -27,6 +28,7 @@ import { OrderItem, WhereOptions, Includeable, + where, } from 'sequelize'; import { Literal } from 'sequelize/types/lib/utils'; @@ -41,6 +43,7 @@ import { ICommunityPendingDetails, IManagerDetailsManager, } from '../../types/endpoints'; +import { calcuateCommunitiesMetrics } from '../../worker/jobs/cron/community'; import { CommunityContentStorage, PromoterContentStorage } from '../storage'; import CommunityContractService from './communityContract'; import CommunityStateService from './communityState'; @@ -65,6 +68,8 @@ export default class CommunityService { public static appMediaContent = models.appMediaContent; public static appMediaThumbnail = models.appMediaThumbnail; public static ubiBeneficiaryRegistry = models.ubiBeneficiaryRegistry; + public static beneficiary = models.beneficiary; + public static ubiClaim = models.ubiClaim; public static sequelize = sequelize; private static communityContentStorage = new CommunityContentStorage(); @@ -429,11 +434,11 @@ export default class CommunityService { if (query.fields) { const fields = fetchData(query.fields); include = this._generateInclude(fields); - attributes = fields.root + attributes = fields.root ? fields.root.length > 0 ? fields.root.filter((el: string) => !exclude.includes(el)) : { exclude } - : [] + : []; } else { include = this._oldInclude(query.extended); attributes = { @@ -727,12 +732,257 @@ export default class CommunityService { } public static async getState(communityId: number) { + const yesterdayDateOnly = new Date(); + yesterdayDateOnly.setUTCHours(0, 0, 0, 0); + yesterdayDateOnly.setDate(yesterdayDateOnly.getDate() - 1); + const yesterdayDate = yesterdayDateOnly.toISOString().split('T')[0]; + // calcuateCommunitiesMetrics() + const todayDateOnly = new Date(); + todayDateOnly.setUTCHours(0, 0, 0, 0); + const todayDate = todayDateOnly.toISOString().split('T')[0]; + const result = await this.ubiCommunityState.findOne({ where: { communityId, }, }); - return result !== null ? (result.toJSON() as UbiCommunityState) : null; + + const previousDate = await this.ubiCommunityDailyState.findOne({ + attributes: [ + 'totalClaimed', + 'totalRaised', + 'totalBeneficiaries', + 'totalManagers', + ], + where: { + communityId, + date: yesterdayDate, + }, + }); + + const communityClaimActivity: { + totalClaimed: string; + } = (await this.community.findOne({ + attributes: [ + [ + fn( + 'coalesce', + fn('sum', col('beneficiaries->claim.amount')), + '0' + ), + 'totalClaimed', + ], + ], + include: [ + { + model: this.beneficiary, + as: 'beneficiaries', + attributes: [], + required: false, + include: [ + { + model: this.ubiClaim, + as: 'claim', + attributes: [], + required: false, + where: { + txAt: where( + fn( + 'date', + col('beneficiaries->claim.txAt') + ), + '=', + todayDate + ), + }, + }, + ], + }, + ], + where: { + id: communityId, + }, + group: ['Community.id'], + raw: true, + })) as any; + + const communityInflowActivity: { + totalRaised: string; + } = (await models.community.findOne({ + attributes: [ + [ + fn('sum', fn('coalesce', col('inflow.amount'), 0)), + 'totalRaised', + ], + ], + include: [ + { + model: models.inflow, + as: 'inflow', + attributes: [], + required: false, + where: { + txAt: where( + fn('date', col('inflow."txAt"')), + '=', + todayDate + ), + }, + }, + ], + where: { + id: communityId, + }, + group: ['Community.id'], + raw: true, + })) as any; + + const communityNewBeneficiaryActivity: { + beneficiaries: string; + } = (await models.community.findOne({ + attributes: [ + [fn('count', col('beneficiaries.address')), 'beneficiaries'], + ], + include: [ + { + model: models.beneficiary, + as: 'beneficiaries', + attributes: [], + required: false, + where: { + txAt: where( + fn('date', col('beneficiaries."txAt"')), + '=', + todayDate + ), + }, + }, + ], + where: { + id: communityId, + }, + group: ['Community.id'], + raw: true, + })) as any; + + const communityRemovedBeneficiaryActivity: { + beneficiaries: string; + } = (await models.community.findOne({ + attributes: [ + [fn('count', col('beneficiaries.address')), 'beneficiaries'], + ], + include: [ + { + model: models.beneficiary, + as: 'beneficiaries', + attributes: [], + required: false, + where: { + updatedAt: where( + fn('date', col('beneficiaries."updatedAt"')), + '=', + todayDate + ), + active: false, + }, + }, + ], + where: { + id: communityId, + }, + group: ['Community.id'], + raw: true, + })) as any; + + const communityNewManagerActivity: { + managers: string; + } = (await models.community.findOne({ + attributes: [[fn('count', col('managers.address')), 'managers']], + include: [ + { + model: models.manager, + as: 'managers', + attributes: [], + required: false, + where: { + createdAt: where( + fn('date', col('managers."createdAt"')), + '=', + todayDate + ), + }, + }, + ], + where: { + id: communityId, + }, + group: ['Community.id'], + raw: true, + })) as any; + + const communityRemovedManagerActivity: { + managers: string; + } = (await models.community.findOne({ + attributes: [[fn('count', col('managers.address')), 'managers']], + include: [ + { + model: models.manager, + as: 'managers', + attributes: [], + required: false, + where: { + updatedAt: where( + fn('date', col('managers."updatedAt"')), + '=', + todayDate + ), + active: false, + }, + }, + ], + where: { + id: communityId, + }, + group: ['Community.id'], + raw: true, + })) as any; + + if (result === null) return null; + + const previousClaims = previousDate?.totalClaimed + ? new BigNumber(previousDate.totalClaimed) + : new BigNumber(0); + const todayClaims = communityClaimActivity?.totalClaimed + ? new BigNumber(communityClaimActivity.totalClaimed) + : new BigNumber(0); + const previousRaise = previousDate?.totalRaised + ? new BigNumber(previousDate.totalRaised) + : new BigNumber(0); + const todayRaise = communityInflowActivity?.totalRaised + ? new BigNumber(communityInflowActivity.totalRaised) + : new BigNumber(0); + + const previousBeneficiaries = previousDate?.totalBeneficiaries + ? previousDate.totalBeneficiaries + : 0; + const todayBeneficiaries = + parseInt(communityNewBeneficiaryActivity.beneficiaries) - + parseInt(communityRemovedBeneficiaryActivity.beneficiaries); + + const previousManagers = previousDate?.totalManagers + ? previousDate.totalManagers + : 0; + const todayManagers = + parseInt(communityNewManagerActivity.managers) - + parseInt(communityRemovedManagerActivity.managers); + + return { + ...(result.toJSON() as UbiCommunityState), + claimed: previousClaims.plus(todayClaims).toString(), + raised: previousRaise.plus(todayRaise).toString(), + beneficiaries: previousBeneficiaries + todayBeneficiaries, + manager: previousManagers + todayManagers, + }; } public static async getCampaign(communityId: number) { @@ -1576,13 +1826,13 @@ export default class CommunityService { as: 'cover', duplicating: false, include: [ - { - attributes: ['url', 'width', 'height', 'pixelRatio'], - model: this.appMediaThumbnail, - as: 'thumbnails', - separate: true, - }, - ], + { + attributes: ['url', 'width', 'height', 'pixelRatio'], + model: this.appMediaThumbnail, + as: 'thumbnails', + separate: true, + }, + ], }); } diff --git a/src/worker/jobs/cron/community.ts b/src/worker/jobs/cron/community.ts index 3969fa87a..ba2dbb551 100644 --- a/src/worker/jobs/cron/community.ts +++ b/src/worker/jobs/cron/community.ts @@ -52,6 +52,11 @@ export async function calcuateCommunitiesMetrics(): Promise { reachOut: string; fundingRate: string; beneficiaries: number; + managers: number; + totalClaimed: string; + totalRaised: string; + totalBeneficiaries: number; + totalManagers: number; }; }; @@ -90,9 +95,9 @@ export async function calcuateCommunitiesMetrics(): Promise { { model: models.ubiCommunityState, as: 'state', - where: { - raised: { [Op.ne]: 0 }, - }, + // where: { + // raised: { [Op.ne]: 0 }, + // }, }, { model: models.ubiCommunityDailyMetrics, @@ -263,6 +268,57 @@ export async function calcuateCommunitiesMetrics(): Promise { raw: true, })) as any; + const communityNewManagerActivity: { + id: string; + managers: string; + }[] = (await models.community.findAll({ + attributes: ['id', [fn('count', col('managers.address')), 'managers']], + include: [ + { + model: models.manager, + as: 'managers', + attributes: [], + required: false, + where: { + createdAt: { [Op.between]: [yesterday, today] }, + }, + }, + ], + where: { + ...whereCommunity, + visibility: 'public', + } as any, + group: ['Community.id'], + order: [['id', 'DESC']], + raw: true, + })) as any; + + const communityRemovedManagerActivity: { + id: string; + managers: string; + }[] = (await models.community.findAll({ + attributes: ['id', [fn('count', col('managers.address')), 'managers']], + include: [ + { + model: models.manager, + as: 'managers', + attributes: [], + required: false, + where: { + updatedAt: { [Op.between]: [yesterday, today] }, + active: false, + }, + }, + ], + where: { + ...whereCommunity, + visibility: 'public', + }, + group: ['Community.id'], + order: [['id', 'DESC']], + raw: true, + })) as any; + const communityInflowActivity: { id: string; raised: string; @@ -441,6 +497,27 @@ export async function calcuateCommunitiesMetrics(): Promise { raw: true, })) as any; + const previousState: { + communityId: number; + totalClaimed: string; + totalRaised: string; + totalBeneficiaries: number; + totalManagers: number; + }[] = await models.ubiCommunityDailyState.findAll({ + attributes: [ + [ + literal('DISTINCT ON ("communityId") "communityId"'), + 'communityId', + ], + 'totalClaimed', + 'totalRaised', + 'totalBeneficiaries', + 'totalManagers', + ], + order: ['communityId', ['date', 'DESC']], + raw: true, + }); + // build communities object const communitiesState = communitiesStatePre.map( (c) => c.toJSON() as CommunityAttributes @@ -471,6 +548,15 @@ export async function calcuateCommunitiesMetrics(): Promise { const crb = communityRemovedBeneficiaryActivity.find( (c) => parseInt(c.id, 10) === communitiesState[index].id ); + const cnm = communityNewManagerActivity.find( + (c) => parseInt(c.id, 10) === communitiesState[index].id + ); + const crm = communityRemovedManagerActivity.find( + (c) => parseInt(c.id, 10) === communitiesState[index].id + ); + const pcm = previousState.find( + (c) => c.communityId === communitiesState[index].id + ); communities.push({ ...communitiesState[index], beneficiariesClaiming: cn @@ -486,6 +572,14 @@ export async function calcuateCommunitiesMetrics(): Promise { ...(cea ? cea : { volume: '0', txs: '0', reach: '0', reachOut: '0' }), + ...(pcm + ? pcm + : { + totalClaimed: '0', + totalRaised: '0', + totalBeneficiaries: 0, + totalManagers: 0, + }), fundingRate: cm ? new BigNumber(cm.raised) .minus(cm.claimed) @@ -500,6 +594,13 @@ export async function calcuateCommunitiesMetrics(): Promise { parseInt(crb.beneficiaries, 10), } : { beneficiaries: 0 }), + ...(cnm && crm + ? { + managers: + parseInt(cnm.managers, 10) - + parseInt(crm.managers, 10), + } + : { managers: 0 }), }, }); } @@ -507,6 +608,17 @@ export async function calcuateCommunitiesMetrics(): Promise { const calculateMetrics = async (community: ICommunityToMetrics) => { // if (community.activity !== undefined) { + const totalClaimed = community.activity.totalClaimed + ? new BigNumber(community.activity.totalClaimed).plus( + new BigNumber(community.activity.claimed) + ) + : '0'; + const totalRaised = community.activity.totalRaised + ? new BigNumber(community.activity.totalRaised).plus( + new BigNumber(community.activity.raised) + ) + : '0'; + await models.ubiCommunityDailyState.create({ transactions: parseInt(community.activity.txs, 10), reach: parseInt(community.activity.reach, 10), @@ -521,6 +633,16 @@ export async function calcuateCommunitiesMetrics(): Promise { beneficiaries: community.activity.beneficiaries, communityId: community.id, date: yesterday, + totalClaimed: totalClaimed.toString(), + totalRaised: totalRaised.toString(), + totalBeneficiaries: community.activity.totalBeneficiaries + ? community.activity.totalBeneficiaries + + community.activity.beneficiaries + : 0, + totalManagers: community.activity.totalManagers + ? community.activity.totalManagers + + community.activity.managers + : 0, }); } // if no activity, do not calculate diff --git a/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts b/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts index dea05c2d6..d23c55497 100644 --- a/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts +++ b/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts @@ -146,6 +146,10 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 0, + totalClaimed: '4000000000000000000', + totalRaised: '15000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); @@ -215,6 +219,10 @@ describe('calcuateCommunitiesMetrics', () => { claims: 0, fundingRate: 80, beneficiaries: 0, + totalClaimed: '4000000000000000000', + totalRaised: '15000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); @@ -298,6 +306,10 @@ describe('calcuateCommunitiesMetrics', () => { claims: 2, fundingRate: 80, beneficiaries: 0, + totalClaimed: '6000000000000000000', + totalRaised: '20000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); @@ -348,6 +360,10 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 0, + totalClaimed: '6000000000000000000', + totalRaised: '15000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); @@ -379,6 +395,10 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 2, + totalClaimed: '0', + totalRaised: '10000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); @@ -409,6 +429,10 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 2, + totalClaimed: '0', + totalRaised: '5000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); @@ -446,6 +470,10 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 2, + totalClaimed: '2000000000000000000', + totalRaised: '10000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); @@ -481,6 +509,10 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 2, + totalClaimed: '1000000000000000000', + totalRaised: '10000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); @@ -510,6 +542,10 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 0, + totalClaimed: '0', + totalRaised: '5000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); }); @@ -627,6 +663,10 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: -2, communityId: communities[0].id, date: match.any, + totalClaimed: '6000000000000000000', + totalRaised: '15000000000000000000', + totalBeneficiaries: 0, + totalManagers: 0, }); }); }); From 71f1592c8570a46b804cfeba540eb0707753cd25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Fri, 19 Nov 2021 14:30:49 +0000 Subject: [PATCH 02/10] fix calculateCommunitiesMetrics tests --- src/services/ubi/community.ts | 101 ++++++++++++------ src/worker/jobs/cron/community.ts | 19 ++-- .../cron/calcuateCommunitiesMetrics.test.ts | 18 ++-- tests/integration/services/community.test.ts | 39 ++++++- 4 files changed, 123 insertions(+), 54 deletions(-) diff --git a/src/services/ubi/community.ts b/src/services/ubi/community.ts index 1cb602521..1c0fd2d82 100644 --- a/src/services/ubi/community.ts +++ b/src/services/ubi/community.ts @@ -322,7 +322,7 @@ export default class CommunityService { }): Promise<{ count: number; rows: CommunityAttributes[] }> { let extendedWhere: WhereOptions = {}; const orderOption: OrderItem[] = []; - + if (query.orderBy) { const orders = query.orderBy.split(';'); @@ -367,13 +367,13 @@ export default class CommunityService { // this requires extended query.extended = 'true'; extendedWhere = { - '$state.beneficiaries$': { + '$"dailyState"."totalBeneficiaries"$': { [Op.not]: 0, }, } as any; orderOption.push([ literal( - '(state.raised - state.claimed) / metrics."ubiRate" / state.beneficiaries' + '("dailyState"."totalRaised" - "dailyState"."totalClaimed") / metrics."ubiRate" / "dailyState"."totalBeneficiaries"' ), orderType ? orderType : 'ASC', ]); @@ -387,14 +387,14 @@ export default class CommunityService { break; default: orderOption.push([ - literal('state.beneficiaries'), + literal('"dailyState"."totalBeneficiaries"'), orderType ? orderType : 'DESC', ]); break; } }); } else { - orderOption.push([literal('state.beneficiaries'), 'DESC']); + orderOption.push([literal('"dailyState"."totalBeneficiaries"'), 'DESC']); } if (query.filter === 'featured') { @@ -433,7 +433,7 @@ export default class CommunityService { const exclude = ['email']; if (query.fields) { const fields = fetchData(query.fields); - include = this._generateInclude(fields); + include = this._generateInclude(fields, query.extended); attributes = fields.root ? fields.root.length > 0 ? fields.root.filter((el: string) => !exclude.includes(el)) @@ -736,7 +736,6 @@ export default class CommunityService { yesterdayDateOnly.setUTCHours(0, 0, 0, 0); yesterdayDateOnly.setDate(yesterdayDateOnly.getDate() - 1); const yesterdayDate = yesterdayDateOnly.toISOString().split('T')[0]; - // calcuateCommunitiesMetrics() const todayDateOnly = new Date(); todayDateOnly.setUTCHours(0, 0, 0, 0); const todayDate = todayDateOnly.toISOString().split('T')[0]; @@ -1799,7 +1798,7 @@ export default class CommunityService { }; } - private static _generateInclude(fields: any): Includeable[] { + private static _generateInclude(fields: any, extended?: string): Includeable[] { const extendedInclude: Includeable[] = []; if (fields.suspect) { extendedInclude.push({ @@ -1845,10 +1844,51 @@ export default class CommunityService { }); } - if (fields.metrics) { + const stateExclude = ['id', 'communityId']; + + const yesterdayDateOnly = new Date(); + yesterdayDateOnly.setUTCHours(0, 0, 0, 0); + yesterdayDateOnly.setDate(yesterdayDateOnly.getDate() - 1); + const yesterdayDate = yesterdayDateOnly.toISOString().split('T')[0]; + + const dailyState = fields.dailyState + ? fields.dailyState.length > 0 + ? fields.dailyState + : { exclude: stateExclude } + : []; + extendedInclude.push({ + attributes: dailyState, + model: this.ubiCommunityDailyState, + as: 'dailyState', + duplicating: false, + where: { + date: yesterdayDate + }, + required: false, + }); + + const stateAttributes = fields.state + ? fields.state.length > 0 + ? fields.state.filter( + (el: string) => !stateExclude.includes(el) + ) + : { exclude: stateExclude } + : []; + extendedInclude.push({ + model: this.ubiCommunityState, + attributes: stateAttributes, + as: 'state', + }); + + if (extended || fields.metrics) { + const metricsAttributes = fields.metrics + ? fields.metrics.length > 0 + ? fields.metrics + : undefined + : []; + extendedInclude.push({ - attributes: - fields.metrics.length > 0 ? fields.metrics : undefined, + attributes: metricsAttributes, model: this.ubiCommunityDailyMetrics, required: false, duplicating: false, @@ -1863,25 +1903,15 @@ export default class CommunityService { }); } - const stateExclude = ['id', 'communityId']; - const stateAttributes = fields.state - ? fields.state.length > 0 - ? fields.state.filter( - (el: string) => !stateExclude.includes(el) - ) - : { exclude: stateExclude } - : []; - extendedInclude.push({ - model: this.ubiCommunityState, - attributes: stateAttributes, - as: 'state', - }); - return extendedInclude; } private static _oldInclude(extended?: string): Includeable[] { const extendedInclude: Includeable[] = []; + const yesterdayDateOnly = new Date(); + yesterdayDateOnly.setUTCHours(0, 0, 0, 0); + yesterdayDateOnly.setDate(yesterdayDateOnly.getDate() - 1); + const yesterdayDate = yesterdayDateOnly.toISOString().split('T')[0]; // TODO: deprecated in mobile@1.1.6 if (extended) { @@ -1921,11 +1951,6 @@ export default class CommunityService { }, }, }, - { - model: this.ubiCommunityState, - attributes: { exclude: ['id', 'communityId'] }, - as: 'state', - }, { model: this.appMediaContent, as: 'cover', @@ -1938,6 +1963,22 @@ export default class CommunityService { }, ], }, + { + attributes: { exclude: ['id', 'communityId'] }, + model: this.ubiCommunityDailyState, + as: 'dailyState', + duplicating: false, + where: { + date: yesterdayDate + }, + required: false, + }, + // TODO: deprecated, use dailyState + { + model: this.ubiCommunityState, + attributes: { exclude: ['id', 'communityId'] }, + as: 'state', + }, ...extendedInclude, ] as Includeable[]; } diff --git a/src/worker/jobs/cron/community.ts b/src/worker/jobs/cron/community.ts index ba2dbb551..85f61693c 100644 --- a/src/worker/jobs/cron/community.ts +++ b/src/worker/jobs/cron/community.ts @@ -95,9 +95,12 @@ export async function calcuateCommunitiesMetrics(): Promise { { model: models.ubiCommunityState, as: 'state', - // where: { - // raised: { [Op.ne]: 0 }, - // }, + }, + { + attributes: [], + model: models.inflow, + as: 'inflow', + required: true, }, { model: models.ubiCommunityDailyMetrics, @@ -635,14 +638,8 @@ export async function calcuateCommunitiesMetrics(): Promise { date: yesterday, totalClaimed: totalClaimed.toString(), totalRaised: totalRaised.toString(), - totalBeneficiaries: community.activity.totalBeneficiaries - ? community.activity.totalBeneficiaries + - community.activity.beneficiaries - : 0, - totalManagers: community.activity.totalManagers - ? community.activity.totalManagers + - community.activity.managers - : 0, + totalBeneficiaries: community.activity.totalBeneficiaries + community.activity.beneficiaries, + totalManagers: community.activity.totalManagers + community.activity.managers, }); } // if no activity, do not calculate diff --git a/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts b/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts index d23c55497..ece48820a 100644 --- a/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts +++ b/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts @@ -148,7 +148,7 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: 0, totalClaimed: '4000000000000000000', totalRaised: '15000000000000000000', - totalBeneficiaries: 0, + totalBeneficiaries: 2, totalManagers: 0, }); }); @@ -221,7 +221,7 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: 0, totalClaimed: '4000000000000000000', totalRaised: '15000000000000000000', - totalBeneficiaries: 0, + totalBeneficiaries: 2, totalManagers: 0, }); }); @@ -308,7 +308,7 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: 0, totalClaimed: '6000000000000000000', totalRaised: '20000000000000000000', - totalBeneficiaries: 0, + totalBeneficiaries: 2, totalManagers: 0, }); }); @@ -362,7 +362,7 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: 0, totalClaimed: '6000000000000000000', totalRaised: '15000000000000000000', - totalBeneficiaries: 0, + totalBeneficiaries: 2, totalManagers: 0, }); }); @@ -397,7 +397,7 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: 2, totalClaimed: '0', totalRaised: '10000000000000000000', - totalBeneficiaries: 0, + totalBeneficiaries: 2, totalManagers: 0, }); }); @@ -431,7 +431,7 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: 2, totalClaimed: '0', totalRaised: '5000000000000000000', - totalBeneficiaries: 0, + totalBeneficiaries: 2, totalManagers: 0, }); }); @@ -472,7 +472,7 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: 2, totalClaimed: '2000000000000000000', totalRaised: '10000000000000000000', - totalBeneficiaries: 0, + totalBeneficiaries: 2, totalManagers: 0, }); }); @@ -511,7 +511,7 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: 2, totalClaimed: '1000000000000000000', totalRaised: '10000000000000000000', - totalBeneficiaries: 0, + totalBeneficiaries: 2, totalManagers: 0, }); }); @@ -665,7 +665,7 @@ describe('calcuateCommunitiesMetrics', () => { date: match.any, totalClaimed: '6000000000000000000', totalRaised: '15000000000000000000', - totalBeneficiaries: 0, + totalBeneficiaries: 2, totalManagers: 0, }); }); diff --git a/tests/integration/services/community.test.ts b/tests/integration/services/community.test.ts index cdbd73089..61a6e132e 100644 --- a/tests/integration/services/community.test.ts +++ b/tests/integration/services/community.test.ts @@ -2,6 +2,7 @@ import { expect } from 'chai'; import faker from 'faker'; import { Sequelize } from 'sequelize'; import Sinon, { assert, replace, spy } from 'sinon'; +import tk from 'timekeeper'; import { models } from '../../../src/database'; import { AppMediaContent } from '../../../src/interfaces/app/appMediaContent'; @@ -11,13 +12,16 @@ import { CommunityContentStorage } from '../../../src/services/storage'; import BeneficiaryService from '../../../src/services/ubi/beneficiary'; import CommunityService from '../../../src/services/ubi/community'; import ManagerService from '../../../src/services/ubi/managers'; +import { calcuateCommunitiesMetrics } from '../../../src/worker/jobs/cron/community'; import { verifyDeletedAccounts } from '../../../src/worker/jobs/cron/user'; import BeneficiaryFactory from '../../factories/beneficiary'; import CommunityFactory from '../../factories/community'; import ManagerFactory from '../../factories/manager'; import UserFactory from '../../factories/user'; import truncate, { sequelizeSetup } from '../../utils/sequelizeSetup'; -import { randomTx } from '../../utils/utils'; +import { randomTx, jumpToTomorrowMidnight } from '../../utils/utils'; +import InflowFactory from '../../factories/inflow'; +import ClaimFactory from '../../factories/claim'; // in this test there are users being assined with suspicious activity and others being removed describe('community service', () => { @@ -605,7 +609,8 @@ describe('community service', () => { ]); for (const community of communities) { - await BeneficiaryFactory( + await InflowFactory(community); + const beneficiaries = await BeneficiaryFactory( await UserFactory({ n: community.requestByAddress === users[0].address @@ -614,8 +619,16 @@ describe('community service', () => { }), community.publicId ); + for (const beneficiary of beneficiaries) { + await ClaimFactory(beneficiary, community) + await ClaimFactory(beneficiary, community) + } } + tk.travel(jumpToTomorrowMidnight()); + + await calcuateCommunitiesMetrics(); + const result = await CommunityService.list({}); expect(result.rows[0]).to.include({ @@ -745,7 +758,8 @@ describe('community service', () => { ]); for (const community of communities) { - await BeneficiaryFactory( + await InflowFactory(community); + const beneficiaries = await BeneficiaryFactory( await UserFactory({ n: community.requestByAddress === users[1].address @@ -754,8 +768,16 @@ describe('community service', () => { }), community.publicId ); + for (const beneficiary of beneficiaries) { + await ClaimFactory(beneficiary, community) + await ClaimFactory(beneficiary, community) + } } + tk.travel(jumpToTomorrowMidnight()); + + await calcuateCommunitiesMetrics(); + const result = await CommunityService.list({ orderBy: 'nearest:ASC;bigger:DESC', lat: '-15.8697203', @@ -841,7 +863,8 @@ describe('community service', () => { ]); for (const community of communities) { - await BeneficiaryFactory( + await InflowFactory(community); + const beneficiaries = await BeneficiaryFactory( await UserFactory({ n: community.requestByAddress === users[2].address @@ -850,8 +873,16 @@ describe('community service', () => { }), community.publicId ); + for (const beneficiary of beneficiaries) { + await ClaimFactory(beneficiary, community) + await ClaimFactory(beneficiary, community) + } } + tk.travel(jumpToTomorrowMidnight()); + + await calcuateCommunitiesMetrics(); + const result = await CommunityService.list({ orderBy: 'bigger:ASC;nearest:DESC', lat: '-15.8697203', From 8a493b730e9a781fcb2bdd31f1d978ef35f20436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Mon, 22 Nov 2021 21:53:46 +0000 Subject: [PATCH 03/10] remove unused code --- .../migrations/z1603886582-create-communityDailyState.js | 2 -- src/database/models/ubi/communityDailyState.ts | 2 -- src/services/ubi/community.ts | 3 --- 3 files changed, 7 deletions(-) diff --git a/src/database/migrations/z1603886582-create-communityDailyState.js b/src/database/migrations/z1603886582-create-communityDailyState.js index 523fd393b..e143c0dc1 100644 --- a/src/database/migrations/z1603886582-create-communityDailyState.js +++ b/src/database/migrations/z1603886582-create-communityDailyState.js @@ -71,12 +71,10 @@ module.exports = { allowNull: false, }, totalClaimed: { - // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 type: Sequelize.DECIMAL(27), // max 999,999,999 - plus 18 decimals defaultValue: 0, }, totalRaised: { - // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 type: Sequelize.DECIMAL(27), // max 999,999,999 - plus 18 decimals defaultValue: 0, }, diff --git a/src/database/models/ubi/communityDailyState.ts b/src/database/models/ubi/communityDailyState.ts index 11ec2587f..6d002660a 100644 --- a/src/database/models/ubi/communityDailyState.ts +++ b/src/database/models/ubi/communityDailyState.ts @@ -98,12 +98,10 @@ export function initializeUbiCommunityDailyState(sequelize: Sequelize): void { allowNull: false, }, totalClaimed: { - // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 type: DataTypes.DECIMAL(29), // max 99,999,999,999 - plus 18 decimals defaultValue: 0, }, totalRaised: { - // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 type: DataTypes.DECIMAL(29), // max 99,999,999,999 - plus 18 decimals defaultValue: 0, }, diff --git a/src/services/ubi/community.ts b/src/services/ubi/community.ts index c122e9121..21c5d509e 100644 --- a/src/services/ubi/community.ts +++ b/src/services/ubi/community.ts @@ -1,5 +1,4 @@ import { UbiRequestChangeParams } from '@interfaces/ubi/requestChangeParams'; -import { UbiBeneficiaryRegistryType } from '@interfaces/ubi/ubiBeneficiaryRegistry'; import { UbiCommunityCampaign } from '@interfaces/ubi/ubiCommunityCampaign'; import { UbiCommunityContract } from '@interfaces/ubi/ubiCommunityContract'; import { UbiCommunityDailyMetrics } from '@interfaces/ubi/ubiCommunityDailyMetrics'; @@ -36,14 +35,12 @@ import config from '../../config'; import CommunityContractABI from '../../contracts/CommunityABI.json'; import ImpactMarketContractABI from '../../contracts/ImpactMarketABI.json'; import { models, sequelize } from '../../database'; -import { ICommunityContractParams } from '../../types'; import { ICommunity, ICommunityLightDetails, ICommunityPendingDetails, IManagerDetailsManager, } from '../../types/endpoints'; -import { calcuateCommunitiesMetrics } from '../../worker/jobs/cron/community'; import { CommunityContentStorage, PromoterContentStorage } from '../storage'; import CommunityContractService from './communityContract'; import CommunityStateService from './communityState'; From 5d475253985309f939db7fc0f131d37cfb29f03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Wed, 24 Nov 2021 17:29:10 +0000 Subject: [PATCH 04/10] refactor getState --- src/database/migrations/zz-create-triggers.js | 67 ---- src/services/ubi/community.ts | 303 ++++-------------- tests/integration/services/community.test.ts | 20 +- 3 files changed, 66 insertions(+), 324 deletions(-) diff --git a/src/database/migrations/zz-create-triggers.js b/src/database/migrations/zz-create-triggers.js index 85b0be494..8b5a560da 100644 --- a/src/database/migrations/zz-create-triggers.js +++ b/src/database/migrations/zz-create-triggers.js @@ -5,40 +5,10 @@ module.exports = { up: async (queryInterface, Sequelize) => { // use datagrip for better understanding + highlight - // create trigger to update total beneficiaries ina given community, by day and since ever -// await queryInterface.sequelize.query(` -// CREATE OR REPLACE FUNCTION update_beneficiaries_community_states() -// RETURNS TRIGGER AS $$ -// declare -// community_id integer; -// BEGIN -// SELECT id INTO community_id FROM community where "publicId"=NEW."communityId"; -// IF (TG_OP = 'INSERT') THEN -- INSERT operations (beneficiary being added from community) -// -- update overall state -// UPDATE ubi_community_state SET beneficiaries = beneficiaries + 1 WHERE "communityId"=community_id; -// -- update daily state -// -- UPDATE ubi_community_daily_state SET beneficiaries = beneficiaries + 1 WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); -// ELSEIF (OLD.active IS TRUE AND NEW.active IS FALSE) THEN -- beneficiary being removed from community -// -- update overall state -// UPDATE ubi_community_state SET beneficiaries = beneficiaries - 1, "removedBeneficiaries" = "removedBeneficiaries" + 1 WHERE "communityId"=community_id; -// -- update daily state -// -- UPDATE ubi_community_daily_state SET beneficiaries = beneficiaries - 1 WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); -// END IF; -// RETURN NEW; -// END; -// $$ LANGUAGE plpgsql; -// CREATE TRIGGER update_beneficiaries_community_states -// BEFORE INSERT OR UPDATE -// ON beneficiary -// FOR EACH ROW -// EXECUTE PROCEDURE update_beneficiaries_community_states();`); - await queryInterface.sequelize.query(` CREATE OR REPLACE FUNCTION update_claim_states() RETURNS TRIGGER AS $$ declare - -- state_claimed numeric(29); - -- state_daily_claimed numeric(29); beneficiary_claimed numeric(22); beneficiary_last_claim_at timestamp with time zone; community_public_id uuid; @@ -46,17 +16,11 @@ module.exports = { SELECT "publicId" INTO community_public_id FROM community where id=NEW."communityId"; -- update claims UPDATE ubi_community_state SET claims = claims + 1 WHERE "communityId"=NEW."communityId"; - -- UPDATE ubi_community_daily_state SET claims = claims + 1 WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); -- update beneficiary table as well SELECT "lastClaimAt" INTO beneficiary_last_claim_at FROM beneficiary WHERE "communityId"=community_public_id AND address=NEW.address; UPDATE beneficiary SET claims = claims + 1, "penultimateClaimAt"=beneficiary_last_claim_at, "lastClaimAt"=NEW."txAt" WHERE "communityId"=community_public_id AND address=NEW.address; SELECT SUM(claimed + NEW.amount) INTO beneficiary_claimed FROM beneficiary WHERE "communityId"=community_public_id AND address=NEW.address; UPDATE beneficiary SET claimed = beneficiary_claimed WHERE "communityId"=community_public_id AND address=NEW.address; - -- update total claimed - -- SELECT SUM(claimed + NEW.amount) INTO state_claimed FROM ubi_community_state WHERE "communityId"=NEW."communityId"; - -- UPDATE ubi_community_state SET claimed = state_claimed WHERE "communityId"=NEW."communityId"; - -- SELECT SUM(claimed + NEW.amount) INTO state_daily_claimed FROM ubi_community_daily_state WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); - -- UPDATE ubi_community_daily_state SET claimed = state_daily_claimed WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); return NEW; END; $$ LANGUAGE plpgsql; @@ -66,36 +30,10 @@ ON ubi_claim FOR EACH ROW EXECUTE PROCEDURE update_claim_states();`); -// await queryInterface.sequelize.query(` -// CREATE OR REPLACE FUNCTION update_managers_community_state() -// RETURNS TRIGGER AS -// $$ -// declare -// community_id integer; -// BEGIN -// SELECT id INTO community_id FROM community where "publicId" = NEW."communityId"; -// IF (TG_OP = 'INSERT') THEN -- INSERT operations -// -- update overall state -// UPDATE ubi_community_state SET managers = managers + 1 WHERE "communityId" = community_id; -// ELSEIF (OLD.active IS TRUE AND NEW.active IS FALSE) THEN -- manager being removed from community -// -- update overall state -// UPDATE ubi_community_state SET managers = managers - 1 WHERE "communityId" = community_id; -// END IF; -// RETURN NEW; -// END; -// $$ LANGUAGE plpgsql; -// CREATE TRIGGER update_managers_community_state -// BEFORE INSERT OR UPDATE -// ON manager -// FOR EACH ROW -// EXECUTE PROCEDURE update_managers_community_state();`); - await queryInterface.sequelize.query(` CREATE OR REPLACE FUNCTION update_inflow_community_states() RETURNS TRIGGER AS $$ declare - -- state_raised numeric(29); - -- state_daily_raised numeric(29); n_backer bigint; community_id integer; BEGIN @@ -109,11 +47,6 @@ EXECUTE PROCEDURE update_claim_states();`); IF n_backer = 0 THEN UPDATE ubi_community_state SET backers = backers + 1 WHERE "communityId"=community_id; end if; - -- update total raised - -- SELECT SUM(raised + NEW.amount) INTO state_raised FROM ubi_community_state WHERE "communityId"=community_id; - -- UPDATE ubi_community_state SET raised = state_raised WHERE "communityId"=community_id; - -- SELECT SUM(raised + NEW.amount) INTO state_daily_raised FROM ubi_community_daily_state WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); - -- UPDATE ubi_community_daily_state SET raised = state_daily_raised WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); return NEW; END; $$ LANGUAGE plpgsql; diff --git a/src/services/ubi/community.ts b/src/services/ubi/community.ts index 21c5d509e..510188b35 100644 --- a/src/services/ubi/community.ts +++ b/src/services/ubi/community.ts @@ -319,7 +319,7 @@ export default class CommunityService { }): Promise<{ count: number; rows: CommunityAttributes[] }> { let extendedWhere: WhereOptions = {}; const orderOption: OrderItem[] = []; - + if (query.orderBy) { const orders = query.orderBy.split(';'); @@ -391,7 +391,10 @@ export default class CommunityService { } }); } else { - orderOption.push([literal('"dailyState"."totalBeneficiaries"'), 'DESC']); + orderOption.push([ + literal('"dailyState"."totalBeneficiaries"'), + 'DESC', + ]); } if (query.filter === 'featured') { @@ -729,255 +732,73 @@ export default class CommunityService { } public static async getState(communityId: number) { - const yesterdayDateOnly = new Date(); - yesterdayDateOnly.setUTCHours(0, 0, 0, 0); - yesterdayDateOnly.setDate(yesterdayDateOnly.getDate() - 1); - const yesterdayDate = yesterdayDateOnly.toISOString().split('T')[0]; - const todayDateOnly = new Date(); - todayDateOnly.setUTCHours(0, 0, 0, 0); - const todayDate = todayDateOnly.toISOString().split('T')[0]; - const result = await this.ubiCommunityState.findOne({ where: { communityId, }, }); - const previousDate = await this.ubiCommunityDailyState.findOne({ - attributes: [ - 'totalClaimed', - 'totalRaised', - 'totalBeneficiaries', - 'totalManagers', - ], + const community = await this.community.findOne({ + attributes: ['contractAddress', 'publicId'], where: { - communityId, - date: yesterdayDate, + id: communityId, }, }); - const communityClaimActivity: { - totalClaimed: string; - } = (await this.community.findOne({ - attributes: [ - [ - fn( - 'coalesce', - fn('sum', col('beneficiaries->claim.amount')), - '0' - ), - 'totalClaimed', + const communityClaimActivity = ( + await this.ubiClaim.findAll({ + attributes: [ + [fn('coalesce', fn('sum', col('amount')), '0'), 'amount'], ], - ], - include: [ - { - model: this.beneficiary, - as: 'beneficiaries', - attributes: [], - required: false, - include: [ - { - model: this.ubiClaim, - as: 'claim', - attributes: [], - required: false, - where: { - txAt: where( - fn( - 'date', - col('beneficiaries->claim.txAt') - ), - '=', - todayDate - ), - }, - }, - ], + where: { + communityId, }, - ], - where: { - id: communityId, - }, - group: ['Community.id'], - raw: true, - })) as any; + }) + )[0]; - const communityInflowActivity: { - totalRaised: string; - } = (await models.community.findOne({ - attributes: [ - [ - fn('sum', fn('coalesce', col('inflow.amount'), 0)), - 'totalRaised', + const communityInflowActivity = ( + await models.inflow.findAll({ + attributes: [ + [fn('sum', fn('coalesce', col('amount'), 0)), 'amount'], ], - ], - include: [ - { - model: models.inflow, - as: 'inflow', - attributes: [], - required: false, - where: { - txAt: where( - fn('date', col('inflow."txAt"')), - '=', - todayDate - ), - }, - }, - ], - where: { - id: communityId, - }, - group: ['Community.id'], - raw: true, - })) as any; - - const communityNewBeneficiaryActivity: { - beneficiaries: string; - } = (await models.community.findOne({ - attributes: [ - [fn('count', col('beneficiaries.address')), 'beneficiaries'], - ], - include: [ - { - model: models.beneficiary, - as: 'beneficiaries', - attributes: [], - required: false, - where: { - txAt: where( - fn('date', col('beneficiaries."txAt"')), - '=', - todayDate - ), - }, + where: { + contractAddress: community?.contractAddress, }, - ], - where: { - id: communityId, - }, - group: ['Community.id'], - raw: true, - })) as any; + }) + )[0]; - const communityRemovedBeneficiaryActivity: { - beneficiaries: string; - } = (await models.community.findOne({ - attributes: [ - [fn('count', col('beneficiaries.address')), 'beneficiaries'], - ], - include: [ - { - model: models.beneficiary, - as: 'beneficiaries', - attributes: [], - required: false, - where: { - updatedAt: where( - fn('date', col('beneficiaries."updatedAt"')), - '=', - todayDate - ), - active: false, - }, - }, - ], + const communityBeneficiaryActivity = (await this.beneficiary.findAll({ + attributes: [[fn('COUNT', col('address')), 'count'], 'active'], where: { - id: communityId, + communityId: community?.publicId, }, - group: ['Community.id'], + group: ['active'], raw: true, })) as any; - const communityNewManagerActivity: { - managers: string; - } = (await models.community.findOne({ - attributes: [[fn('count', col('managers.address')), 'managers']], - include: [ - { - model: models.manager, - as: 'managers', - attributes: [], - required: false, - where: { - createdAt: where( - fn('date', col('managers."createdAt"')), - '=', - todayDate - ), - }, - }, - ], + const communityManagerActivity = await this.manager.count({ where: { - id: communityId, + communityId: community?.publicId, + active: true, }, - group: ['Community.id'], - raw: true, - })) as any; - - const communityRemovedManagerActivity: { - managers: string; - } = (await models.community.findOne({ - attributes: [[fn('count', col('managers.address')), 'managers']], - include: [ - { - model: models.manager, - as: 'managers', - attributes: [], - required: false, - where: { - updatedAt: where( - fn('date', col('managers."updatedAt"')), - '=', - todayDate - ), - active: false, - }, - }, - ], - where: { - id: communityId, - }, - group: ['Community.id'], - raw: true, - })) as any; + }); if (result === null) return null; - const previousClaims = previousDate?.totalClaimed - ? new BigNumber(previousDate.totalClaimed) - : new BigNumber(0); - const todayClaims = communityClaimActivity?.totalClaimed - ? new BigNumber(communityClaimActivity.totalClaimed) - : new BigNumber(0); - const previousRaise = previousDate?.totalRaised - ? new BigNumber(previousDate.totalRaised) - : new BigNumber(0); - const todayRaise = communityInflowActivity?.totalRaised - ? new BigNumber(communityInflowActivity.totalRaised) - : new BigNumber(0); - - const previousBeneficiaries = previousDate?.totalBeneficiaries - ? previousDate.totalBeneficiaries - : 0; - const todayBeneficiaries = - parseInt(communityNewBeneficiaryActivity.beneficiaries) - - parseInt(communityRemovedBeneficiaryActivity.beneficiaries); - - const previousManagers = previousDate?.totalManagers - ? previousDate.totalManagers - : 0; - const todayManagers = - parseInt(communityNewManagerActivity.managers) - - parseInt(communityRemovedManagerActivity.managers); + const beneficiaries: { count: string; active: boolean } = + communityBeneficiaryActivity.find((el: any) => el.active); + const removedBeneficiaries: { count: string; active: boolean } = + communityBeneficiaryActivity.find((el: any) => !el.active); return { ...(result.toJSON() as UbiCommunityState), - claimed: previousClaims.plus(todayClaims).toString(), - raised: previousRaise.plus(todayRaise).toString(), - beneficiaries: previousBeneficiaries + todayBeneficiaries, - manager: previousManagers + todayManagers, + claimed: communityClaimActivity.amount, + raised: communityInflowActivity.amount, + beneficiaries: beneficiaries ? Number(beneficiaries.count) : 0, + removedBeneficiaries: removedBeneficiaries + ? Number(removedBeneficiaries.count) + : 0, + managers: communityManagerActivity, }; } @@ -1797,7 +1618,10 @@ export default class CommunityService { }; } - private static _generateInclude(fields: any, extended?: string): Includeable[] { + private static _generateInclude( + fields: any, + extended?: string + ): Includeable[] { const extendedInclude: Includeable[] = []; if (fields.suspect) { extendedInclude.push({ @@ -1851,40 +1675,27 @@ export default class CommunityService { const yesterdayDate = yesterdayDateOnly.toISOString().split('T')[0]; const dailyState = fields.dailyState - ? fields.dailyState.length > 0 - ? fields.dailyState - : { exclude: stateExclude } - : []; + ? fields.dailyState.length > 0 + ? fields.dailyState + : { exclude: stateExclude } + : []; extendedInclude.push({ attributes: dailyState, model: this.ubiCommunityDailyState, as: 'dailyState', duplicating: false, where: { - date: yesterdayDate + date: yesterdayDate, }, required: false, }); - const stateAttributes = fields.state - ? fields.state.length > 0 - ? fields.state.filter( - (el: string) => !stateExclude.includes(el) - ) - : { exclude: stateExclude } - : []; - extendedInclude.push({ - model: this.ubiCommunityState, - attributes: stateAttributes, - as: 'state', - }); - if (extended || fields.metrics) { const metricsAttributes = fields.metrics - ? fields.metrics.length > 0 - ? fields.metrics - : undefined - : []; + ? fields.metrics.length > 0 + ? fields.metrics + : undefined + : []; extendedInclude.push({ attributes: metricsAttributes, @@ -1968,7 +1779,7 @@ export default class CommunityService { as: 'dailyState', duplicating: false, where: { - date: yesterdayDate + date: yesterdayDate, }, required: false, }, diff --git a/tests/integration/services/community.test.ts b/tests/integration/services/community.test.ts index ac9a6418f..17fa34782 100644 --- a/tests/integration/services/community.test.ts +++ b/tests/integration/services/community.test.ts @@ -15,13 +15,13 @@ import ManagerService from '../../../src/services/ubi/managers'; import { calcuateCommunitiesMetrics } from '../../../src/worker/jobs/cron/community'; import { verifyDeletedAccounts } from '../../../src/worker/jobs/cron/user'; import BeneficiaryFactory from '../../factories/beneficiary'; +import ClaimFactory from '../../factories/claim'; import CommunityFactory from '../../factories/community'; +import InflowFactory from '../../factories/inflow'; import ManagerFactory from '../../factories/manager'; import UserFactory from '../../factories/user'; import truncate, { sequelizeSetup } from '../../utils/sequelizeSetup'; import { randomTx, jumpToTomorrowMidnight } from '../../utils/utils'; -import InflowFactory from '../../factories/inflow'; -import ClaimFactory from '../../factories/claim'; // in this test there are users being assined with suspicious activity and others being removed describe('community service', () => { @@ -620,8 +620,8 @@ describe('community service', () => { community.publicId ); for (const beneficiary of beneficiaries) { - await ClaimFactory(beneficiary, community) - await ClaimFactory(beneficiary, community) + await ClaimFactory(beneficiary, community); + await ClaimFactory(beneficiary, community); } } @@ -769,8 +769,8 @@ describe('community service', () => { community.publicId ); for (const beneficiary of beneficiaries) { - await ClaimFactory(beneficiary, community) - await ClaimFactory(beneficiary, community) + await ClaimFactory(beneficiary, community); + await ClaimFactory(beneficiary, community); } } @@ -874,8 +874,8 @@ describe('community service', () => { community.publicId ); for (const beneficiary of beneficiaries) { - await ClaimFactory(beneficiary, community) - await ClaimFactory(beneficiary, community) + await ClaimFactory(beneficiary, community); + await ClaimFactory(beneficiary, community); } } @@ -1099,7 +1099,7 @@ describe('community service', () => { ); const result = await CommunityService.list({ - fields: 'id;publicId;contractAddress;contract.*;state.claimed;cover.*', + fields: 'id;publicId;contractAddress;contract.*;cover.*', }); expect(result.rows[0]).to.have.deep.keys([ @@ -1107,7 +1107,6 @@ describe('community service', () => { 'publicId', 'contractAddress', 'contract', - 'state', 'cover', ]); expect(result.rows[0].contract).to.have.deep.keys([ @@ -1121,7 +1120,6 @@ describe('community service', () => { 'createdAt', 'updatedAt', ]); - expect(result.rows[0].state).to.have.deep.keys(['claimed']); expect(result.rows[0].cover).to.have.deep.keys([ 'id', 'url', From f4c9fdfa74d62d4ff9ef19feac8d42b97fbe6831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Thu, 2 Dec 2021 00:44:30 +0000 Subject: [PATCH 05/10] refactor list community --- .../z1603886582-create-communityDailyState.js | 16 - ...636638661-update-ubiCommunityDailyState.js | 35 -- ...636638954-update-ubiCommunityDailyState.js | 64 -- ...637070248-update-ubiCommunityDailyState.js | 23 - src/database/models/associations/community.ts | 5 + .../models/ubi/communityDailyState.ts | 20 - src/interfaces/ubi/ubiCommunityDailyState.ts | 8 - src/interfaces/ubi/ubiCommunityState.ts | 12 +- src/services/ubi/community.ts | 557 ++++++++++++++---- src/worker/jobs/cron/community.ts | 65 -- .../cron/calcuateCommunitiesMetrics.test.ts | 40 -- tests/integration/services/community.test.ts | 48 +- 12 files changed, 470 insertions(+), 423 deletions(-) delete mode 100644 src/database/migrations/z1636638661-update-ubiCommunityDailyState.js delete mode 100644 src/database/migrations/z1636638954-update-ubiCommunityDailyState.js delete mode 100644 src/database/migrations/z1637070248-update-ubiCommunityDailyState.js diff --git a/src/database/migrations/z1603886582-create-communityDailyState.js b/src/database/migrations/z1603886582-create-communityDailyState.js index e143c0dc1..097154916 100644 --- a/src/database/migrations/z1603886582-create-communityDailyState.js +++ b/src/database/migrations/z1603886582-create-communityDailyState.js @@ -70,22 +70,6 @@ module.exports = { type: Sequelize.DATEONLY, allowNull: false, }, - totalClaimed: { - type: Sequelize.DECIMAL(27), // max 999,999,999 - plus 18 decimals - defaultValue: 0, - }, - totalRaised: { - type: Sequelize.DECIMAL(27), // max 999,999,999 - plus 18 decimals - defaultValue: 0, - }, - totalBeneficiaries: { - type: Sequelize.INTEGER, - defaultValue: 0, - }, - totalManagers: { - type: Sequelize.INTEGER, - defaultValue: 0, - }, }); }, down: (queryInterface) => { diff --git a/src/database/migrations/z1636638661-update-ubiCommunityDailyState.js b/src/database/migrations/z1636638661-update-ubiCommunityDailyState.js deleted file mode 100644 index 4434e7e67..000000000 --- a/src/database/migrations/z1636638661-update-ubiCommunityDailyState.js +++ /dev/null @@ -1,35 +0,0 @@ -'use strict'; - -module.exports = { - up: async (queryInterface, Sequelize) => { - if (process.env.NODE_ENV === 'test') { - return; - } - - await queryInterface.addColumn('ubi_community_daily_state', 'totalClaimed', { - type: Sequelize.DECIMAL(29), - allowNull: false, - defaultValue: 0 - }); - - await queryInterface.addColumn('ubi_community_daily_state', 'totalRaised', { - type: Sequelize.DECIMAL(29), - allowNull: false, - defaultValue: 0 - }); - - await queryInterface.addColumn('ubi_community_daily_state', 'totalBeneficiaries', { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: 0 - }); - - await queryInterface.addColumn('ubi_community_daily_state', 'totalManagers', { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: 0 - }); - }, - - down(queryInterface, Sequelize) {}, -}; diff --git a/src/database/migrations/z1636638954-update-ubiCommunityDailyState.js b/src/database/migrations/z1636638954-update-ubiCommunityDailyState.js deleted file mode 100644 index 62f4b8edc..000000000 --- a/src/database/migrations/z1636638954-update-ubiCommunityDailyState.js +++ /dev/null @@ -1,64 +0,0 @@ -'use strict'; - -module.exports = { - up: async (queryInterface, Sequelize) => { - if (process.env.NODE_ENV === 'test') { - return; - } - - const communities = (await queryInterface.sequelize.query(` - select "communityId" - from ubi_community_daily_state - group by "communityId"; - `, { raw: true }))[0]; - - // update the first day of each community - for(let i = 0; i < communities.length; i++) { - //update totalClaimed, totalRaised, totalBeneficiaries - await queryInterface.sequelize.query(` - update ubi_community_daily_state - set "totalClaimed" = claimed, - "totalRaised" = raised, - "totalBeneficiaries" = beneficiaries - where date = (select date - from ubi_community_daily_state - where "communityId" = ${communities[i].communityId} - order by date - limit 1 - ) and "communityId" = ${communities[i].communityId}; - `); - } - - // update all others by day - const startDate = new Date('2020-09-22'); - startDate.setUTCHours(0, 0, 0, 0); - - const today = new Date(); - today.setUTCHours(0, 0, 0, 0); - - while (startDate <= today) { - const query = ` - update ubi_community_daily_state - set "totalClaimed" = previous."totalClaimed" + today.claimed, - "totalRaised" = previous."totalRaised" + today.raised, - "totalBeneficiaries" = previous."totalBeneficiaries" + today.beneficiaries - from ( - select distinct on ("communityId") "communityId", "totalClaimed", "totalRaised", "totalBeneficiaries" - from ubi_community_daily_state - where date < '${startDate.toISOString().split('T')[0]}' - order by "communityId", date DESC - ) as previous, - (select claimed, raised, beneficiaries, "communityId" from ubi_community_daily_state where date = '${startDate.toISOString().split('T')[0]}') as today - where ubi_community_daily_state."communityId" = previous."communityId" - and ubi_community_daily_state."communityId" = today."communityId" - and ubi_community_daily_state.date = '${startDate.toISOString().split('T')[0]}'; - `; - - await queryInterface.sequelize.query(query); - - startDate.setDate(startDate.getDate() + 1); - } - }, - - down(queryInterface, Sequelize) {}, -}; diff --git a/src/database/migrations/z1637070248-update-ubiCommunityDailyState.js b/src/database/migrations/z1637070248-update-ubiCommunityDailyState.js deleted file mode 100644 index cf24d99a4..000000000 --- a/src/database/migrations/z1637070248-update-ubiCommunityDailyState.js +++ /dev/null @@ -1,23 +0,0 @@ -'use strict'; - -module.exports = { - up: async (queryInterface, Sequelize) => { - if (process.env.NODE_ENV === 'test') { - return; - } - - const lastState = (await queryInterface.sequelize.query( - `select date from ubi_community_daily_state order by date DESC limit 1`, - { raw: true, type: Sequelize.QueryTypes.SELECT } - ))[0]; - - await queryInterface.sequelize.query(` - update ubi_community_daily_state - set "totalManagers" = state.managers - from (select managers, "communityId" from ubi_community_state) as state - where date = '${lastState.date}' and ubi_community_daily_state."communityId" = state."communityId"; - `); - }, - - down(queryInterface, Sequelize) {}, -}; diff --git a/src/database/models/associations/community.ts b/src/database/models/associations/community.ts index 767dfdad1..48f0b6b52 100644 --- a/src/database/models/associations/community.ts +++ b/src/database/models/associations/community.ts @@ -109,4 +109,9 @@ export function communityAssociation(sequelize: Sequelize) { sourceKey: 'proposalId', as: 'proposal', }); + sequelize.models.Community.hasMany(sequelize.models.UbiClaimModel, { + foreignKey: 'communityId', + sourceKey: 'id', + as: 'claims', + }); } diff --git a/src/database/models/ubi/communityDailyState.ts b/src/database/models/ubi/communityDailyState.ts index 6d002660a..6ba3d31e7 100644 --- a/src/database/models/ubi/communityDailyState.ts +++ b/src/database/models/ubi/communityDailyState.ts @@ -22,10 +22,6 @@ export class UbiCommunityDailyStateModel extends Model< public reachOut!: number; public fundingRate!: number; public date!: Date; - public totalClaimed!: string; - public totalRaised!: string; - public totalBeneficiaries!: number; - public totalManagers!: number; } export function initializeUbiCommunityDailyState(sequelize: Sequelize): void { @@ -97,22 +93,6 @@ export function initializeUbiCommunityDailyState(sequelize: Sequelize): void { type: DataTypes.DATEONLY, allowNull: false, }, - totalClaimed: { - type: DataTypes.DECIMAL(29), // max 99,999,999,999 - plus 18 decimals - defaultValue: 0, - }, - totalRaised: { - type: DataTypes.DECIMAL(29), // max 99,999,999,999 - plus 18 decimals - defaultValue: 0, - }, - totalBeneficiaries: { - type: DataTypes.INTEGER, - defaultValue: 0, - }, - totalManagers: { - type: DataTypes.INTEGER, - defaultValue: 0, - }, }, { tableName: 'ubi_community_daily_state', diff --git a/src/interfaces/ubi/ubiCommunityDailyState.ts b/src/interfaces/ubi/ubiCommunityDailyState.ts index 8e6816ff0..daf375df7 100644 --- a/src/interfaces/ubi/ubiCommunityDailyState.ts +++ b/src/interfaces/ubi/ubiCommunityDailyState.ts @@ -13,10 +13,6 @@ export interface UbiCommunityDailyState { reachOut: number; fundingRate: number; date: Date; - totalClaimed: string; - totalRaised: string; - totalBeneficiaries: number; - totalManagers: number; } export interface UbiCommunityDailyStateCreation { communityId: number; @@ -32,8 +28,4 @@ export interface UbiCommunityDailyStateCreation { reachOut: number; fundingRate: number; date: Date; - totalClaimed: string; - totalRaised: string; - totalBeneficiaries: number; - totalManagers: number; } diff --git a/src/interfaces/ubi/ubiCommunityState.ts b/src/interfaces/ubi/ubiCommunityState.ts index 01971d4a5..546d6586d 100644 --- a/src/interfaces/ubi/ubiCommunityState.ts +++ b/src/interfaces/ubi/ubiCommunityState.ts @@ -40,18 +40,18 @@ * description: Number of unique backers since contract inception */ export interface UbiCommunityState { - communityId: number; + communityId?: number; claimed: string; - claims: number; + claims?: number; beneficiaries: number; // only in community - removedBeneficiaries: number; - managers: number; + removedBeneficiaries?: number; + managers?: number; raised: string; backers: number; // timestamps - createdAt: Date; - updatedAt: Date; + createdAt?: Date; + updatedAt?: Date; } export interface UbiCommunityStateCreation { diff --git a/src/services/ubi/community.ts b/src/services/ubi/community.ts index 3acb51b3d..62290aaa2 100644 --- a/src/services/ubi/community.ts +++ b/src/services/ubi/community.ts @@ -27,7 +27,6 @@ import { OrderItem, WhereOptions, Includeable, - where, } from 'sequelize'; import { Literal } from 'sequelize/types/lib/utils'; @@ -319,12 +318,79 @@ export default class CommunityService { }): Promise<{ count: number; rows: CommunityAttributes[] }> { let extendedWhere: WhereOptions = {}; const orderOption: OrderItem[] = []; + const orderBeneficiary: OrderItem[] = []; + let orderOutOfFunds: string | undefined; + + let beneficiariesState: + | [ + { + id: number; + beneficiaries: string; + } + ] + | undefined = undefined; + let claimsState: + | [ + { + communityId: number; + claimed: string; + } + ] + | undefined = undefined; + let inflowState: + | [ + { + id: number; + raised: string; + } + ] + | undefined = undefined; + let backerState: + | [ + { + id: number; + backers: string; + } + ] + | undefined = undefined; + let communitiesId: number[] = []; + + if (query.filter === 'featured') { + const featuredIds = ( + await this.ubiCommunityLabels.findAll({ + attributes: ['communityId'], + where: { label: 'featured' }, + }) + ).map((c) => (c.toJSON() as UbiCommunityLabel).communityId); + extendedWhere = { + ...extendedWhere, + id: { [Op.in]: featuredIds }, + }; + } + + if (query.name) { + extendedWhere = { + ...extendedWhere, + name: { + [Op.iLike]: `%${query.name}%`, + }, + }; + } + + if (query.country) { + extendedWhere = { + ...extendedWhere, + country: { + [Op.in]: query.country.split(';'), + }, + }; + } if (query.orderBy) { const orders = query.orderBy.split(';'); - orders.forEach((element) => { - const [order, orderType] = element.split(':'); + for (let i = 0; i < orders.length; i++) { + const [order, orderType] = orders[i].split(':'); switch (order) { case 'nearest': { @@ -363,17 +429,23 @@ export default class CommunityService { case 'out_of_funds': { // this requires extended query.extended = 'true'; - extendedWhere = { - '$"dailyState"."totalBeneficiaries"$': { - [Op.not]: 0, - }, - } as any; - orderOption.push([ - literal( - '("dailyState"."totalRaised" - "dailyState"."totalClaimed") / metrics."ubiRate" / "dailyState"."totalBeneficiaries"' - ), - orderType ? orderType : 'ASC', - ]); + // check if there was another order previously + if ( + orderOption.length === 0 && + orderBeneficiary.length === 0 + ) { + const result = await this._getOutOfFunds( + { + limit: query.limit, + offset: query.offset, + }, + orderType + ); + communitiesId = result.map((el) => el.id); + } else { + // list communities out of funds after + orderOutOfFunds = orderType; + } break; } case 'newest': @@ -382,58 +454,52 @@ export default class CommunityService { orderType ? orderType : 'DESC', ]); break; - default: - orderOption.push([ - literal('"dailyState"."totalBeneficiaries"'), - orderType ? orderType : 'DESC', - ]); + default: { + // check if there was another order previously + if (orderOption.length === 0 && !orderOutOfFunds) { + beneficiariesState = + await this._getBeneficiaryState( + { + status: query.status, + limit: query.limit, + offset: query.offset, + }, + extendedWhere, + orderType + ); + communitiesId = beneficiariesState!.map( + (el) => el.id + ); + } else { + orderBeneficiary.push([ + literal('count("beneficiaries"."address")'), + orderType ? orderType : 'DESC', + ]); + } + break; + } } - }); + } } else { - orderOption.push([ - literal('"dailyState"."totalBeneficiaries"'), - 'DESC', - ]); - } - - if (query.filter === 'featured') { - const featuredIds = ( - await this.ubiCommunityLabels.findAll({ - attributes: ['communityId'], - where: { label: 'featured' }, - }) - ).map((c) => (c.toJSON() as UbiCommunityLabel).communityId); - extendedWhere = { - ...extendedWhere, - id: { [Op.in]: featuredIds }, - }; - } - - if (query.name) { - extendedWhere = { - ...extendedWhere, - name: { - [Op.iLike]: `%${query.name}%`, - }, - }; - } - - if (query.country) { - extendedWhere = { - ...extendedWhere, - country: { - [Op.in]: query.country.split(';'), + beneficiariesState = await this._getBeneficiaryState( + { + status: query.status, + limit: query.limit, + offset: query.offset, }, - }; + extendedWhere + ); } let include: Includeable[]; let attributes: any; + let returnState = true; const exclude = ['email']; if (query.fields) { const fields = fetchData(query.fields); - include = this._generateInclude(fields, query.extended); + if (!fields.state) returnState = false; + include = this._generateInclude(fields); attributes = fields.root ? fields.root.length > 0 ? fields.root.filter((el: string) => !exclude.includes(el)) @@ -446,29 +512,209 @@ export default class CommunityService { }; } - const communitiesResult = await this.community.findAndCountAll({ - attributes, + const communityCount = await this.community.count({ where: { status: query.status ? query.status : 'valid', visibility: 'public', ...extendedWhere, }, - include, - order: orderOption, - offset: query.offset - ? parseInt(query.offset, 10) - : config.defaultOffset, - limit: query.limit - ? parseInt(query.limit, 10) - : config.defaultLimit, }); - const communities = communitiesResult.rows.map((c) => - c.toJSON() - ) as CommunityAttributes[]; + let communitiesResult: Community[]; + + if (communitiesId.length > 0) { + communitiesResult = await this.community.findAll({ + attributes, + order: orderOption, + where: { + id: { + [Op.in]: communitiesId, + }, + }, + include, + }); + } else { + communitiesResult = await this.community.findAll({ + attributes, + include, + order: orderOption, + where: { + status: query.status ? query.status : 'valid', + visibility: 'public', + ...extendedWhere, + }, + offset: query.offset + ? parseInt(query.offset, 10) + : config.defaultOffset, + limit: query.limit + ? parseInt(query.limit, 10) + : config.defaultLimit, + }); + communitiesId = communitiesResult!.map((el) => el.id); + } + + if (orderOutOfFunds) { + const result = await this._getOutOfFunds( + { + limit: query.limit, + offset: query.offset, + }, + undefined, + communitiesId + ); + // re-order by out of funds + communitiesId = result.map((el) => el.id); + } + + if (returnState) { + if (!beneficiariesState) { + beneficiariesState = (await models.community.findAll({ + attributes: [ + 'id', + [ + fn( + 'count', + fn('distinct', col('beneficiaries.address')) + ), + 'beneficiaries', + ], + ], + order: orderBeneficiary, + include: [ + { + attributes: [], + model: models.beneficiary, + as: 'beneficiaries', + where: { + active: true, + }, + duplicating: false, + }, + ], + where: { + id: { + [Op.in]: communitiesId, + }, + }, + group: ['Community.id'], + raw: true, + })) as any; + + // re-order by total beneficiaries + if (orderBeneficiary.length > 0) { + communitiesId = beneficiariesState!.map((el) => el.id); + } + } + + if (!claimsState) { + claimsState = (await models.ubiClaim.findAll({ + attributes: [ + 'communityId', + [ + fn('coalesce', fn('sum', col('amount')), '0'), + 'claimed', + ], + ], + where: { + communityId: { + [Op.in]: communitiesId, + }, + }, + group: ['communityId'], + raw: true, + })) as any; + } + + if (!inflowState) { + inflowState = (await models.community.findAll({ + attributes: [ + 'id', + [ + fn('coalesce', fn('sum', col('amount')), '0'), + 'raised', + ], + ], + include: [ + { + attributes: [], + model: models.inflow, + as: 'inflow', + }, + ], + where: { + id: { + [Op.in]: communitiesId, + }, + }, + group: ['Community.id'], + raw: true, + })) as any; + } + + if (!backerState) { + backerState = (await models.community.findAll({ + attributes: [ + 'id', + [ + fn('count', fn('distinct', col('inflow.from'))), + 'backers', + ], + ], + include: [ + { + attributes: [], + model: models.inflow, + as: 'inflow', + }, + ], + where: { + id: { + [Op.in]: communitiesId, + }, + }, + group: ['Community.id'], + raw: true, + })) as any; + } + } + + //formate response + const communities: CommunityAttributes[] = []; + communitiesId.forEach((id) => { + let community: CommunityAttributes; + const filteredCommunity = communitiesResult.find( + (el) => el.id === id + ); + community = { + ...(filteredCommunity?.toJSON() as CommunityAttributes), + }; + + if (returnState) { + const beneficiariesModel = beneficiariesState?.find( + (el) => el.id === id + ); + const claimModel = claimsState?.find( + (el) => el.communityId === id + ); + const raiseModel = inflowState?.find((el) => el.id === id); + const backerModel = backerState?.find((el) => el.id === id); + community = { + ...community, + state: { + beneficiaries: beneficiariesModel + ? Number(beneficiariesModel.beneficiaries) + : 0, + claimed: claimModel ? claimModel.claimed : '0', + raised: raiseModel ? raiseModel.raised : '0', + backers: backerModel ? Number(backerModel.backers) : 0, + }, + }; + } + communities.push(community); + }); return { - count: communitiesResult.count, + count: communityCount, rows: communities, }; } @@ -732,27 +978,31 @@ export default class CommunityService { } public static async getState(communityId: number) { - const result = await this.ubiCommunityState.findOne({ + const community = await this.community.findOne({ + attributes: ['contractAddress', 'publicId'], where: { - communityId, + id: communityId, }, }); - const community = await this.community.findOne({ - attributes: ['contractAddress', 'publicId'], + const communityBackers = await models.inflow.count({ + distinct: true, + col: 'from', where: { - id: communityId, + contractAddress: community?.contractAddress, }, }); const communityClaimActivity = ( await this.ubiClaim.findAll({ attributes: [ - [fn('coalesce', fn('sum', col('amount')), '0'), 'amount'], + [fn('coalesce', fn('sum', col('amount')), '0'), 'claimed'], + [fn('coalesce', fn('count', col('amount')), '0'), 'claims'], ], where: { communityId, }, + raw: true, }) )[0]; @@ -783,22 +1033,26 @@ export default class CommunityService { }, }); - if (result === null) return null; - const beneficiaries: { count: string; active: boolean } = communityBeneficiaryActivity.find((el: any) => el.active); const removedBeneficiaries: { count: string; active: boolean } = communityBeneficiaryActivity.find((el: any) => !el.active); return { - ...(result.toJSON() as UbiCommunityState), - claimed: communityClaimActivity.amount, + claims: communityClaimActivity + ? Number((communityClaimActivity as any).claims) + : 0, + claimed: communityClaimActivity + ? (communityClaimActivity as any).claimed + : '0', raised: communityInflowActivity.amount, beneficiaries: beneficiaries ? Number(beneficiaries.count) : 0, removedBeneficiaries: removedBeneficiaries ? Number(removedBeneficiaries.count) : 0, managers: communityManagerActivity, + backers: communityBackers, + communityId, }; } @@ -1618,11 +1872,9 @@ export default class CommunityService { }; } - private static _generateInclude( - fields: any, - extended?: string - ): Includeable[] { + private static _generateInclude(fields: any): Includeable[] { const extendedInclude: Includeable[] = []; + if (fields.suspect) { extendedInclude.push({ model: this.ubiCommunitySuspect, @@ -1675,30 +1927,7 @@ export default class CommunityService { }); } - const stateExclude = ['id', 'communityId']; - - const yesterdayDateOnly = new Date(); - yesterdayDateOnly.setUTCHours(0, 0, 0, 0); - yesterdayDateOnly.setDate(yesterdayDateOnly.getDate() - 1); - const yesterdayDate = yesterdayDateOnly.toISOString().split('T')[0]; - - const dailyState = fields.dailyState - ? fields.dailyState.length > 0 - ? fields.dailyState - : { exclude: stateExclude } - : []; - extendedInclude.push({ - attributes: dailyState, - model: this.ubiCommunityDailyState, - as: 'dailyState', - duplicating: false, - where: { - date: yesterdayDate, - }, - required: false, - }); - - if (extended || fields.metrics) { + if (fields.metrics) { const metricsAttributes = fields.metrics ? fields.metrics.length > 0 ? fields.metrics @@ -1791,13 +2020,115 @@ export default class CommunityService { }, required: false, }, - // TODO: deprecated, use dailyState - { - model: this.ubiCommunityState, - attributes: { exclude: ['id', 'communityId'] }, - as: 'state', - }, ...extendedInclude, ] as Includeable[]; } + + private static async _getOutOfFunds( + query: { + limit?: string; + offset?: string; + }, + orderType?: string, + communitiesId?: number[] + ): Promise<[{ id: number }]> { + const sql = `SELECT "Beneficiary".id + FROM ( + SELECT "Community"."id", count(distinct("beneficiaries"."address")) AS "beneficiaries" + FROM "community" AS "Community" + INNER JOIN "beneficiary" AS "beneficiaries" ON "Community"."publicId" = "beneficiaries"."communityId" AND "beneficiaries"."active" = true + WHERE ${ + !!communitiesId && communitiesId.length > 0 + ? `"Community"."id" IN (${communitiesId.join()})` + : `"Community"."status" = 'valid' AND "Community"."visibility" = 'public'` + } + GROUP BY "Community"."id" + ) AS "Beneficiary" INNER JOIN ( + SELECT "Community"."id", sum("claims"."amount") AS "claimed" + FROM "community" AS "Community" + INNER JOIN "ubi_claim" AS "claims" ON "Community"."id" = "claims"."communityId" + WHERE "Community"."status" = 'valid' AND "Community"."visibility" = 'public' + GROUP BY "Community"."id" + ) AS "Claims" on "Claims".id = "Beneficiary".id INNER JOIN ( + SELECT "Community"."id", sum("inflow"."amount") AS "raised" + FROM "community" AS "Community" + INNER JOIN "inflow" AS "inflow" ON "Community"."contractAddress" = "inflow"."contractAddress" + WHERE "Community"."status" = 'valid' AND "Community"."visibility" = 'public' + GROUP BY "Community"."id" + ) AS "Inflow" ON "Beneficiary".id = "Inflow".id INNER JOIN ( + SELECT "communityId", "ubiRate" + FROM ubi_community_daily_metrics + WHERE date = ( + SELECT date from ubi_community_daily_metrics order by date DESC limit 1 + ) + ) AS "Metrics" ON "Metrics"."communityId" = "Beneficiary".id + WHERE "Beneficiary".beneficiaries != 0 + ORDER BY ("Inflow".raised - "Claims".claimed) / "Metrics"."ubiRate" / "Beneficiary".beneficiaries ${ + orderType ? orderType : 'ASC' + } + LIMIT ${ + query.limit ? parseInt(query.limit, 10) : config.defaultLimit + } OFFSET ${ + query.offset ? parseInt(query.offset, 10) : config.defaultOffset + }`; + + const result = ( + await this.sequelize.query(sql, { + raw: true, + }) + )[0] as [{ id: number }]; + + return result; + } + + private static async _getBeneficiaryState( + query: { + status?: string; + limit?: string; + offset?: string; + }, + extendedWhere: WhereOptions, + orderType?: string + ): Promise { + const result = (await models.community.findAll({ + attributes: [ + 'id', + [ + fn('count', fn('distinct', col('beneficiaries.address'))), + 'beneficiaries', + ], + ], + where: { + status: query.status ? query.status : 'valid', + visibility: 'public', + ...extendedWhere, + }, + include: [ + { + attributes: [], + model: models.beneficiary, + as: 'beneficiaries', + where: { + active: true, + }, + duplicating: false, + }, + ], + group: ['Community.id'], + order: [ + [ + literal('count("beneficiaries"."address")'), + orderType ? orderType : 'DESC', + ], + ], + raw: true, + offset: query.offset + ? parseInt(query.offset, 10) + : config.defaultOffset, + limit: query.limit + ? parseInt(query.limit, 10) + : config.defaultLimit, + })) as any; + return result; + } } diff --git a/src/worker/jobs/cron/community.ts b/src/worker/jobs/cron/community.ts index 23daa56a2..019d04a6d 100644 --- a/src/worker/jobs/cron/community.ts +++ b/src/worker/jobs/cron/community.ts @@ -52,11 +52,6 @@ export async function calcuateCommunitiesMetrics(): Promise { reachOut: string; fundingRate: string; beneficiaries: number; - managers: number; - totalClaimed: string; - totalRaised: string; - totalBeneficiaries: number; - totalManagers: number; }; }; @@ -500,27 +495,6 @@ export async function calcuateCommunitiesMetrics(): Promise { raw: true, })) as any; - const previousState: { - communityId: number; - totalClaimed: string; - totalRaised: string; - totalBeneficiaries: number; - totalManagers: number; - }[] = await models.ubiCommunityDailyState.findAll({ - attributes: [ - [ - literal('DISTINCT ON ("communityId") "communityId"'), - 'communityId', - ], - 'totalClaimed', - 'totalRaised', - 'totalBeneficiaries', - 'totalManagers', - ], - order: ['communityId', ['date', 'DESC']], - raw: true, - }); - // build communities object const communitiesState = communitiesStatePre.map( (c) => c.toJSON() as CommunityAttributes @@ -551,15 +525,6 @@ export async function calcuateCommunitiesMetrics(): Promise { const crb = communityRemovedBeneficiaryActivity.find( (c) => parseInt(c.id, 10) === communitiesState[index].id ); - const cnm = communityNewManagerActivity.find( - (c) => parseInt(c.id, 10) === communitiesState[index].id - ); - const crm = communityRemovedManagerActivity.find( - (c) => parseInt(c.id, 10) === communitiesState[index].id - ); - const pcm = previousState.find( - (c) => c.communityId === communitiesState[index].id - ); communities.push({ ...communitiesState[index], beneficiariesClaiming: cn @@ -575,14 +540,6 @@ export async function calcuateCommunitiesMetrics(): Promise { ...(cea ? cea : { volume: '0', txs: '0', reach: '0', reachOut: '0' }), - ...(pcm - ? pcm - : { - totalClaimed: '0', - totalRaised: '0', - totalBeneficiaries: 0, - totalManagers: 0, - }), fundingRate: cm ? new BigNumber(cm.raised) .minus(cm.claimed) @@ -597,13 +554,6 @@ export async function calcuateCommunitiesMetrics(): Promise { parseInt(crb.beneficiaries, 10), } : { beneficiaries: 0 }), - ...(cnm && crm - ? { - managers: - parseInt(cnm.managers, 10) - - parseInt(crm.managers, 10), - } - : { managers: 0 }), }, }); } @@ -611,17 +561,6 @@ export async function calcuateCommunitiesMetrics(): Promise { const calculateMetrics = async (community: ICommunityToMetrics) => { // if (community.activity !== undefined) { - const totalClaimed = community.activity.totalClaimed - ? new BigNumber(community.activity.totalClaimed).plus( - new BigNumber(community.activity.claimed) - ) - : '0'; - const totalRaised = community.activity.totalRaised - ? new BigNumber(community.activity.totalRaised).plus( - new BigNumber(community.activity.raised) - ) - : '0'; - await models.ubiCommunityDailyState.create({ transactions: parseInt(community.activity.txs, 10), reach: parseInt(community.activity.reach, 10), @@ -636,10 +575,6 @@ export async function calcuateCommunitiesMetrics(): Promise { beneficiaries: community.activity.beneficiaries, communityId: community.id, date: yesterday, - totalClaimed: totalClaimed.toString(), - totalRaised: totalRaised.toString(), - totalBeneficiaries: community.activity.totalBeneficiaries + community.activity.beneficiaries, - totalManagers: community.activity.totalManagers + community.activity.managers, }); } // if no activity, do not calculate diff --git a/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts b/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts index a2b4b24ea..77c73ac33 100644 --- a/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts +++ b/tests/integration/jobs/cron/calcuateCommunitiesMetrics.test.ts @@ -146,10 +146,6 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 0, - totalClaimed: '4000000000000000000', - totalRaised: '15000000000000000000', - totalBeneficiaries: 2, - totalManagers: 0, }); }); @@ -219,10 +215,6 @@ describe('calcuateCommunitiesMetrics', () => { claims: 0, fundingRate: 80, beneficiaries: 0, - totalClaimed: '4000000000000000000', - totalRaised: '15000000000000000000', - totalBeneficiaries: 2, - totalManagers: 0, }); }); @@ -306,10 +298,6 @@ describe('calcuateCommunitiesMetrics', () => { claims: 2, fundingRate: 80, beneficiaries: 0, - totalClaimed: '6000000000000000000', - totalRaised: '20000000000000000000', - totalBeneficiaries: 2, - totalManagers: 0, }); }); @@ -360,10 +348,6 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 0, - totalClaimed: '6000000000000000000', - totalRaised: '15000000000000000000', - totalBeneficiaries: 2, - totalManagers: 0, }); }); @@ -395,10 +379,6 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 2, - totalClaimed: '0', - totalRaised: '10000000000000000000', - totalBeneficiaries: 2, - totalManagers: 0, }); }); @@ -429,10 +409,6 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 2, - totalClaimed: '0', - totalRaised: '5000000000000000000', - totalBeneficiaries: 2, - totalManagers: 0, }); }); @@ -470,10 +446,6 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 2, - totalClaimed: '2000000000000000000', - totalRaised: '10000000000000000000', - totalBeneficiaries: 2, - totalManagers: 0, }); }); @@ -509,10 +481,6 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 2, - totalClaimed: '1000000000000000000', - totalRaised: '10000000000000000000', - totalBeneficiaries: 2, - totalManagers: 0, }); }); @@ -542,10 +510,6 @@ describe('calcuateCommunitiesMetrics', () => { communityId: communities[0].id, date: match.any, beneficiaries: 0, - totalClaimed: '0', - totalRaised: '5000000000000000000', - totalBeneficiaries: 0, - totalManagers: 0, }); }); }); @@ -663,10 +627,6 @@ describe('calcuateCommunitiesMetrics', () => { beneficiaries: -2, communityId: communities[0].id, date: match.any, - totalClaimed: '6000000000000000000', - totalRaised: '15000000000000000000', - totalBeneficiaries: 2, - totalManagers: 0, }); }); }); diff --git a/tests/integration/services/community.test.ts b/tests/integration/services/community.test.ts index 5497175ad..8e04d371f 100644 --- a/tests/integration/services/community.test.ts +++ b/tests/integration/services/community.test.ts @@ -758,8 +758,7 @@ describe('community service', () => { ]); for (const community of communities) { - await InflowFactory(community); - const beneficiaries = await BeneficiaryFactory( + await BeneficiaryFactory( await UserFactory({ n: community.requestByAddress === users[1].address @@ -768,16 +767,8 @@ describe('community service', () => { }), community.publicId ); - for (const beneficiary of beneficiaries) { - await ClaimFactory(beneficiary, community); - await ClaimFactory(beneficiary, community); - } } - tk.travel(jumpToTomorrowMidnight()); - - await calcuateCommunitiesMetrics(); - const result = await CommunityService.list({ orderBy: 'nearest:ASC;bigger:DESC', lat: '-15.8697203', @@ -785,23 +776,23 @@ describe('community service', () => { }); expect(result.rows[0]).to.include({ - id: communities[2].id, - name: communities[2].name, - country: communities[2].country, - requestByAddress: users[2].address, - }); - expect(result.rows[1]).to.include({ id: communities[1].id, name: communities[1].name, country: communities[1].country, requestByAddress: users[1].address, }); - expect(result.rows[2]).to.include({ + expect(result.rows[1]).to.include({ id: communities[0].id, name: communities[0].name, country: communities[0].country, requestByAddress: users[0].address, }); + expect(result.rows[2]).to.include({ + id: communities[2].id, + name: communities[2].name, + country: communities[2].country, + requestByAddress: users[2].address, + }); }); it('fewer beneficiaries and farthest', async () => { @@ -863,8 +854,7 @@ describe('community service', () => { ]); for (const community of communities) { - await InflowFactory(community); - const beneficiaries = await BeneficiaryFactory( + await BeneficiaryFactory( await UserFactory({ n: community.requestByAddress === users[2].address @@ -873,16 +863,8 @@ describe('community service', () => { }), community.publicId ); - for (const beneficiary of beneficiaries) { - await ClaimFactory(beneficiary, community); - await ClaimFactory(beneficiary, community); - } } - tk.travel(jumpToTomorrowMidnight()); - - await calcuateCommunitiesMetrics(); - const result = await CommunityService.list({ orderBy: 'bigger:ASC;nearest:DESC', lat: '-15.8697203', @@ -896,17 +878,17 @@ describe('community service', () => { requestByAddress: users[2].address, }); expect(result.rows[1]).to.include({ - id: communities[1].id, - name: communities[1].name, - country: communities[1].country, - requestByAddress: users[1].address, - }); - expect(result.rows[2]).to.include({ id: communities[0].id, name: communities[0].name, country: communities[0].country, requestByAddress: users[0].address, }); + expect(result.rows[2]).to.include({ + id: communities[1].id, + name: communities[1].name, + country: communities[1].country, + requestByAddress: users[1].address, + }); }); }); From 2cb6767ebf3efc004a16ef34b21ecc6f95ed9d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Thu, 2 Dec 2021 00:46:25 +0000 Subject: [PATCH 06/10] remove unused code --- src/services/ubi/community.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/services/ubi/community.ts b/src/services/ubi/community.ts index 62290aaa2..7b916fd0b 100644 --- a/src/services/ubi/community.ts +++ b/src/services/ubi/community.ts @@ -16,7 +16,6 @@ import { ManagerAttributes } from '@models/ubi/manager'; import { BaseError } from '@utils/baseError'; import { fetchData } from '@utils/dataFetching'; import { notifyManagerAdded } from '@utils/util'; -import BigNumber from 'bignumber.js'; import { ethers } from 'ethers'; import { Op, From 037e829094db1de7e0e4c33b2aee5a36ce7b2e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Thu, 2 Dec 2021 13:33:48 +0000 Subject: [PATCH 07/10] fix tests --- src/services/ubi/community.ts | 21 ++++++++++--- tests/integration/services/community.test.ts | 33 ++++++-------------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/services/ubi/community.ts b/src/services/ubi/community.ts index 7b916fd0b..1fe0c466b 100644 --- a/src/services/ubi/community.ts +++ b/src/services/ubi/community.ts @@ -318,7 +318,10 @@ export default class CommunityService { let extendedWhere: WhereOptions = {}; const orderOption: OrderItem[] = []; const orderBeneficiary: OrderItem[] = []; - let orderOutOfFunds: string | undefined; + let orderOutOfFunds = { + active: false, + orderType: '' + }; let beneficiariesState: | [ @@ -443,7 +446,8 @@ export default class CommunityService { communitiesId = result.map((el) => el.id); } else { // list communities out of funds after - orderOutOfFunds = orderType; + orderOutOfFunds.active = true; + orderOutOfFunds.orderType = orderType; } break; } @@ -455,7 +459,7 @@ export default class CommunityService { break; default: { // check if there was another order previously - if (orderOption.length === 0 && !orderOutOfFunds) { + if (orderOption.length === 0 && !orderOutOfFunds.active) { beneficiariesState = await this._getBeneficiaryState( { @@ -489,6 +493,9 @@ export default class CommunityService { }, extendedWhere ); + communitiesId = beneficiariesState!.map( + (el) => el.id + ); } let include: Includeable[]; @@ -532,6 +539,10 @@ export default class CommunityService { }, include, }); + // re-order + if (orderOption.length > 0) { + communitiesId = communitiesResult!.map((el) => el.id); + } } else { communitiesResult = await this.community.findAll({ attributes, @@ -552,13 +563,13 @@ export default class CommunityService { communitiesId = communitiesResult!.map((el) => el.id); } - if (orderOutOfFunds) { + if (orderOutOfFunds.active) { const result = await this._getOutOfFunds( { limit: query.limit, offset: query.offset, }, - undefined, + orderOutOfFunds.orderType, communitiesId ); // re-order by out of funds diff --git a/tests/integration/services/community.test.ts b/tests/integration/services/community.test.ts index 8e04d371f..ea40e4abc 100644 --- a/tests/integration/services/community.test.ts +++ b/tests/integration/services/community.test.ts @@ -2,7 +2,6 @@ import { expect } from 'chai'; import faker from 'faker'; import { Sequelize } from 'sequelize'; import Sinon, { assert, replace, spy } from 'sinon'; -import tk from 'timekeeper'; import { models } from '../../../src/database'; import { AppMediaContent } from '../../../src/interfaces/app/appMediaContent'; @@ -12,16 +11,13 @@ import { CommunityContentStorage } from '../../../src/services/storage'; import BeneficiaryService from '../../../src/services/ubi/beneficiary'; import CommunityService from '../../../src/services/ubi/community'; import ManagerService from '../../../src/services/ubi/managers'; -import { calcuateCommunitiesMetrics } from '../../../src/worker/jobs/cron/community'; import { verifyDeletedAccounts } from '../../../src/worker/jobs/cron/user'; import BeneficiaryFactory from '../../factories/beneficiary'; -import ClaimFactory from '../../factories/claim'; import CommunityFactory from '../../factories/community'; -import InflowFactory from '../../factories/inflow'; import ManagerFactory from '../../factories/manager'; import UserFactory from '../../factories/user'; import truncate, { sequelizeSetup } from '../../utils/sequelizeSetup'; -import { randomTx, jumpToTomorrowMidnight } from '../../utils/utils'; +import { randomTx } from '../../utils/utils'; // in this test there are users being assined with suspicious activity and others being removed describe('community service', () => { @@ -609,8 +605,7 @@ describe('community service', () => { ]); for (const community of communities) { - await InflowFactory(community); - const beneficiaries = await BeneficiaryFactory( + await BeneficiaryFactory( await UserFactory({ n: community.requestByAddress === users[0].address @@ -619,16 +614,8 @@ describe('community service', () => { }), community.publicId ); - for (const beneficiary of beneficiaries) { - await ClaimFactory(beneficiary, community); - await ClaimFactory(beneficiary, community); - } } - tk.travel(jumpToTomorrowMidnight()); - - await calcuateCommunitiesMetrics(); - const result = await CommunityService.list({}); expect(result.rows[0]).to.include({ @@ -872,10 +859,10 @@ describe('community service', () => { }); expect(result.rows[0]).to.include({ - id: communities[2].id, - name: communities[2].name, - country: communities[2].country, - requestByAddress: users[2].address, + id: communities[1].id, + name: communities[1].name, + country: communities[1].country, + requestByAddress: users[1].address, }); expect(result.rows[1]).to.include({ id: communities[0].id, @@ -884,10 +871,10 @@ describe('community service', () => { requestByAddress: users[0].address, }); expect(result.rows[2]).to.include({ - id: communities[1].id, - name: communities[1].name, - country: communities[1].country, - requestByAddress: users[1].address, + id: communities[2].id, + name: communities[2].name, + country: communities[2].country, + requestByAddress: users[2].address, }); }); }); From 8e48e69d3bdf5f90a2c9b5f562a4eb11ae3541e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Thu, 2 Dec 2021 13:51:02 +0000 Subject: [PATCH 08/10] remove unused code --- src/worker/jobs/cron/community.ts | 56 ------------------------------- 1 file changed, 56 deletions(-) diff --git a/src/worker/jobs/cron/community.ts b/src/worker/jobs/cron/community.ts index 019d04a6d..e55d7ce6a 100644 --- a/src/worker/jobs/cron/community.ts +++ b/src/worker/jobs/cron/community.ts @@ -87,10 +87,6 @@ export async function calcuateCommunitiesMetrics(): Promise { model: models.ubiCommunityContract, as: 'contract', }, - { - model: models.ubiCommunityState, - as: 'state', - }, { attributes: [], model: models.inflow, @@ -266,57 +262,6 @@ export async function calcuateCommunitiesMetrics(): Promise { raw: true, })) as any; - const communityNewManagerActivity: { - id: string; - managers: string; - }[] = (await models.community.findAll({ - attributes: ['id', [fn('count', col('managers.address')), 'managers']], - include: [ - { - model: models.manager, - as: 'managers', - attributes: [], - required: false, - where: { - createdAt: { [Op.between]: [yesterday, today] }, - }, - }, - ], - where: { - ...whereCommunity, - visibility: 'public', - } as any, - group: ['Community.id'], - order: [['id', 'DESC']], - raw: true, - })) as any; - - const communityRemovedManagerActivity: { - id: string; - managers: string; - }[] = (await models.community.findAll({ - attributes: ['id', [fn('count', col('managers.address')), 'managers']], - include: [ - { - model: models.manager, - as: 'managers', - attributes: [], - required: false, - where: { - updatedAt: { [Op.between]: [yesterday, today] }, - active: false, - }, - }, - ], - where: { - ...whereCommunity, - visibility: 'public', - }, - group: ['Community.id'], - order: [['id', 'DESC']], - raw: true, - })) as any; - const communityInflowActivity: { id: string; raised: string; @@ -579,7 +524,6 @@ export async function calcuateCommunitiesMetrics(): Promise { } // if no activity, do not calculate if ( - community.state === undefined || community.contract === undefined || community.beneficiaries === undefined || community.beneficiaries.length === 0 || From 27c7461b391b9d856ce6829b5d95062d41ae1f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Fri, 3 Dec 2021 23:06:09 +0100 Subject: [PATCH 09/10] remove ubiCommunityState --- .../migrations/z1636472208-update-triggers.js | 17 ---- src/database/migrations/zz-create-triggers.js | 2 - src/database/models/associations/community.ts | 5 -- src/database/models/index.ts | 6 +- src/database/models/ubi/communityState.ts | 90 ------------------- src/interfaces/ubi/ubiCommunityState.ts | 8 -- src/services/ubi/community.ts | 44 +++------ src/services/ubi/communityState.ts | 44 --------- src/types/db.ts | 6 +- tests/factories/community.ts | 2 - 10 files changed, 15 insertions(+), 209 deletions(-) delete mode 100644 src/database/models/ubi/communityState.ts delete mode 100644 src/services/ubi/communityState.ts diff --git a/src/database/migrations/z1636472208-update-triggers.js b/src/database/migrations/z1636472208-update-triggers.js index 608b92f10..a60e761bc 100644 --- a/src/database/migrations/z1636472208-update-triggers.js +++ b/src/database/migrations/z1636472208-update-triggers.js @@ -11,8 +11,6 @@ module.exports = { CREATE OR REPLACE FUNCTION update_inflow_community_states() RETURNS TRIGGER AS $$ declare - -- state_raised numeric(29); - -- state_daily_raised numeric(29); n_backer bigint; community_id integer; BEGIN @@ -26,11 +24,6 @@ module.exports = { IF n_backer = 0 THEN UPDATE ubi_community_state SET backers = backers + 1 WHERE "communityId"=community_id; end if; - -- update total raised - -- SELECT SUM(raised + NEW.amount) INTO state_raised FROM ubi_community_state WHERE "communityId"=community_id; - -- UPDATE ubi_community_state SET raised = state_raised WHERE "communityId"=community_id; - -- SELECT SUM(raised + NEW.amount) INTO state_daily_raised FROM ubi_community_daily_state WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); - -- UPDATE ubi_community_daily_state SET raised = state_daily_raised WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); return NEW; END; $$ LANGUAGE plpgsql;`); @@ -39,26 +32,16 @@ $$ LANGUAGE plpgsql;`); CREATE OR REPLACE FUNCTION update_claim_states() RETURNS TRIGGER AS $$ declare - -- state_claimed numeric(29); - -- state_daily_claimed numeric(29); beneficiary_claimed numeric(22); beneficiary_last_claim_at timestamp with time zone; community_public_id uuid; BEGIN SELECT "publicId" INTO community_public_id FROM community where id=NEW."communityId"; - -- update claims - UPDATE ubi_community_state SET claims = claims + 1 WHERE "communityId"=NEW."communityId"; - -- UPDATE ubi_community_daily_state SET claims = claims + 1 WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); -- update beneficiary table as well SELECT "lastClaimAt" INTO beneficiary_last_claim_at FROM beneficiary WHERE "communityId"=community_public_id AND address=NEW.address; UPDATE beneficiary SET claims = claims + 1, "penultimateClaimAt"=beneficiary_last_claim_at, "lastClaimAt"=NEW."txAt" WHERE "communityId"=community_public_id AND address=NEW.address; SELECT SUM(claimed + NEW.amount) INTO beneficiary_claimed FROM beneficiary WHERE "communityId"=community_public_id AND address=NEW.address; UPDATE beneficiary SET claimed = beneficiary_claimed WHERE "communityId"=community_public_id AND address=NEW.address; - -- update total claimed - -- SELECT SUM(claimed + NEW.amount) INTO state_claimed FROM ubi_community_state WHERE "communityId"=NEW."communityId"; - -- UPDATE ubi_community_state SET claimed = state_claimed WHERE "communityId"=NEW."communityId"; - -- SELECT SUM(claimed + NEW.amount) INTO state_daily_claimed FROM ubi_community_daily_state WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); - -- UPDATE ubi_community_daily_state SET claimed = state_daily_claimed WHERE "communityId"=community_id AND date=DATE(NEW."txAt"); return NEW; END; $$ LANGUAGE plpgsql;`); diff --git a/src/database/migrations/zz-create-triggers.js b/src/database/migrations/zz-create-triggers.js index 8b5a560da..ba7f9bee9 100644 --- a/src/database/migrations/zz-create-triggers.js +++ b/src/database/migrations/zz-create-triggers.js @@ -14,8 +14,6 @@ module.exports = { community_public_id uuid; BEGIN SELECT "publicId" INTO community_public_id FROM community where id=NEW."communityId"; - -- update claims - UPDATE ubi_community_state SET claims = claims + 1 WHERE "communityId"=NEW."communityId"; -- update beneficiary table as well SELECT "lastClaimAt" INTO beneficiary_last_claim_at FROM beneficiary WHERE "communityId"=community_public_id AND address=NEW.address; UPDATE beneficiary SET claims = claims + 1, "penultimateClaimAt"=beneficiary_last_claim_at, "lastClaimAt"=NEW."txAt" WHERE "communityId"=community_public_id AND address=NEW.address; diff --git a/src/database/models/associations/community.ts b/src/database/models/associations/community.ts index 48f0b6b52..45af87a3b 100644 --- a/src/database/models/associations/community.ts +++ b/src/database/models/associations/community.ts @@ -26,11 +26,6 @@ export function communityAssociation(sequelize: Sequelize) { constraints: false, }); // used to query from the community with incude - sequelize.models.Community.hasOne(sequelize.models.UbiCommunityStateModel, { - foreignKey: 'communityId', - as: 'state', - }); - // used to query from the community with incude // TODO: used only once, should be removed sequelize.models.Community.hasMany( sequelize.models.UbiCommunityDailyStateModel, diff --git a/src/database/models/index.ts b/src/database/models/index.ts index 46a62be52..e0829f717 100644 --- a/src/database/models/index.ts +++ b/src/database/models/index.ts @@ -5,7 +5,7 @@ import { initializeAppAnonymousReport } from './app/anonymousReport'; import { initializeAppMediaContent } from './app/appMediaContent'; import { initializeAppMediaThumbnail } from './app/appMediaThumbnail'; import { initializeAppNotification } from './app/appNotification'; -import { initializeUbiBeneficiarySurvey } from './ubi/ubiBeneficiarySurvey'; +import { initializeAppProposal } from './app/appProposal'; import { initializeAppUser } from './app/appUser'; import { initializeAppUserThroughTrust } from './app/appUserThroughTrust'; import { initializeAppUserTrust } from './app/appUserTrust'; @@ -32,11 +32,11 @@ import { initializeUbiCommunityContract } from './ubi/communityContract'; import { initializeUbiCommunityDailyMetrics } from './ubi/communityDailyMetrics'; import { initializeUbiCommunityDailyState } from './ubi/communityDailyState'; import { initializeUbiCommunityDemographics } from './ubi/communityDemographics'; -import { initializeUbiCommunityState } from './ubi/communityState'; import { initializeInflow } from './ubi/inflow'; import { initializeManager } from './ubi/manager'; import { initializeUbiRequestChangeParams } from './ubi/requestChangeParams'; import { initializeUbiBeneficiaryRegistry } from './ubi/ubiBeneficiaryRegistry'; +import { initializeUbiBeneficiarySurvey } from './ubi/ubiBeneficiarySurvey'; import { initializeUbiBeneficiaryTransaction } from './ubi/ubiBeneficiaryTransaction'; import { initializeUbiClaim } from './ubi/ubiClaim'; import { initializeUbiClaimLocation } from './ubi/ubiClaimLocation'; @@ -46,7 +46,6 @@ import { initializeUbiCommunityPromoter } from './ubi/ubiCommunityPromoter'; import { initializeUbiCommunitySuspect } from './ubi/ubiCommunitySuspect'; import { initializeUbiPromoter } from './ubi/ubiPromoter'; import { initializeUbiPromoterSocialMedia } from './ubi/ubiPromoterSocialMedia'; -import { initializeAppProposal } from './app/appProposal'; export default function initModels(sequelize: Sequelize): void { // app @@ -66,7 +65,6 @@ export default function initModels(sequelize: Sequelize): void { // ubi initializeCommunity(sequelize); - initializeUbiCommunityState(sequelize); initializeUbiCommunityContract(sequelize); initializeUbiCommunityDailyState(sequelize); initializeUbiCommunityDailyMetrics(sequelize); diff --git a/src/database/models/ubi/communityState.ts b/src/database/models/ubi/communityState.ts deleted file mode 100644 index 35d44c8f8..000000000 --- a/src/database/models/ubi/communityState.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { - UbiCommunityState, - UbiCommunityStateCreation, -} from '@interfaces/ubi/ubiCommunityState'; -import { Sequelize, DataTypes, Model } from 'sequelize'; - -export class UbiCommunityStateModel extends Model< - UbiCommunityState, - UbiCommunityStateCreation -> { - public communityId!: number; - public claimed!: string; - public claims!: number; - public beneficiaries!: number; - public removedBeneficiaries!: number; - public managers!: number; - public raised!: string; - public backers!: number; - - // timestamps! - public readonly createdAt!: Date; - public readonly updatedAt!: Date; -} - -export function initializeUbiCommunityState(sequelize: Sequelize): void { - UbiCommunityStateModel.init( - { - communityId: { - type: DataTypes.INTEGER, - primaryKey: true, - unique: true, - references: { - model: 'community', // name of Target model - key: 'id', // key in Target model that we're referencing - }, - onDelete: 'CASCADE', - allowNull: false, - }, - claimed: { - // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 - type: DataTypes.DECIMAL(29), // max 99,999,999,999 - plus 18 decimals - defaultValue: 0, - allowNull: false, - }, - claims: { - type: DataTypes.INTEGER, // max 2,147,483,647 - defaultValue: 0, - allowNull: false, - }, - beneficiaries: { - type: DataTypes.INTEGER, // max 2,147,483,647 - defaultValue: 0, - allowNull: false, - }, - removedBeneficiaries: { - type: DataTypes.INTEGER, // max 2,147,483,647 - defaultValue: 0, - allowNull: false, - }, - managers: { - type: DataTypes.INTEGER, // max 2,147,483,647 - defaultValue: 0, - allowNull: false, - }, - raised: { - // https://github.com/sequelize/sequelize/blob/2874c54915b2594225e939809ca9f8200b94f454/lib/dialects/postgres/data-types.js#L102 - type: DataTypes.DECIMAL(29), // max 99,999,999,999 - plus 18 decimals - defaultValue: 0, - allowNull: false, - }, - backers: { - type: DataTypes.INTEGER, // max 2,147,483,647 - defaultValue: 0, - allowNull: false, - }, - createdAt: { - type: DataTypes.DATE, - allowNull: false, - }, - updatedAt: { - type: DataTypes.DATE, - allowNull: false, - }, - }, - { - tableName: 'ubi_community_state', - sequelize, - } - ); -} diff --git a/src/interfaces/ubi/ubiCommunityState.ts b/src/interfaces/ubi/ubiCommunityState.ts index 546d6586d..3121feb00 100644 --- a/src/interfaces/ubi/ubiCommunityState.ts +++ b/src/interfaces/ubi/ubiCommunityState.ts @@ -48,12 +48,4 @@ export interface UbiCommunityState { managers?: number; raised: string; backers: number; - - // timestamps - createdAt?: Date; - updatedAt?: Date; -} - -export interface UbiCommunityStateCreation { - communityId: number; } diff --git a/src/services/ubi/community.ts b/src/services/ubi/community.ts index 1fe0c466b..cedce4aec 100644 --- a/src/services/ubi/community.ts +++ b/src/services/ubi/community.ts @@ -41,7 +41,6 @@ import { } from '../../types/endpoints'; import { CommunityContentStorage, PromoterContentStorage } from '../storage'; import CommunityContractService from './communityContract'; -import CommunityStateService from './communityState'; import ManagerService from './managers'; export default class CommunityService { @@ -49,7 +48,6 @@ export default class CommunityService { public static manager = models.manager; public static appUser = models.appUser; public static ubiCommunityContract = models.ubiCommunityContract; - public static ubiCommunityState = models.ubiCommunityState; public static ubiCommunityDailyMetrics = models.ubiCommunityDailyMetrics; public static ubiCommunityDailyState = models.ubiCommunityDailyState; public static ubiCommunityDemographics = models.ubiCommunityDemographics; @@ -154,11 +152,6 @@ export default class CommunityService { contractParams!, t ); - if (createObject.visibility === 'private') { - // in case it's public, will be added when accepted - await CommunityStateService.add(community.id, t); - // private communities don't need daily state - } // await CommunityStateService.add if (txReceipt !== undefined) { await ManagerService.add(managerAddress, community.publicId, t); @@ -318,9 +311,9 @@ export default class CommunityService { let extendedWhere: WhereOptions = {}; const orderOption: OrderItem[] = []; const orderBeneficiary: OrderItem[] = []; - let orderOutOfFunds = { + const orderOutOfFunds = { active: false, - orderType: '' + orderType: '', }; let beneficiariesState: @@ -459,7 +452,10 @@ export default class CommunityService { break; default: { // check if there was another order previously - if (orderOption.length === 0 && !orderOutOfFunds.active) { + if ( + orderOption.length === 0 && + !orderOutOfFunds.active + ) { beneficiariesState = await this._getBeneficiaryState( { @@ -493,9 +489,7 @@ export default class CommunityService { }, extendedWhere ); - communitiesId = beneficiariesState!.map( - (el) => el.id - ); + communitiesId = beneficiariesState!.map((el) => el.id); } let include: Includeable[]; @@ -762,10 +756,6 @@ export default class CommunityService { exclude: ['email'], }, include: [ - { - model: this.ubiCommunityState, - as: 'state', - }, { model: this.ubiCommunityContract, as: 'contract', @@ -791,6 +781,7 @@ export default class CommunityService { id, }, }); + const state = await this.getState(Number(id)); // add reachedLastMonth const aMonthAgo = new Date(); aMonthAgo.setDate(aMonthAgo.getDate() - 30); @@ -828,7 +819,7 @@ export default class CommunityService { model: models.ubiBeneficiaryTransaction, as: 'transactions', where: literal( - `date("beneficiaries->transactions"."date") = '${ + `date("beneficiaries->transactions"."txAt") = '${ aMonthAgo.toISOString().split('T')[0] }'` ), @@ -845,6 +836,7 @@ export default class CommunityService { })) as any; return { ...result!.toJSON(), + state, reachedLastMonth, } as CommunityAttributes & { reachedLastMonth: { @@ -1293,7 +1285,6 @@ export default class CommunityService { ' after acceptance!' ); } - await CommunityStateService.add(dbUpdate[1][0].id, t); // If the execution reaches this line, no errors were thrown. // We commit the transaction. await t.commit(); @@ -1398,10 +1389,6 @@ export default class CommunityService { model: this.ubiCommunityContract, as: 'contract', }, - { - model: this.ubiCommunityState, - as: 'state', - }, { model: this.appMediaContent, as: 'cover', @@ -1484,10 +1471,6 @@ export default class CommunityService { model: this.ubiCommunityContract, as: 'contract', }, - { - model: this.ubiCommunityState, - as: 'state', - }, { model: this.ubiCommunityDailyMetrics, // required: false, @@ -1677,12 +1660,7 @@ export default class CommunityService { 'Not found community ' + contractAddress ); } - const communityState = await this.ubiCommunityState.findOne({ - where: { - communityId: community.id, - }, - raw: true, - }); + const communityState = await this.getState(community.id); const communityContract = await this.ubiCommunityContract.findOne({ where: { communityId: community.id, diff --git a/src/services/ubi/communityState.ts b/src/services/ubi/communityState.ts deleted file mode 100644 index 6f1c4f085..000000000 --- a/src/services/ubi/communityState.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { UbiCommunityState } from '@interfaces/ubi/ubiCommunityState'; -import { QueryTypes, Transaction } from 'sequelize'; - -import { models, sequelize } from '../../database'; - -export default class CommunityStateService { - public static ubiCommunityState = models.ubiCommunityState; - public static sequelize = sequelize; - - public static async add( - communityId: number, - t: Transaction | undefined = undefined - ): Promise { - return await this.ubiCommunityState.create( - { - communityId, - }, - { transaction: t } - ); - } - - public static async get(communityId: number): Promise { - return (await this.ubiCommunityState.findOne({ - attributes: ['claimed', 'raised', 'beneficiaries', 'backers'], - where: { communityId }, - raw: true, - }))!; - } - - /** - * Only public valid - */ - public static getAllCommunitiesState(): Promise { - const query = `select "communityId", claimed, raised, beneficiaries, backers - from ubi_community_state cs , community c - where cs."communityId" = c.id - and c.status = 'valid' - and c.visibility = 'public'`; - - return this.sequelize.query(query, { - type: QueryTypes.SELECT, - }); - } -} diff --git a/src/types/db.ts b/src/types/db.ts index 6441b289e..2ca8f0535 100644 --- a/src/types/db.ts +++ b/src/types/db.ts @@ -2,7 +2,7 @@ import { AppAnonymousReportModel } from '@models/app/anonymousReport'; import { AppMediaContentModel } from '@models/app/appMediaContent'; import { AppMediaThumbnailModel } from '@models/app/appMediaThumbnail'; import { AppNotificationModel } from '@models/app/appNotification'; -import { UbiBeneficiarySurveyModel } from '@models/ubi/ubiBeneficiarySurvey'; +import { AppProposalModel } from '@models/app/appProposal'; import { AppUserModel } from '@models/app/appUser'; import { AppUserThroughTrustModel } from '@models/app/appUserThroughTrust'; import { AppUserTrustModel } from '@models/app/appUserTrust'; @@ -25,11 +25,11 @@ import { UbiCommunityContractModel } from '@models/ubi/communityContract'; import { UbiCommunityDailyMetricsModel } from '@models/ubi/communityDailyMetrics'; import { UbiCommunityDailyStateModel } from '@models/ubi/communityDailyState'; import { UbiCommunityDemographicsModel } from '@models/ubi/communityDemographics'; -import { UbiCommunityStateModel } from '@models/ubi/communityState'; import { Inflow } from '@models/ubi/inflow'; import { Manager } from '@models/ubi/manager'; import { UbiRequestChangeParamsModel } from '@models/ubi/requestChangeParams'; import { UbiBeneficiaryRegistryModel } from '@models/ubi/ubiBeneficiaryRegistry'; +import { UbiBeneficiarySurveyModel } from '@models/ubi/ubiBeneficiarySurvey'; import { UbiBeneficiaryTransactionModel } from '@models/ubi/ubiBeneficiaryTransaction'; import { UbiClaimModel } from '@models/ubi/ubiClaim'; import { ClaimLocationModel } from '@models/ubi/ubiClaimLocation'; @@ -39,7 +39,6 @@ import { UbiCommunitySuspectModel } from '@models/ubi/ubiCommunitySuspect'; import { UbiPromoterModel } from '@models/ubi/ubiPromoter'; import { UbiPromoterSocialMediaModel } from '@models/ubi/ubiPromoterSocialMedia'; import { ModelCtor, Sequelize } from 'sequelize/types'; -import { AppProposalModel } from '@models/app/appProposal'; export interface DbModels { appUser: ModelCtor; @@ -60,7 +59,6 @@ export interface DbModels { community: ModelCtor; ubiCommunitySuspect: ModelCtor; ubiCommunityContract: ModelCtor; - ubiCommunityState: ModelCtor; ubiCommunityDailyState: ModelCtor; ubiCommunityDailyMetrics: ModelCtor; ubiCommunityDemographics: ModelCtor; diff --git a/tests/factories/community.ts b/tests/factories/community.ts index 9a6ca8cb7..944f3bf64 100644 --- a/tests/factories/community.ts +++ b/tests/factories/community.ts @@ -6,7 +6,6 @@ import { CommunityAttributes, } from '../../src/database/models/ubi/community'; import { UbiCommunityContractModel } from '../../src/database/models/ubi/communityContract'; -import { UbiCommunityStateModel } from '../../src/database/models/ubi/communityState'; import { UbiCommunitySuspectModel } from '../../src/database/models/ubi/ubiCommunitySuspect'; import { UbiCommunityContract, @@ -102,7 +101,6 @@ const CommunityFactory = async (props: ICreateProps[]) => { ...(newCommunity.toJSON() as CommunityAttributes), contract: newContract.toJSON() as UbiCommunityContract, }); - await UbiCommunityStateModel.create({ communityId: newCommunity.id }); if (props[index].suspect !== undefined) { await UbiCommunitySuspectModel.create({ communityId: newCommunity.id, From ed7a6d719df926571f52aa4a5e62fefc38f723d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro?= Date: Fri, 3 Dec 2021 23:15:53 +0100 Subject: [PATCH 10/10] fix tests --- src/database/index.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/database/index.ts b/src/database/index.ts index 0950ad32d..6a02b24e6 100644 --- a/src/database/index.ts +++ b/src/database/index.ts @@ -25,7 +25,6 @@ import { UbiCommunityContractModel } from '@models/ubi/communityContract'; import { UbiCommunityDailyMetricsModel } from '@models/ubi/communityDailyMetrics'; import { UbiCommunityDailyStateModel } from '@models/ubi/communityDailyState'; import { UbiCommunityDemographicsModel } from '@models/ubi/communityDemographics'; -import { UbiCommunityStateModel } from '@models/ubi/communityState'; import { Inflow } from '@models/ubi/inflow'; import { Manager } from '@models/ubi/manager'; import { UbiRequestChangeParamsModel } from '@models/ubi/requestChangeParams'; @@ -88,8 +87,6 @@ const models: DbModels = { .UbiCommunitySuspectModel as ModelCtor, ubiCommunityContract: sequelize.models .UbiCommunityContractModel as ModelCtor, - ubiCommunityState: sequelize.models - .UbiCommunityStateModel as ModelCtor, ubiCommunityDailyState: sequelize.models .UbiCommunityDailyStateModel as ModelCtor, ubiCommunityDailyMetrics: sequelize.models