feat: trending queries

Makefile

@@ -1,47 +1,20 @@
-BRANCH_NAME := $(shell git rev-parse --abbrev-ref HEAD)
-DOCKER_COMPOSE := BRANCH_NAME=$(BRANCH_NAME) docker compose
-
-DEV_NET_ENDPOINT = ws://100.28.51.29:9945
-DEV_NET = chain_endpoint $(DEV_NET_ENDPOINT)
-
 TESTNET = network test
 MAINNET = network finney
 
 ########################################################################
 #####                       SELECT YOUR ENV                        #####
 ########################################################################
 
-# SUBTENSOR_ENVIRONMENT = $(DEV_NET)
 # SUBTENSOR_ENVIRONMENT = $(TESTNET)
 SUBTENSOR_ENVIRONMENT = $(MAINNET)
 
-# NETUID = 1 # devnet
 # NETUID = 165 # testnet
 NETUID = 42 # mainnet
 
 ########################################################################
 #####                       USEFUL COMMANDS                        #####
 ########################################################################
 
-## Wallet funding
-fund-owner-wallet:
-	btcli wallet faucet --wallet.name owner --subtensor.$(SUBTENSOR_ENVIRONMENT)
-
-fund-validator-wallet:
-	btcli wallet faucet --wallet.name validator --subtensor.$(SUBTENSOR_ENVIRONMENT)
-
-fund-miner-wallet:
-	btcli wallet faucet --wallet.name miner --subtensor.$(SUBTENSOR_ENVIRONMENT)
-
-## Send TAO
-send:
-	btcli w transfer --subtensor.$(SUBTENSOR_ENVIRONMENT)
-
-## Subnet creation
-create-subnet:
-	btcli subnet create --wallet.name owner --subtensor.$(SUBTENSOR_ENVIRONMENT)
-
-## Subnet and wallet info
 list-wallets:
 	btcli wallet list
 
@@ -54,123 +27,41 @@ balance-all:
 list-subnets:
 	btcli subnets list --subtensor.$(SUBTENSOR_ENVIRONMENT)
 
-## Validator setup
-stake-validator:
-	btcli stake add --wallet.name validator --wallet.hotkey default --subtensor.$(SUBTENSOR_ENVIRONMENT) --netuid $(NETUID)
+register-miner:
+	btcli subnet register --wallet.name miner --wallet.hotkey default --subtensor.$(SUBTENSOR_ENVIRONMENT) --netuid $(NETUID)
 
 register-validator:
 	btcli subnet register --wallet.name validator --wallet.hotkey default --subtensor.$(SUBTENSOR_ENVIRONMENT) --netuid $(NETUID)
 
 register-validator-root:
 	btcli root register --wallet.name validator --wallet.hotkey default --subtensor.$(SUBTENSOR_ENVIRONMENT)
 
-## Register miner + Key Registration Validation
-register-miner:
-	btcli subnet register --wallet.name miner --wallet.hotkey default --subtensor.$(SUBTENSOR_ENVIRONMENT) --netuid $(NETUID)
-
-validate-key-registration:
-	btcli subnet list --subtensor.$(SUBTENSOR_ENVIRONMENT)
+stake-validator:
+	btcli stake add --wallet.name validator --wallet.hotkey default --subtensor.$(SUBTENSOR_ENVIRONMENT) --netuid $(NETUID)
 
-## Setup weights
 boost-root:
 	btcli root boost --netuid $(NETUID) --increase 1 --wallet.name validator --wallet.hotkey default --subtensor.$(SUBTENSOR_ENVIRONMENT)
 
 set-weights:
 	btcli root weights --subtensor.$(SUBTENSOR_ENVIRONMENT)
 
-## Run miner and validator
 run-miner:
 	python neurons/miner.py --netuid $(NETUID) --subtensor.$(SUBTENSOR_ENVIRONMENT) --wallet.name miner --wallet.hotkey default --axon.port 8091 --neuron.debug --logging.debug --blacklist.force_validator_permit
 
 run-validator:
 	python neurons/validator.py --netuid $(NETUID) --subtensor.$(SUBTENSOR_ENVIRONMENT) --wallet.name validator --wallet.hotkey default --axon.port 8092 --neuron.debug --logging.debug --neuron.axon_off
 
-## Docker commands
-docker-build:
-	docker build -f Dockerfile.masa -t masa-subtensor .
-
-docker-run:
-	docker run -d --name masa-subtensor -p 30333:30333 -p 9933:9933 -p 9944:9944 -p 9945:9945 -p 9946:9946 masa-subtensor
-
-docker-run-remote:
-	docker run -d --name masa-subtensor -p 30333:30333 -p 9933:9933 -p 9944:9944 -p 9945:9945 -p 9946:9946 ghcr.io/masa-finance/subtensor:arm-latest
-
-
-########################################################################
-#####                       VALIDATOR API                          #####
-########################################################################
-
-test-profile:
-	curl -X GET "http://localhost:8000/data/twitter/brendanplayford" -H "Authorization: Bearer 1234"
-
-########################################################################
-########################################################################
-
-
-## Helpful commands (generally don't run well on their own with Makefiles)
-## because each line in recipe runs on its own shell invocation
-activate-environment:
-	conda activate bittensor
-bittensor-path-export:
-	export PYTHONPATH=$(BITTENSOR_PATH)
-########################################################################
-
-## Subtensor repo related commands (not needed here)
-build-binary:
-	cargo build --release --features pow-faucet
-
-run-localnet:
-	BUILD_BINARY=0 ./scripts/localnet.sh
-
-## Hyperparameters and metagraph
 hyperparameters:
 	btcli subnets hyperparameters --subtensor.$(SUBTENSOR_ENVIRONMENT) --netuid $(NETUID)
 
 metagraph:
 	btcli subnets metagraph --subtensor.$(SUBTENSOR_ENVIRONMENT) --netuid $(NETUID)
 
-set-hyperparam:
-	btcli sudo set --param weights_version --value 10 --netuid $(NETUID) --subtensor.$(SUBTENSOR_ENVIRONMENT)
-
-########################################################################
-#####                   DOCKER COMPOSE COMMANDS                    #####
-########################################################################
-
-.PHONY: up down build logs
-
-pull:
-	$(DOCKER_COMPOSE) pull
-
-up:
-	$(DOCKER_COMPOSE) up -d --pull always
-
-down:
-	$(DOCKER_COMPOSE) down
-
-build:
-	$(DOCKER_COMPOSE) build
-
-logs:
-	$(DOCKER_COMPOSE) logs -f
-
-# You can keep your existing docker commands or replace them with these:
-docker-build:
-	$(DOCKER_COMPOSE) build
-
-docker-up:
-	$(DOCKER_COMPOSE) up -d --pull always
-
-docker-down:
-	$(DOCKER_COMPOSE) down
-
-test-mocks:
-	pytest -s -p no:warnings tests/test_mocks.py
-
-test-utils:
-	pytest -s -p no:warnings tests/test_utils.py
-
 test-miner:
 	pytest -s -p no:warnings tests/test_miner.py
 
+test-validator:
+	pytest -s -p no:warnings tests/test_validator.py
+
 test-all:
-	pytest -s -p no:warnings tests
+	pytest -s -p no:warnings tests/test_miner.py tests/test_validator.py

masa/api/server.py

@@ -49,52 +49,6 @@ def __init__(self, validator, config=None):
             tags=["twitter"],
         )
 
-        # self.app.add_api_route(
-        #     "/data/discord/profile",
-        #     self.validator.forwarder.get_discord_profile,
-        #     methods=["GET"],
-        #     dependencies=[Depends(self.get_self)],
-        #     response_description="Get the Discord profile for the given user ID",
-        #     tags=["discord"],
-        # )
-
-        # self.app.add_api_route(
-        #     "/data/discord/channels/{channel_id}/messages",
-        #     self.validator.forwarder.get_discord_channel_messages,
-        #     methods=["GET"],
-        #     dependencies=[Depends(self.get_self)],
-        #     response_description="Get the Discord channel messages for the given channel ID",
-        #     tags=["discord"],
-        # )
-
-        # self.app.add_api_route(
-        #     "/data/discord/guilds/{guild_id}/channels",
-        #     self.validator.forwarder.get_discord_guild_channels,
-        #     methods=["GET"],
-        #     dependencies=[Depends(self.get_self)],
-        #     response_description="Get the Discord channels for the given guild ID",
-        #     tags=["discord"],
-        # )
-
-        # self.app.add_api_route(
-        #     "/data/discord/user/guilds",
-        #     self.validator.forwarder.get_discord_user_guilds,
-        #     methods=["GET"],
-        #     dependencies=[Depends(self.get_self)],
-        #     response_description="Get the Discord guilds for the user",
-        #     tags=["discord"],
-        # )
-
-        # self.app.add_api_route(
-        #     "/data/discord/guilds/all",
-        #     self.validator.forwarder.get_discord_all_guilds,
-        #     methods=["GET"],
-        #     dependencies=[Depends(self.get_self)],
-        #     response_description="Get all guilds that all the Discord workers are apart of",
-        #     tags=["discord"],
-        # )
-
-        # note, healthcheck for the validator
         self.app.add_api_route(
             "/healthcheck",
             self.healthcheck,
@@ -131,27 +85,6 @@ def __init__(self, validator, config=None):
             tags=["scoring"],
         )
 
-        # note, only for testing / wiping state
-        # self.app.add_api_route(
-        #     "/volumes",
-        #     self.delete_miner_volumes,
-        #     methods=["DELETE"],
-        #     dependencies=[Depends(self.get_self)],
-        #     response_description="Delete volumes state",
-        #     tags=["scoring"],
-        # )
-
-        # note, only for testing, this also runs on a dedciated thread
-        # self.app.add_api_route(
-        #     "/score",
-        #     self.validator.scorer.score_miner_volumes,
-        #     methods=["GET"],
-        #     dependencies=[Depends(self.get_self)],
-        #     response_description="Score miner volumes",
-        #     tags=["scoring"],
-        # )
-
-        # note, show the scores the validator has computed
         self.app.add_api_route(
             "/scores",
             self.show_scores,
@@ -162,23 +95,22 @@ def __init__(self, validator, config=None):
         )
 
         self.app.add_api_route(
-            "/tweets",
-            self.show_indexed_tweets,
+            "/tweets_by_query",
+            self.show_tweets_by_query,
             methods=["GET"],
             dependencies=[Depends(self.get_self)],
-            response_description="Get indexed tweets",
+            response_description="Get indexed tweets by query",
             tags=["data"],
         )
 
-        # note, only for testing / wiping state
-        # self.app.add_api_route(
-        #     "/tweets",
-        #     self.delete_indexed_tweets,
-        #     methods=["DELETE"],
-        #     dependencies=[Depends(self.get_self)],
-        #     response_description="Delete indexed tweets",
-        #     tags=["data"],
-        # )
+        self.app.add_api_route(
+            "/tweets_by_uid",
+            self.show_tweets_by_uid,
+            methods=["GET"],
+            dependencies=[Depends(self.get_self)],
+            response_description="Get indexed tweets by UID",
+            tags=["data"],
+        )
 
         self.start_server()
 
@@ -201,29 +133,20 @@ async def show_scores(self):
             return JSONResponse(content=scores.tolist())
         return JSONResponse(content=[])
 
-    async def show_indexed_tweets(self):
-        tweets = self.validator.indexed_tweets
+    async def show_tweets_by_query(self):
+        tweets = self.validator.tweets_by_query
         if len(tweets) > 0:
             return JSONResponse(content=tweets)
         return JSONResponse(content=[])
 
-    def delete_miner_volumes(self):
-        self.validator.volumes = []
-        return JSONResponse(
-            content={
-                "message": "Volumes state deleted",
-                "volumes": self.validator.volumes,
-            }
-        )
-
-    def delete_indexed_tweets(self):
-        self.validator.indexed_tweets = []
-        return JSONResponse(
-            content={
-                "message": "Index tweets deleted",
-                "volumes": self.validator.indexed_tweets,
+    async def show_tweets_by_uid(self):
+        tweets = self.validator.tweets_by_uid
+        if len(tweets) > 0:
+            serializable_tweets = {
+                uid: list(tweet_set) for uid, tweet_set in tweets.items()
             }
-        )
+            return JSONResponse(content=serializable_tweets)
+        return JSONResponse(content=[])
 
     def get_axons(self):
         return self.validator.metagraph.axons

masa/base/healthcheck.py

@@ -2,6 +2,8 @@
 import requests
 import bittensor as bt
 
+# TODO we can refactor this to synapses directory, as both vali and miner use
+
 
 class PingAxonSynapse(bt.Synapse):
     sent_from: typing.Optional[str]

masa/base/miner.py

@@ -33,7 +33,7 @@
 
 from masa.miner.twitter.profile import forward_twitter_profile
 from masa.miner.twitter.followers import forward_twitter_followers
-from masa.miner.twitter.tweets import forward_recent_tweets
+from masa.miner.twitter.tweets import forward_recent_tweets, RecentTweetsSynapse
 
 
 class BaseMinerNeuron(BaseNeuron):
@@ -86,7 +86,7 @@ def __init__(self, config=None):
         )
 
         self.axon.attach(
-            forward_fn=forward_recent_tweets,
+            forward_fn=self.forward_tweets_synapse,
             blacklist_fn=self.blacklist_recent_tweets,
             priority_fn=self.priority_recent_tweets,
         )
@@ -109,6 +109,13 @@ def __init__(self, config=None):
 
         self.load_state()
 
+    def forward_tweets_synapse(
+        self, synapse: RecentTweetsSynapse
+    ) -> RecentTweetsSynapse:
+        return forward_recent_tweets(
+            synapse, self.config.twitter.max_tweets_per_request
+        )
+
     def forward_ping_synapse(self, synapse: PingAxonSynapse) -> PingAxonSynapse:
         return forward_ping(synapse, self.spec_version)