Skip to content

Commit

Permalink
Sushobhan/update/lenskit (#140)
Browse files Browse the repository at this point in the history 
In this PR, we have:

- updated `item_id` and `profile_id` in `evaluation/generate/worker.py`
- updated `evaluation/metrics` to use new LensKit metrics.

---------

Co-authored-by: Sushobhan Parajuli <sp3886@gmail.com>
  • Loading branch information
@sushobhan2024
sushobhan2024 and Sushobhan Parajuli authored Dec 9, 2024
1 parent 24e40b6 commit e37cfba
Show file tree
Hide file tree
Showing 12 changed files with 7,258 additions and 9,380 deletions.
94 changes: 47 additions & 47 deletions dvc.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,22 @@ stages:
size: 103456245
- path: outputs/mind-val/recommendations
hash: md5
md5: 7d1cad4d5fffb9e47e6abb62d2d27f16.dir
size: 44878647
nfiles: 8
md5: a5f7397ce4c38060e42a02551f2bcb47.dir
size: 44888273
nfiles: 4
- path: src/poprox_recommender/evaluation/evaluate.py
hash: md5
md5: 63f8d61ecc6708c41f542eba397c0d59
size: 4608
md5: 1fa966d8dbe31b297e098628e0e05812
size: 4614
outs:
- path: outputs/mind-val/metrics.csv
hash: md5
md5: 4885ed026f007b16fb6a39c48f9a9e68
size: 670
md5: 50a3a65a930866c0bc468620217d8320
size: 674
- path: outputs/mind-val/profile-metrics.csv.gz
hash: md5
md5: 40e34937389500c9ce74cb3d93d215df
size: 34000151
md5: 6c7b95588821fb2559f949d9b56b953c
size: 34027013
recommend-mind-val:
cmd: python -m poprox_recommender.evaluation.generate -M MINDlarge_dev -o outputs/mind-val
deps:
Expand All @@ -34,19 +34,19 @@ stages:
size: 103456245
- path: src/poprox_recommender/evaluation/generate/
hash: md5
md5: 5a40f035067811292117bdb23065a02a.dir
size: 31867
md5: 6f2992f4d0d735b09cc17410e6bad6e1.dir
size: 31876
nfiles: 8
outs:
- path: outputs/mind-val/embeddings.parquet
hash: md5
md5: 1de40da0c2aed9255997c20239c55f3f
size: 20621749
md5: da71c58527af174d3b58cb858e1300b8
size: 20620796
- path: outputs/mind-val/recommendations
hash: md5
md5: 7d1cad4d5fffb9e47e6abb62d2d27f16.dir
size: 44878647
nfiles: 8
md5: a5f7397ce4c38060e42a02551f2bcb47.dir
size: 44888273
nfiles: 4
recommend-mind-small:
cmd: python -m poprox_recommender.evaluation.generate -M MINDsmall_dev -o outputs/mind-small
deps:
Expand All @@ -56,19 +56,19 @@ stages:
size: 30948560
- path: src/poprox_recommender/evaluation/generate/
hash: md5
md5: 5a40f035067811292117bdb23065a02a.dir
size: 31867
md5: 6f2992f4d0d735b09cc17410e6bad6e1.dir
size: 31876
nfiles: 8
outs:
- path: outputs/mind-small/embeddings.parquet
hash: md5
md5: 153c2f32568ac61f1c07b6cdb0f951ba
size: 15967347
md5: ee025fd68418ffc418edbb49113d4d4f
size: 15964361
- path: outputs/mind-small/recommendations
hash: md5
md5: e3268d2d750b0b5f3e6c5afa5ba13453.dir
size: 9111609
nfiles: 8
md5: f54bae3d2cd57caa7d3a9bc8351afd61.dir
size: 8878494
nfiles: 4
measure-mind-small:
cmd: python -m poprox_recommender.evaluation.evaluate -M MINDsmall_dev mind-small
deps:
Expand All @@ -78,22 +78,22 @@ stages:
size: 103456245
- path: outputs/mind-small/recommendations
hash: md5
md5: e3268d2d750b0b5f3e6c5afa5ba13453.dir
size: 9111609
nfiles: 8
md5: f54bae3d2cd57caa7d3a9bc8351afd61.dir
size: 8878494
nfiles: 4
- path: src/poprox_recommender/evaluation/evaluate.py
hash: md5
md5: 63f8d61ecc6708c41f542eba397c0d59
size: 4608
md5: 1fa966d8dbe31b297e098628e0e05812
size: 4614
outs:
- path: outputs/mind-small/metrics.csv
hash: md5
md5: 4823af1d567142b8f1d628f2f0961589
size: 671
md5: 5474ab1fb9453d78824e623636dc65c6
size: 673
- path: outputs/mind-small/profile-metrics.csv.gz
hash: md5
md5: e1ddec19dc5ff6cf886186524fbf1a4c
size: 6606761
md5: 00b276998fd64237d27a26af0f14c462
size: 6610776
recommend-mind-subset:
cmd: python -m poprox_recommender.evaluation.generate -M MINDsmall_dev --subset=1000
-o outputs/mind-subset
Expand All @@ -104,19 +104,19 @@ stages:
size: 30948560
- path: src/poprox_recommender/evaluation/generate/
hash: md5
md5: 5a40f035067811292117bdb23065a02a.dir
size: 31867
md5: 6f2992f4d0d735b09cc17410e6bad6e1.dir
size: 31876
nfiles: 8
outs:
- path: outputs/mind-subset/embeddings.parquet
hash: md5
md5: bf5b727866ba25845198b5745d043d4f
size: 5128758
md5: e6f3322836f72186d3730168ea980563
size: 5128601
- path: outputs/mind-subset/recommendations
hash: md5
md5: c57e508e552b25f9ebc71e4a695b11ac.dir
size: 230729
nfiles: 8
md5: 28adc10a047a3baf74b145ef40f4bc93.dir
size: 151634
nfiles: 4
measure-mind-subset:
cmd: python -m poprox_recommender.evaluation.evaluate -M MINDsmall_dev mind-subset
deps:
Expand All @@ -126,19 +126,19 @@ stages:
size: 103456245
- path: outputs/mind-subset/recommendations
hash: md5
md5: c57e508e552b25f9ebc71e4a695b11ac.dir
size: 230729
nfiles: 8
md5: 28adc10a047a3baf74b145ef40f4bc93.dir
size: 151634
nfiles: 4
- path: src/poprox_recommender/evaluation/evaluate.py
hash: md5
md5: 63f8d61ecc6708c41f542eba397c0d59
size: 4608
md5: 1fa966d8dbe31b297e098628e0e05812
size: 4614
outs:
- path: outputs/mind-subset/metrics.csv
hash: md5
md5: cf0b1f2b4d185d4d479a2b5995ee40f6
md5: 1a27df3220279344d9d685c3847dfd98
size: 673
- path: outputs/mind-subset/profile-metrics.csv.gz
hash: md5
md5: 4a658459093af34e63ec8905f64b8146
size: 90608
md5: c26a89abe3caf41fa96ef9e50bb7f141
size: 90902
10 changes: 5 additions & 5 deletions outputs/mind-small/metrics.csv
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
recommender,NDCG@5,NDCG@10,MRR,RBO@5,RBO@10
locality-cali,0.4171536059941225,0.48251905284633967,0.37054744784506105,0.3911296293061023,0.6076199467392752
locality-cali,0.4171536059941225,0.48251905284633967,0.37054744784506105,0.3911298784448818,0.6076202266288985
mmr,0.35579750600797894,0.4981208173313458,0.31145743826292543,0.22695393184055115,0.32274881943529315
nrms,0.4171357416899212,0.4825323850285321,0.3706339062217917,0.39113024753937,0.6076209216365874
pfar,0.4171536059941225,0.48251905284633967,0.37054744784506105,0.3911296293061023,0.6076199467392752
softmax,0.3833981720081835,0.45081795059531876,0.3321380617353386,0.20969089732338142,0.3722051551606116
topic-cali,0.4171536059941225,0.48251905284633967,0.37054744784506105,0.3911296293061023,0.6076199467392752
nrms,0.41681471151538513,0.48223014369342754,0.3704193063106695,0.39113024753937,0.6076209216365874
pfar,0.4171536059941225,0.48251905284633967,0.37054744784506105,0.3911298784448818,0.6076202266288985
softmax,0.383472073372685,0.45112788158494194,0.3330375749472288,0.20964197587215655,0.37225452926811303
topic-cali,0.4171536059941225,0.48251905284633967,0.37054744784506105,0.3911298784448818,0.6076202266288985
4 changes: 2 additions & 2 deletions outputs/mind-subset/metrics.csv
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
recommender,NDCG@5,NDCG@10,MRR,RBO@5,RBO@10
locality-cali,0.41258029427212617,0.4799584099761152,0.3773281746031746,0.38897697799999986,0.6038961667861186
mmr,0.36260738955375116,0.5197700868114026,0.3196722222222222,0.22704047299999996,0.3233538800333043
nrms,0.4095967741826367,0.4761685169507715,0.3724476190476191,0.38897697799999986,0.6038961667861186
nrms,0.4104521021399195,0.4773817870867811,0.3760670634920635,0.38897697799999986,0.6038961667861186
pfar,0.41258029427212617,0.4799584099761152,0.3773281746031746,0.38897697799999986,0.6038961667861186
softmax,0.3854226917588238,0.45140865141914105,0.3379642857142857,0.21319334099999995,0.37530310939543277
softmax,0.40128548396432234,0.4650886829823084,0.34703531746031746,0.20744908699999998,0.3696869246064028
topic-cali,0.41258029427212617,0.4799584099761152,0.3773281746031746,0.38897697799999986,0.6038961667861186
12 changes: 6 additions & 6 deletions outputs/mind-val/metrics.csv
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
recommender,NDCG@5,NDCG@10,MRR,RBO@5,RBO@10
locality-cali,0.41784484004402095,0.4829450574043462,0.37157370476437546,0.3912673030485747,0.6078328581766465
mmr,0.3589295721326262,0.5028667882975844,0.31276849335092866,0.22722075092370989,0.3233362905715243
nrms,0.4178254938949696,0.4829404987679788,0.3715477262461625,0.39126863333961964,0.6078344546963851
pfar,0.41784484004402095,0.4829450574043462,0.37157370476437546,0.3912673030485747,0.6078328581766465
softmax,0.3828735992908713,0.45077278535229554,0.33250694429424,0.21017851942380683,0.3729055717830188
topic-cali,0.41784484004402095,0.4829450574043462,0.37157370476437546,0.3912673030485747,0.6078328581766465
locality-cali,0.41784484004402095,0.482945068169885,0.37157370476437546,0.3912679157704045,0.6078335362712364
mmr,0.3589295721326262,0.5028667882975845,0.31276849335092866,0.22722075092370989,0.32333626754560724
nrms,0.41780612275433043,0.48297561184714516,0.3715650508799915,0.39126863333961964,0.6078344546963851
pfar,0.41784484004402095,0.482945068169885,0.37157370476437546,0.3912679157704045,0.6078335362712364
softmax,0.3828069016307366,0.45078525476124115,0.33283129314583265,0.21018380654021154,0.37289327365206193
topic-cali,0.41784484004402095,0.482945068169885,0.37157370476437546,0.3912679157704045,0.6078335362712364
Loading

0 comments on commit e37cfba

Please sign in to comment.