From aba6e8232a69ea3dc7963835ce65fbb239409030 Mon Sep 17 00:00:00 2001 From: chainsawriot Date: Sun, 11 Jun 2023 16:03:19 +0200 Subject: [PATCH] Correct all docs [no ci] --- .Rbuildignore | 1 + R/oolong.R | 4 ++-- R/oolong_summary.R | 2 +- btm_gh.md | 4 ++-- deploy_gh.md | 4 ++-- man/create_oolong.Rd | 4 ++-- man/summarize_oolong.Rd | 2 +- overview_gh.Rmd | 4 ++-- overview_gh.md | 32 ++++++++++++++++---------------- vig_body.Rmd | 4 ++-- vignettes/overview.Rmd | 4 ++-- 11 files changed, 33 insertions(+), 32 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index 9dca73c..a5206aa 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -40,3 +40,4 @@ ^CODE_OF_CONDUCT\.md$ ^\.github$ ^rawdata/ +^CRAN-SUBMISSION$ diff --git a/R/oolong.R b/R/oolong.R index 7fdf984..d5f6cd5 100644 --- a/R/oolong.R +++ b/R/oolong.R @@ -42,7 +42,7 @@ Oolong_test <- R6::R6Class( #' @section Usage: #' #' Use \code{wi}, \code{ti}, \code{witi}, \code{wsi} or \code{gs} to generate an oolong test of your choice. It is recommended to supply also \code{userid} (current coder). -#' The names of the tests (word intrusion test and topic intrusion test) follow Chang et al (2009). In Ying et al. (forthcoming), topic intrusion test is named "T8WSI" (Top 8 Word Set Intrusion). Word set intrusion test in this package is actually the "R4WSI" (Random 4 Word Set Intrusion) in Lu et al (forthcoming). The default settings of \code{wi}, \code{witi}, and \code{ti} follow Chang et al (2009), e.g. \code{n_top_terms} = 5; instead of \code{n_top_terms} = 4 as in Lu et al (forthcoming). The default setting of \code{wsi} follows Ying et al. (forthcoming), e.g. \code{n_topiclabel_words} = 4. +#' The names of the tests (word intrusion test and topic intrusion test) follow Chang et al (2009). In Ying et al. (2021), topic intrusion test is named "T8WSI" (Top 8 Word Set Intrusion). Word set intrusion test in this package is actually the "R4WSI" (Random 4 Word Set Intrusion) in Ying et al. The default settings of \code{wi}, \code{witi}, and \code{ti} follow Chang et al (2009), e.g. \code{n_top_terms} = 5; instead of \code{n_top_terms} = 4 as in Ying et al. The default setting of \code{wsi} follows Ying et al., e.g. \code{n_topiclabel_words} = 4. #' As suggested by Song et al. (2020), 1% of the articles from \code{input_corpus} are randomly selected as the test cases of both \code{ti} and \code{gs}, i.e. \code{frac} = 0.01. However, it is generally believed that this proportion is dependent of the size of \code{input_corpus}, e.g. it does not make sense to draw 1% of the articles from only 100 articles. Use \code{exact_n} in these cases. #' @section About create_oolong: #' @@ -102,7 +102,7 @@ Oolong_test <- R6::R6Class( #' #' Song et al. (2020) In validations we trust? The impact of imperfect human annotations as a gold standard on the quality of validation of automated content analysis. Political Communication. #' -#' Ying, L., Montgomery, J. M., & Stewart, B. M. (Forthcoming). Inferring concepts from topics: Towards procedures for validating topics as measures. Political Analysis +#' Ying, L., Montgomery, J. M., & Stewart, B. M. (2021). Topics, Concepts, and Measurement: A Crowdsourced Procedure for Validating Topics as Measures. Political Analysis #' #' @export create_oolong <- function(input_model = NULL, input_corpus = NULL, n_top_terms = 5, bottom_terms_percentile = 0.6, exact_n = NULL, frac = 0.01, n_top_topics = 3, n_topiclabel_words = 8, use_frex_words = FALSE, difficulty = 1, input_dfm = NULL, construct = "positive", btm_dataframe = NULL, n_correct_ws = 3, wsi_n_top_terms = 20, userid = NA, type = "witi") { diff --git a/R/oolong_summary.R b/R/oolong_summary.R index ff9d102..ce660cb 100644 --- a/R/oolong_summary.R +++ b/R/oolong_summary.R @@ -130,7 +130,7 @@ plot.oolong_summary <- function(x, ...) { #' #' Song et al. (2020) In validations we trust? The impact of imperfect human annotations as a gold standard on the quality of validation of automated content analysis. Political Communication. #' -#' Ying, L., Montgomery, J. M., & Stewart, B. M. (Forthcoming). Inferring concepts from topics: Towards procedures for validating topics as measures. Political Analysis. +#' Ying, L., Montgomery, J. M., & Stewart, B. M. (2021). Topics, Concepts, and Measurement: A Crowdsourced Procedure for Validating Topics as Measures. Political Analysis. #' @export summarize_oolong <- function(..., target_value = NULL, n_iter = 1500) { obj_list <- list(...) diff --git a/btm_gh.md b/btm_gh.md index 31374fe..a64ab28 100644 --- a/btm_gh.md +++ b/btm_gh.md @@ -118,7 +118,7 @@ with other topic models. oolong <- create_oolong(trump_btm) oolong #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✔ WI ✖ TI ✖ WSI #> ℹ WI: k = 8, 0 coded. #> @@ -136,7 +136,7 @@ frame you used for training (in this case `trump_dat`). Your oolong <- create_oolong(trump_btm, trump_corpus, btm_dataframe = trump_dat) oolong #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✔ WI ✔ TI ✖ WSI #> ℹ WI: k = 8, 0 coded. #> ℹ TI: n = 20, 0 coded. diff --git a/deploy_gh.md b/deploy_gh.md index 2769eab..390ed7c 100644 --- a/deploy_gh.md +++ b/deploy_gh.md @@ -27,7 +27,7 @@ library(oolong) wsi_test <- wsi(abstracts_keyatm) wsi_test #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✖ WI ✖ TI ✔ WSI #> ℹ WSI: n = 10, 0 coded. #> @@ -116,7 +116,7 @@ revert_oolong(wsi_test, "oolong_2021-05-22 20 51 26 Hadley Wickham.RDS") ``` #> - #> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + #> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✖ WI ✖ TI ✔ WSI #> ☺ Hadley Wickham #> ℹ WSI: n = 10, 10 coded. diff --git a/man/create_oolong.Rd b/man/create_oolong.Rd index 9acbb7d..03a6ca2 100644 --- a/man/create_oolong.Rd +++ b/man/create_oolong.Rd @@ -131,7 +131,7 @@ an oolong test object. Use \code{wi}, \code{ti}, \code{witi}, \code{wsi} or \code{gs} to generate an oolong test of your choice. It is recommended to supply also \code{userid} (current coder). -The names of the tests (word intrusion test and topic intrusion test) follow Chang et al (2009). In Ying et al. (forthcoming), topic intrusion test is named "T8WSI" (Top 8 Word Set Intrusion). Word set intrusion test in this package is actually the "R4WSI" (Random 4 Word Set Intrusion) in Lu et al (forthcoming). The default settings of \code{wi}, \code{witi}, and \code{ti} follow Chang et al (2009), e.g. \code{n_top_terms} = 5; instead of \code{n_top_terms} = 4 as in Lu et al (forthcoming). The default setting of \code{wsi} follows Ying et al. (forthcoming), e.g. \code{n_topiclabel_words} = 4. +The names of the tests (word intrusion test and topic intrusion test) follow Chang et al (2009). In Ying et al. (2021), topic intrusion test is named "T8WSI" (Top 8 Word Set Intrusion). Word set intrusion test in this package is actually the "R4WSI" (Random 4 Word Set Intrusion) in Lu et al (forthcoming). The default settings of \code{wi}, \code{witi}, and \code{ti} follow Chang et al (2009), e.g. \code{n_top_terms} = 5; instead of \code{n_top_terms} = 4 as in Lu et al (forthcoming). The default setting of \code{wsi} follows Ying et al. (forthcoming), e.g. \code{n_topiclabel_words} = 4. As suggested by Song et al. (2020), 1% of the articles from \code{input_corpus} are randomly selected as the test cases of both \code{ti} and \code{gs}, i.e. \code{frac} = 0.01. However, it is generally believed that this proportion is dependent of the size of \code{input_corpus}, e.g. it does not make sense to draw 1% of the articles from only 100 articles. Use \code{exact_n} in these cases. } @@ -180,7 +180,7 @@ Chang, J., Gerrish, S., Wang, C., Boyd-Graber, J. L., & Blei, D. M. (2009). Read Song et al. (2020) In validations we trust? The impact of imperfect human annotations as a gold standard on the quality of validation of automated content analysis. Political Communication. - Ying, L., Montgomery, J. M., & Stewart, B. M. (Forthcoming). Inferring concepts from topics: Towards procedures for validating topics as measures. Political Analysis + Ying, L., Montgomery, J. M., & Stewart, B. M. (2021). Topics, Concepts, and Measurement: A Crowdsourced Procedure for Validating Topics as Measures. Political Analysis } \author{ Chung-hong Chan, Marius Sältzer diff --git a/man/summarize_oolong.Rd b/man/summarize_oolong.Rd index 2303a40..c86508f 100644 --- a/man/summarize_oolong.Rd +++ b/man/summarize_oolong.Rd @@ -54,7 +54,7 @@ Chang, J., Gerrish, S., Wang, C., Boyd-Graber, J. L., & Blei, D. M. (2009). Read Song et al. (2020) In validations we trust? The impact of imperfect human annotations as a gold standard on the quality of validation of automated content analysis. Political Communication. - Ying, L., Montgomery, J. M., & Stewart, B. M. (Forthcoming). Inferring concepts from topics: Towards procedures for validating topics as measures. Political Analysis. + Ying, L., Montgomery, J. M., & Stewart, B. M. (2021). Topics, Concepts, and Measurement: A Crowdsourced Procedure for Validating Topics as Measures. Political Analysis. } \author{ Chung-hong Chan diff --git a/overview_gh.Rmd b/overview_gh.Rmd index 2a56457..7b16818 100644 --- a/overview_gh.Rmd +++ b/overview_gh.Rmd @@ -341,7 +341,7 @@ summarize_oolong(oolong_test, target_value = all_afinn_score) ### Suggested workflow -Create an oolong object, clone it for another coder. According to Song et al. (Forthcoming), you should at least draw 1% of your data. +Create an oolong object, clone it for another coder. According to Song et al. (2020), you should at least draw 1% of your data. ```{r} trump <- gs(input_corpus = trump2k, exact_n = 40, userid = "JJ") @@ -439,7 +439,7 @@ Read the results. The diagnostic plot consists of 4 subplots. It is a good idea * Subplot (bottom left): Raw correlation between target value and content length. One should want to have no correlation, as an indication of good reliability against the influence of content length. (See Chan et al. 2020) * Subplot (bottom right): Cook's distance of all data point. One should want to have no dot (or at least very few dots) above the threshold. It is an indication of how the raw correlation between human judgement and target value can or cannot be influenced by extreme values in your data. -The textual output contains the Krippendorff's alpha of the codings by your raters. In order to claim validity of your target value, you must first establish the reliability of your gold standard. Song et al. [Forthcoming] suggest Krippendorff's Alpha > 0.7 as an acceptable cut-off. +The textual output contains the Krippendorff's alpha of the codings by your raters. In order to claim validity of your target value, you must first establish the reliability of your gold standard. Song et al. (2020) suggest Krippendorff's Alpha > 0.7 as an acceptable cut-off. ```{r} res diff --git a/overview_gh.md b/overview_gh.md index 1720921..d36db84 100644 --- a/overview_gh.md +++ b/overview_gh.md @@ -84,7 +84,7 @@ be doing the test. oolong_test <- wi(abstracts_keyatm, userid = "Hadley") oolong_test #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✔ WI ✖ TI ✖ WSI #> ☺ Hadley #> ℹ WI: k = 10, 0 coded. @@ -115,7 +115,7 @@ can look at the model precision by printing the oolong test. oolong_test$lock() oolong_test #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✔ WI ✖ TI ✖ WSI #> ☺ Hadley #> ℹ WI: k = 10, 10 coded. @@ -139,7 +139,7 @@ parameter `n_correct_ws` to N - 1. oolong_test <- wsi(abstracts_keyatm, userid = "Garrett") oolong_test #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✖ WI ✖ TI ✔ WSI #> ☺ Garrett #> ℹ WSI: n = 10, 0 coded. @@ -160,7 +160,7 @@ oolong_test$do_word_set_intrusion_test() oolong_test$lock() oolong_test #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✖ WI ✖ TI ✔ WSI #> ☺ Garrett #> ℹ WSI: n = 10, 10 coded. @@ -201,7 +201,7 @@ topic model will generate topic intrusion test cases. oolong_test <- ti(abstracts_keyatm, abstracts$text, userid = "Julia") oolong_test #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✖ WI ✔ TI ✖ WSI #> ☺ Julia #> ℹ TI: n = 25, 0 coded. @@ -224,7 +224,7 @@ oolong_test$lock() ``` r oolong_test #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✖ WI ✔ TI ✖ WSI #> ☺ Julia #> ℹ TI: n = 25, 25 coded. @@ -284,7 +284,7 @@ Get a summary of the two objects. ``` r summarize_oolong(oolong_test_rater1, oolong_test_rater2) #> -#> ── Summary (topic model): ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── Summary (topic model): ──────────────────────────────────────────────────────────────────────────────────────────── #> #> ── Word intrusion test ── #> @@ -353,7 +353,7 @@ You can still generate word intrusion and word set intrusion tests. ``` r wi(newsgroup_nb) #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✔ WI ✖ TI ✖ WSI #> ℹ WI: k = 20, 0 coded. #> @@ -366,7 +366,7 @@ wi(newsgroup_nb) ``` r wsi(newsgroup_nb) #> -#> ── oolong (topic model) ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (topic model) ────────────────────────────────────────────────────────────────────────────────────────────── #> ✖ WI ✖ TI ✔ WSI #> ℹ WSI: n = 20, 0 coded. #> @@ -411,7 +411,7 @@ should be an adjective, e.g. positive, liberal, populistic, etc. oolong_test <- gs(input_corpus = trump2k, construct = "positive", userid = "Joe") oolong_test #> -#> ── oolong (gold standard generation) ─────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (gold standard generation) ───────────────────────────────────────────────────────────────────────────────── #> ☺ Joe #> ℹ GS: n = 20, 0 coded. #> ℹ Construct: positive. @@ -436,7 +436,7 @@ After the coding, you need to first lock the test and then the oolong_test$lock() oolong_test #> -#> ── oolong (gold standard generation) ─────────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── oolong (gold standard generation) ───────────────────────────────────────────────────────────────────────────────── #> ☺ Joe #> ℹ GS: n = 20, 20 coded. #> ℹ Construct: positive. @@ -509,7 +509,7 @@ summarize_oolong(oolong_test, target_value = all_afinn_score) #> `geom_smooth()` using formula = 'y ~ x' #> #> ── Summary (gold standard generation): -#> ───────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ─────────────────────────────────────────────────────────────────────────────── #> ℹ Correlation: 0.718 (p = 4e-04) #> ℹ Effect of content length: -0.319 (p = 0.1709) #> • `` -> `...1` @@ -518,7 +518,7 @@ summarize_oolong(oolong_test, target_value = all_afinn_score) ### Suggested workflow Create an oolong object, clone it for another coder. According to Song -et al. (Forthcoming), you should at least draw 1% of your data. +et al. (2020), you should at least draw 1% of your data. ``` r trump <- gs(input_corpus = trump2k, exact_n = 40, userid = "JJ") @@ -583,13 +583,13 @@ correlation and agreement. The textual output contains the Krippendorff’s alpha of the codings by your raters. In order to claim validity of your target value, you must first establish the reliability of your gold standard. Song et -al. \[Forthcoming\] suggest Krippendorff’s Alpha \> 0.7 as an -acceptable cut-off. +al. (2020) suggest Krippendorff’s Alpha \> 0.7 as an acceptable +cut-off. ``` r res #> -#> ── Summary (gold standard generation): ───────────────────────────────────────────────────────────────────────────────────────────────────────── +#> ── Summary (gold standard generation): ─────────────────────────────────────────────────────────────────────────────── #> ℹ Krippendorff's Alpha: 0.931 #> ℹ Correlation: 0.744 (p = 2e-04) #> ℹ Effect of content length: -0.319 (p = 0.1709) diff --git a/vig_body.Rmd b/vig_body.Rmd index 784485d..c26ad87 100644 --- a/vig_body.Rmd +++ b/vig_body.Rmd @@ -325,7 +325,7 @@ summarize_oolong(oolong_test, target_value = all_afinn_score) ### Suggested workflow -Create an oolong object, clone it for another coder. According to Song et al. (Forthcoming), you should at least draw 1% of your data. +Create an oolong object, clone it for another coder. According to Song et al. (2020), you should at least draw 1% of your data. ```{r} trump <- gs(input_corpus = trump2k, exact_n = 40, userid = "JJ") @@ -423,7 +423,7 @@ Read the results. The diagnostic plot consists of 4 subplots. It is a good idea * Subplot (bottom left): Raw correlation between target value and content length. One should want to have no correlation, as an indication of good reliability against the influence of content length. (See Chan et al. 2020) * Subplot (bottom right): Cook's distance of all data point. One should want to have no dot (or at least very few dots) above the threshold. It is an indication of how the raw correlation between human judgement and target value can or cannot be influenced by extreme values in your data. -The textual output contains the Krippendorff's alpha of the codings by your raters. In order to claim validity of your target value, you must first establish the reliability of your gold standard. Song et al. [Forthcoming] suggest Krippendorff's Alpha > 0.7 as an acceptable cut-off. +The textual output contains the Krippendorff's alpha of the codings by your raters. In order to claim validity of your target value, you must first establish the reliability of your gold standard. Song et al. (2020) suggest Krippendorff's Alpha > 0.7 as an acceptable cut-off. ```{r} res diff --git a/vignettes/overview.Rmd b/vignettes/overview.Rmd index b92603d..49bbea6 100644 --- a/vignettes/overview.Rmd +++ b/vignettes/overview.Rmd @@ -344,7 +344,7 @@ summarize_oolong(oolong_test, target_value = all_afinn_score) ### Suggested workflow -Create an oolong object, clone it for another coder. According to Song et al. (Forthcoming), you should at least draw 1% of your data. +Create an oolong object, clone it for another coder. According to Song et al. (2020), you should at least draw 1% of your data. ```{r} trump <- gs(input_corpus = trump2k, exact_n = 40, userid = "JJ") @@ -442,7 +442,7 @@ Read the results. The diagnostic plot consists of 4 subplots. It is a good idea * Subplot (bottom left): Raw correlation between target value and content length. One should want to have no correlation, as an indication of good reliability against the influence of content length. (See Chan et al. 2020) * Subplot (bottom right): Cook's distance of all data point. One should want to have no dot (or at least very few dots) above the threshold. It is an indication of how the raw correlation between human judgement and target value can or cannot be influenced by extreme values in your data. -The textual output contains the Krippendorff's alpha of the codings by your raters. In order to claim validity of your target value, you must first establish the reliability of your gold standard. Song et al. [Forthcoming] suggest Krippendorff's Alpha > 0.7 as an acceptable cut-off. +The textual output contains the Krippendorff's alpha of the codings by your raters. In order to claim validity of your target value, you must first establish the reliability of your gold standard. Song et al. (2020) suggest Krippendorff's Alpha > 0.7 as an acceptable cut-off. ```{r} res