add code to save html files before any filtering

orchid00 · Nov 30, 2018 · eed7b61 · eed7b61
1 parent a6c10cc
commit eed7b61
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 4 deletions.
diff --git a/README.md b/README.md
@@ -58,6 +58,7 @@ The project structure looks like this:
 └── results
     ├── data
     │   ├── pages
+    │   ├── rawhtml
     │   └── sentences
     │       └── allsentences
     └── RData

diff --git a/docs/index.html b/docs/index.html
diff --git a/scripts/00_installPkgs.R b/scripts/00_installPkgs.R
@@ -42,6 +42,7 @@ if (!dir.exists("results")) {
     dir.create(here("results", "data"))
     dir.create(here("results", "data", "pages"))
     dir.create(here("results", "data", "sentences"))
+    dir.create(here("results", "data", "rawhtml"))
 }
 
 

diff --git a/scripts/01_custome_functions.R b/scripts/01_custome_functions.R
@@ -20,8 +20,11 @@ getPageContent <- function(url, css_class, num){
     #url <- "https://www.indeed.com/rc/clk?jk=788e7e311656fc54&fccid=09fad757f3449fa5&vjs=3"
     #css_class <- ".jobsearch-JobComponent-description"
 
+    htmlpage <-
+        read_html(url)
+
     page_content <-
-        read_html(url) %>% 
+        htmlpage %>% 
         html_nodes(css = css_class) %>% 
         html_text() %>%         # clean text
         str_replace_all(regex("\n"), "\\. ") %>%
@@ -36,6 +39,11 @@ getPageContent <- function(url, css_class, num){
         page_content %>% 
         tibble(doc = as.numeric(num), text = .) # convert character to a tibble
 
+    write_html(htmlpage, 
+              path = here("results", "data","rawhtml", 
+                          paste0(Sys.Date(), "_", num, ".html"))
+    )
+
     save(page_content, 
         file = here("results", "RData", 
                     paste0(Sys.Date(), "_", num, ".RData"))