first commit

.github/workflows/CompatHelper.yml

@@ -0,0 +1,16 @@
+name: CompatHelper
+on:
+  schedule:
+    - cron: 0 0 * * *
+  workflow_dispatch:
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
+        run: julia -e 'using CompatHelper; CompatHelper.main()'

.github/workflows/TagBot.yml

@@ -0,0 +1,13 @@
+name: TagBot
+on:
+  schedule:
+    - cron: 0 0 * * *
+  workflow_dispatch:
+jobs:
+  TagBot:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ssh: ${{ secrets.DOCUMENTER_KEY }}

.gitignore

@@ -0,0 +1,24 @@
+# Files generated by invoking Julia with --code-coverage
+*.jl.cov
+*.jl.*.cov
+
+# Files generated by invoking Julia with --track-allocation
+*.jl.mem
+
+# System-specific files and directories generated by the BinaryProvider and BinDeps packages
+# They contain absolute paths specific to the host computer, and so should not be committed
+deps/deps.jl
+deps/build.log
+deps/downloads/
+deps/usr/
+deps/src/
+
+# Build artifacts for creating documentation generated by the Documenter package
+docs/build/
+docs/site/
+
+# File generated by Pkg, the package manager, based on a corresponding Project.toml
+# It records a fixed state of all packages used by the project. As such, it should not be
+# committed for packages, but should be committed for applications that require a static
+# environment.
+Manifest.toml

.travis.yml

@@ -0,0 +1,28 @@
+# Documentation: http://docs.travis-ci.com/user/languages/julia
+language: julia
+notifications:
+  email: false
+julia:
+  - 1.0
+  - 1.5
+  - nightly
+os:
+  - linux
+  - osx
+  - windows
+arch:
+  - x64
+cache:
+  directories:
+    - ~/.julia/artifacts
+jobs:
+  fast_finish: true
+  allow_failures:
+    - julia: nightly
+after_success:
+  - |
+    julia -e '
+      using Pkg
+      Pkg.add("Coverage")
+      using Coverage
+      Codecov.submit(process_folder())'

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 RTI International and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Project.toml

@@ -0,0 +1,24 @@
+name = "PushshiftRedditDistiller"
+uuid = "860182d5-2f7f-4b01-a084-d8f9ccad7d63"
+authors = ["Peter B <5107405+pmbaumgartner@users.noreply.github.com> and contributors"]
+version = "0.1.0"
+
+[deps]
+CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
+CodecXz = "ba30903b-d9e8-5048-a5ec-d1f5b0d4b47b"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
+DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a"
+ProgressBars = "49802e3a-d2f1-5c88-81d8-b72133a6f568"
+TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
+
+[compat]
+julia = "1"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]

README.md

@@ -0,0 +1,160 @@
+# PushshiftRedditDistiller
+
+This package is intended to assist with downloading, extracting, and distilling the monthly reddit data dumps made available through [pushshift.io](https://files.pushshift.io/reddit/).
+
+# Example Use
+
+## Preexisting File
+If you already have a monthly submissions or comments file downloaded:
+
+```julia
+julia> using PushshiftRedditDistiller
+
+julia> filter = RedditDataFilter(author=["spez"])
+
+julia> spez_comments = distill("~/Downloads/RC_2005-12.bz2", filter)
+
+julia> length(spez_comments)
+7
+
+julia> typeof(spez_comments)
+Array{Dict{Symbol,Any},1}
+
+julia> first(spez_comments)
+Dict{Symbol,Any} with 18 entries:
+  :author_flair_css_class => nothing
+  :gilded                 => 0
+  :parent_id              => "t3_17942"
+  :score                  => 4
+  :link_id                => "t3_17942"
+  :created_utc            => 1134392748
+  :author_flair_text      => nothing
+  :distinguished          => nothing
+  :author                 => "spez"
+  :stickied               => false
+  :subreddit              => "reddit.com"
+  :subreddit_id           => "t5_6"
+  :id                     => "c53"
+  :retrieved_on           => 1473738414
+  :body                   => "still looks like a death trap to me..."
+  :controversiality       => 0
+  :ups                    => 4
+  :edited                 => false
+```
+
+## DataDeps Catalog
+
+All monthly comment and submissions files are cataloged and available using [DataDeps.jl](https://github.com/oxinabox/DataDeps.jl). The format for the `datadep` string macros are `reddit-comments-YYYY-MM` for comments and `reddit-submissions-YYYY-MM` for submissions.
+
+If the file isn't downloaded, you will be prompted to download that archive file before processing.
+
+```julia
+julia> using DataDeps
+
+julia> more_spez_comments = distill(datadep"reddit-comments-2006-04", filter)
+This program has requested access to the data dependency reddit-comments-2006-04.
+which is not currently installed. It can be installed automatically, and you will not see this message again.
+
+Baumgartner, J., Zannettou, S., Keegan, B., Squire, M., & Blackburn, J. (2020, May). 
+The pushshift reddit dataset.
+In Proceedings of the International AAAI Conference on Web and Social Media 
+(Vol. 14, pp. 830-839).
+
+
+
+Do you want to download the dataset from https://files.pushshift.io/reddit/comments/RC_2006-04.bz2 to "~/.julia/datadeps/reddit-comments-2006-04"?
+[y/n]
+y
+
+┌ Info: Downloading
+│   source = "https://files.pushshift.io/reddit/comments/RC_2006-04.bz2"
+│   dest = "~/.julia/datadeps/reddit-comments-2006-04/RC_2006-04.bz2"
+│   progress = 1.0
+│   time_taken = "0.51 s"
+│   time_remaining = "0.0 s"
+│   average_speed = "3.729 MiB/s"
+│   downloaded = "1.891 MiB"
+│   remaining = "0 bytes"
+└   total = "1.891 MiB"
+┌ Warning: Checksum not provided, add to the Datadep Registration the following hash line
+│   hash = "1e757b8a7dd4b1f7281329ac77cf4a20f59571d59899983fd7f347b24b081516"
+
+julia> length(more_spez_comments)
+19
+```
+
+## Multithreaded Support
+
+If you start Julia with more than one thread available, multithreading will be enabled. It's best to play around with the number of threads for a bit if you're looking to optimize parsing speed, as it depends on how complex your filter is and the decompression algorithm available. For example, slower decompression algorithms (`bz2`) will bottleneck the speed in which you can feed lines from the stream to each thread to parse. For faster algorithms, you may have a strict filter that doesn't result in many lines needing to be fully parsed, making the overhead of coordinating threads costly.
+
+The number of threads used while parsing will be displayed in the progress bar.
+
+```bash
+$ julia -t 3
+```
+...
+
+```julia
+julia> results = distill(datadep"reddit-comments-2006-04", filter)
+File: RC_2006-04.bz2; Threads: 3 ┣ ╱   ╱   ╱   ╱   ╱   ╱   ╱   ╱   ╱ ┫ 19090it 00:01 [35469.7 it/s]
+```
+
+
+## Filtering with `RedditDataFilter`
+
+Data can be filtered on the `author` or `subreddit` field currently. The filtering is currently disjunctive (OR), so if both `author` and `subreddit` are passed, it will return data from those author(s) OR those subreddit(s).
+
+In addition, you can control which fields are returned with the `fields` argument.
+
+All arguments are of type `Vector{String}`, though passing a single string to an argument will convert it to a length 1 `Vector`.
+
+**Note that no checking of correct field names is done for you, since the fields available change over time**
+
+```
+julia> using Dates
+
+julia> timestamps_only = RedditDataFilter(fields=["created_utc"])
+
+julia> timestamp_comments = distill(datadep"reddit-comments-2006-04", timestamps_only)
+
+julia> Dates.unix2datetime(first(timestamp_comments)[:created_utc])
+2006-04-01T00:00:55
+```
+
+## Other Usage Notes
+
+### Exporting
+
+`distill` returns `Vector{Dict{Symbol,Any}}`, which can be fed into a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl) (not included).
+
+```julia
+julia> using DataFrames, CSV
+
+julia> spez_comments_df = DataFrame(spez_comments)
+
+julia> CSV.write("spez_comments.csv", spez_comments_df, quotestrings=true)
+```
+
+### Managing DataDeps
+
+Some of the files are large - if you were to download the whole archive it would be over one TB. Because of this, you may want to remove a file after use or change your DataDeps download directory to another drive.
+
+**Removal**
+
+```julia
+julia> rm(datadep"reddit-comments-2006-04", recursive=true)
+```
+
+**New Directory**
+
+```julia
+julia> download_path = "/Users/user/pushshift-datadeps"
+
+julia> mkdir(download_path)
+
+julia> ENV["DATADEPS_LOAD_PATH"] = download_path
+
+julia> ENV["DATADEPS_NO_STANDARD_LOAD_PATH"] = true
+```
+
+

src/PushshiftRedditDistiller.jl

@@ -0,0 +1,18 @@
+module PushshiftRedditDistiller
+
+using DataDeps
+
+export RedditDataFilter, distill
+
+include("deps.jl")
+include("types.jl")
+include("distiller.jl")
+include("filter.jl")
+include("utils.jl")
+
+function __init__()
+    init_deps()
+end
+
+
+end