More parsing (#34)

* checkpoint * checkpoint * checkpoint * checkpoint * refactor * checkpoint * checkpoint * checkpoint * checkpoint * checkpoint * checkpoint * checkpoint * checkpoint
public-law · Dec 13, 2023 · 372bbd3 · 372bbd3
1 parent 48f562c
commit 372bbd3
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 21 deletions.
diff --git a/config/config.exs b/config/config.exs
@@ -13,7 +13,7 @@ config :crawly,
   log_to_file: true,
 
   closespider_timeout: 1,
-  concurrent_requests_per_domain: 8,
+  concurrent_requests_per_domain: 10,
   middlewares: [
     Crawly.Middlewares.DomainFilter,
     Crawly.Middlewares.UniqueRequest,

diff --git a/lib/news/article.ex b/lib/news/article.ex
@@ -88,40 +88,47 @@ defmodule News.Article do
 
   @spec find_citations_in_html(Floki.html_tree) :: [binary]
   def find_citations_in_html(document) do
-    cites_from_hrefs =
-      document
+    (cites_from_hrefs(document) ++ cites_from_text(document))
+      |> filter(&is_binary/1)
+      |> cleanup_list()
+  end
+
+  def cites_from_hrefs(document) do
+    document
       |> hrefs()
       |> map(&href_to_cite/1)
+   end
 
+  def cites_from_text(document) do
     html = Floki.text(document)
-    crs_cites_from_text_1 =
-      Regex.scan(~r/(C.R.S. &#xa7;(?:&#xa7;)? \d+-\d+-\d+)/, html)
-      |> flatten()
-      |> map(fn m -> String.replace(m, ~r/&#xa7; ?/, "", global: true) end)
 
-    crs_cites_from_text_2 =
-      Regex.scan(~r/(\d+-\d+-\d+(?:\.\d+)?) C.R.S./, html)
-      |> map(&last/1)
-      |> map(fn m -> "C.R.S. #{m}" end)
-      |> flatten()
+    [
+      simple_cites(html, ~r/(\d+-\d+-\d+(?:\.\d+)?) C.R.S./,             &("C.R.S. #{&1}")),
+      simple_cites(html, ~r/Colo. Rev. Stat. § (\d+-\d+-\d+(?:\.\d+)?)/, &("C.R.S. #{&1}")),
+      simple_cites(html, ~r/Nev. Rev. Stat. § (\d+[A-Z]?\.\d+[A-Z]?)/,   &("NRS #{&1}")),
+      simple_cites(html, ~r/NY Penal Law § (\d+\.\d+)/,                  &("NY Penal Law Section #{&1}")),
+      simple_cites(html, ~r/Ore. Rev. Stat. § (\d+[A-Z]?\.\d+[A-Z]?)/,   &("ORS #{&1}")),
 
-    crs_cites_from_text_3 =
-      Regex.scan(~r/Colo. Rev. Stat. § (\d+-\d+-\d+(?:\.\d+)?)/, html)
-      |> map(&last/1)
-      |> map(fn m -> "C.R.S. #{m}" end)
+      Regex.scan(~r/(C.R.S. &#xa7;(?:&#xa7;)? \d+-\d+-\d+)/, html)
       |> flatten()
+      |> map(fn m -> String.replace(m, ~r/&#xa7; ?/, "", global: true) end),
 
-    tx_cites_from_text =
       Regex.scan(~r/(Texas \w+ Code Section [\d\w.]+)/, html)
       |> flatten()
       |> map(fn m -> String.replace(m, "Texas ",          "Tex. ")    end)
       |> map(fn m -> String.replace(m, "Family ",         "Fam. ")    end)
-      |> map(fn m -> String.replace(m, "Transportation ", "Transp. ") end)
+      |> map(fn m -> String.replace(m, "Transportation ", "Transp. ") end),
+    ]
+    |> flatten()
+  end
 
 
-     (cites_from_hrefs ++ crs_cites_from_text_1 ++ crs_cites_from_text_2 ++ crs_cites_from_text_3 ++ tx_cites_from_text)
-     |> filter(&is_binary/1)
-     |> cleanup_list()
+  @spec simple_cites(binary(), Regex.t(), (any() -> any())) :: list()
+  def simple_cites(html, regex, replace_func) do
+    Regex.scan(regex, html)
+    |> map(&last/1)
+    |> map(replace_func)
+    |> flatten()
   end
 
 

diff --git a/test/news/article_test.exs b/test/news/article_test.exs
@@ -31,6 +31,10 @@ defmodule News.ArticleTest do
   @snippet_test_cases [
     %{html: "<html></html>", cites: []},
     %{html: "<html><p>under Colo. Rev. Stat. § 24-34-402.7 and</p></html>", cites: ["C.R.S. 24-34-402.7"]},
+    %{html: "<html><p>under Ore. Rev. Stat. § 633.295 and</p></html>",      cites: ["ORS 633.295"]},
+    %{html: "<html><p>under Nev. Rev. Stat. § 675.020 and</p></html>",      cites: ["NRS 675.020"]},
+    %{html: "<html><p>under Nev. Rev. Stat. § 603A.302 and</p></html>",     cites: ["NRS 603A.302"]},
+    %{html: "<html><p>See NY Penal Law § 155.25.</p></html>",               cites: ["NY Penal Law Section 155.25"]},
   ]
 
   Enum.each(@snippet_test_cases, fn %{html: html, cites: cites} ->