From c8bf1d0b39831311319d7ff25957474416a877b7 Mon Sep 17 00:00:00 2001 From: Tyler Young Date: Tue, 6 Dec 2022 11:38:44 -0600 Subject: [PATCH 1/2] Support disabling compile-time HTTP requests This adds a new configuration flag, `:fetch_latest`, which entirely disables the HTTP request at compile-time that tries to get the latest `public_suffix_list.dat`. This behavior was (sort of) achievable without this flag, by configuring the `:public_suffix_list_url` to the empty charlist, but doing so produced a warning on each compile. With the new flag, there's no error. This is desirable for our use-case for a couple reasons: 1. We want the development build to match as closely as possible what we'll deploy in prod 2. We don't want the possibility of failing production builds based on the availability of a third-party web host --- README.md | 27 ++++++++++++++++++++++++++- lib/domainatrex.ex | 14 ++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d5a35a7..308f645 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,32 @@ iex> Domainatrex.parse("blog.someone.id.au") {:ok, %{domain: "someone", subdomain: "blog", tld: "id.au"}} ``` - +## Configuration + +For maximum performance, `Domainatrex` reads the list of all known top-level domains at compile time. +Likewise, by default, the package will attempt to fetch the latest list of TLDs from the web before +falling back to a local (potentially out of date) copy. You can configure this behavior in your +`config.exs` as follows: + +- `:fetch_latest`: A Boolean flag to determine whether `Domainatrex` should try to fetch the latest + list of public suffixes at compile time; default is `true` +- `:public_suffix_list_url`: A charlist URL to the latest public suffix file that `Domainatrex` will + try to fetch at compile time; default is + `'https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat'` +- `:fallback_local_copy`: The path to the local suffix file that `Domainatrex` will use if it wasn't + able to fetch a fresh file from the URL, or if fetching updated files was disabled; default is + the `"lib/public_suffix_list.dat"` file included in the package. + +Here's a complete example of how you might customize this behavior in your `confix.exs`: + +```elixir +config :domainatrex, + # Explicitly allow compile-time HTTP request to fetch the latest list of TLDs (default) + fetch_latest: true, + # Download the public suffix list from the official source (not necessarily tested with Domainatrex!) + public_suffix_list_url: 'https://publicsuffix.org/list/public_suffix_list.dat', + fallback_local_copy: "priv/my_app_custom_suffix_list.dat" +``` ## Changelog diff --git a/lib/domainatrex.ex b/lib/domainatrex.ex index 670b690..8d9923f 100644 --- a/lib/domainatrex.ex +++ b/lib/domainatrex.ex @@ -5,17 +5,23 @@ defmodule Domainatrex do """ @public_suffix_list_url Application.get_env(:domainatrex, :public_suffix_list_url, 'https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat') @fallback_local_copy Application.get_env(:domainatrex, :fallback_local_copy, "lib/public_suffix_list.dat") + @fetch_latest Application.get_env(:domainatrex, :fetch_latest, true) @public_suffix_list nil :inets.start :ssl.start - case :httpc.request(:get, {@public_suffix_list_url, []}, [], []) do - {:ok, {_, _, string}} -> - @public_suffix_list to_string(string) + + with true <- @fetch_latest, + {:ok, {_, _, string}} <- :httpc.request(:get, {@public_suffix_list_url, []}, [], []) do + @public_suffix_list to_string(string) + else _ -> case File.read @fallback_local_copy do {:ok, string} -> - Logger.error "[Domainatrex] Could not read the public suffix list from the internet, trying to read from the backup at #{@fallback_local_copy}" + if @fetch_latest do + Logger.error "[Domainatrex] Could not read the public suffix list from the internet, trying to read from the backup at #{@fallback_local_copy}" + end + @public_suffix_list string _ -> Logger.error "[Domainatrex] Could not read the public suffix list, please make sure that you either have an internet connection or #{@fallback_local_copy} exists" From eb1d3d245c8937b8682a0fd22c28771b95d11488 Mon Sep 17 00:00:00 2001 From: Tyler Young Date: Tue, 6 Dec 2022 12:28:59 -0600 Subject: [PATCH 2/2] Remove warning about unused module attribute when `:fetch_latest` is false --- lib/domainatrex.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/domainatrex.ex b/lib/domainatrex.ex index 8d9923f..8ab7d76 100644 --- a/lib/domainatrex.ex +++ b/lib/domainatrex.ex @@ -3,7 +3,6 @@ defmodule Domainatrex do @moduledoc """ Documentation for Domainatrex. """ - @public_suffix_list_url Application.get_env(:domainatrex, :public_suffix_list_url, 'https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat') @fallback_local_copy Application.get_env(:domainatrex, :fallback_local_copy, "lib/public_suffix_list.dat") @fetch_latest Application.get_env(:domainatrex, :fetch_latest, true) @public_suffix_list nil @@ -12,7 +11,8 @@ defmodule Domainatrex do :ssl.start with true <- @fetch_latest, - {:ok, {_, _, string}} <- :httpc.request(:get, {@public_suffix_list_url, []}, [], []) do + public_suffix_list_url <- Application.get_env(:domainatrex, :public_suffix_list_url, 'https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat'), + {:ok, {_, _, string}} <- :httpc.request(:get, {public_suffix_list_url, []}, [], []) do @public_suffix_list to_string(string) else _ ->