Get content from git when calling polypostbuild and store1 or build and store alll0

.github/workflows/elixir.yml

@@ -24,6 +24,11 @@ jobs:
         elixir-version: '1.17.3'
         otp-version: '27.1'
 
+    - name: Setup test git identity
+      run: |
+        git config --global user.email "github-ci@totemic.dev"
+        git config --global user.name "Github CI"
+
     - name: Restore dependencies cache
       uses: actions/cache@v3
       with:

README.md

@@ -14,31 +14,53 @@ A publishing engine with markdown and code highlighting support.
   * `PolyPost.build_and_store!/1`
   * `PolyPost.build_and_store_all!/0`
 
+## Requirements
+
+* Elixir 1.17 or greater
+* Erlang OTP 27 or greater
+* Git
+* A mostly POSIX compatible environment (linux, darwin, bsd, etc.)
+
 ## Installation
 
-You can add `poly_post` to your list of dependencies in `mix.exs`:
+Add `poly_post` to your list of dependencies in `mix.exs` and include
+any decoders you want to use:
 
 ```elixir
 def deps do
   [
-    {:poly_post, "~> 0.1"}
-    {:jason, "~> 1.4"} # For JSON front matter
-    {:yaml_elixir, "~> 2.11"} # For YAML front matter
-    {:toml, "~> 1.4"} # For TOML front matter
+    {:poly_post, "~> 0.1"},
+    {:jason, "~> 1.4"}, # Optional dependency for JSON front matter
+    {:yaml_elixir, "~> 2.11"}, # Optional dependency for YAML front matter
+    {:toml, "~> 1.4"} # Optoinal dependency for TOML front matter
   ]
 end
 ```
 
-In any of the `config/{config,dev,prod,test}.exs` files you can
-configure the front matter decoder and each resource for your content:
+Then run `mix deps.get` and `mix deps.compile` or just a `mix compile` in your app.
+
+## Configuration
+
+There are two strategies for configuring content: `paths` and `git`.
+
+### For paths
+
+With the following environment variable, you can set your glob pattern:
+
+```
+export ARTICLE_PATH=/path/to/my/markdown/*.md
+```
+
+In the `config/runtime.exs` files you can configure the front matter
+decoder and each resource for your content:
 
 ```elixir
 config :poly_post, :resources,
   front_matter: [decoder: {Jason, :decode, keys: :atoms}],
   content: [
     articles: [
       module: Article,
-      path: "/path/to/my/markdown/*.md"
+      path: System.get_env("ARTICLE_PATH")
     ]
   ]
 ```
@@ -63,12 +85,49 @@ config :poly_post, :resources,
   content: [
     articles: [
       module: Article,
-      path: "/path/to/my/markdown/*.md",
+      path: System.get_env("ARTICLE_PATH"),
       front_matter: [decoder: {Toml, :decode, keys: :atoms}]
     ]
   ]
 ```
 
+### For git
+
+Your environment **MUST** have `git` installed for this to work.
+
+This is similar to the paths strategy, but you need to specify a
+`source` key as well:
+
+```elixir
+config :poly_post, :resources,
+  front_matter: [decoder: {YamlElixir, :read_from_string, []}],
+  content: [
+    articles: [
+      module: Article,
+      source: [
+        dest: System.get_env("SOURCE_PATH"),
+        github: "my-username/my-content",
+        ref: "main"
+      ],
+      path: System.get_env("CONTENT_PATH")
+    ]
+  ]
+```
+
+
+* `dest` - (required)is the folder that git will clone to.
+* `github` - (required if not using `git` config) to access a github repo, expands to `https://github.com/my-username/my-content.git`
+* `git` - (required if not using `github` config) to access a git repo, e.g `https://git.mydomain.com/repo.git` or can be local
+* `ref` - (optional) - the specified branch to use, defaults to whatever the default branch on the repo, usually `main` or `master`
+
+This implementation doesn't manage authentication if you are
+accessing a private repo, you must ensure the user that runs your
+application has read access to your git repo.
+
+If this is a security concern for you, it is recommended that you use
+the `path` strategy and use some other mechanism to retrieve the
+contents into your environment.
+
 ## Basic Usage
 
 ### Loading and Storing Content

lib/poly_post/builder.ex

@@ -1,9 +1,16 @@
 defmodule PolyPost.Builder do
   @moduledoc """
   A module used for building content from Markdown files (resources).
+
+  When configured to use git or github, this will also retrieve
+  content from the specified forge.
   """
 
   alias PolyPost.Resource
+  alias PolyPost.Scm.{
+    Git,
+    Github
+  }
 
   # API
 
@@ -29,20 +36,41 @@ defmodule PolyPost.Builder do
 
   defp build_content!(resources) do
     opts = get_resources_config(resources)
-    module = Keyword.get(opts, :module)
-    filepaths = Keyword.get(opts, :path)
-    fm_config = Keyword.get(opts, :front_matter) || get_front_matter_config()
+    module = get_in(opts, [:module])
+    paths = get_in(opts, [:path])
+    dest = get_in(opts, [:source, :dest])
+    git = get_in(opts, [:source, :git])
+    github = get_in(opts, [:source, :github])
+    ref = get_in(opts, [:source, :ref])
+    fm_config = get_in(opts, [:front_matter]) || get_front_matter_config()
 
     cond do
-      filepaths -> build_via_paths!(module, fm_config, filepaths)
-      :else -> []
+      paths && !(git || github) ->
+        build_via_paths!(module, fm_config, paths)
+      dest && paths && (git || github) ->
+        build_via_git!(module, fm_config, paths, dest, git, github, ref)
+      :else ->
+        []
     end
   end
 
-  defp build_via_filepath!(module, fm_config, filepath) do
-    filename = Path.basename(filepath)
-    {metadata, body} = extract_content!(filepath, fm_config)
-    apply(module, :build, [filename, metadata, body])
+  defp build_via_git!(module, fm_config, paths, dest, git, github, ref) do
+    repo = git || Github.expand_repo(github)
+    git_dir = Path.join(dest, ".git")
+
+    if File.dir?(dest) && File.dir?(git_dir) do
+      if Git.get_status!(dest) != "" do
+        Git.stash!(dest)
+      end
+    else
+      File.mkdir_p!(dest)
+      Git.clone!(repo, dest)
+    end
+
+    Git.checkout!(dest, ref || Git.get_default_branch!(dest))
+    Git.pull!(dest)
+
+    build_via_paths!(module, fm_config, paths)
   end
 
   defp build_via_paths!(module, fm_config, paths, content \\ [])
@@ -56,12 +84,18 @@ defmodule PolyPost.Builder do
       path
       |> Path.wildcard()
       |> Enum.reduce(content, fn filepath, acc ->
-        [build_via_filepath!(module, fm_config, filepath) | acc]
+        [build_via_specific_path!(module, fm_config, filepath) | acc]
       end)
 
     build_via_paths!(module, fm_config, paths, new_content)
   end
 
+  defp build_via_specific_path!(module, fm_config, filepath) do
+    filename = Path.basename(filepath)
+    {metadata, body} = extract_content!(filepath, fm_config)
+    apply(module, :build, [filename, metadata, body])
+  end
+
   defp extract_content!(path, fm_config) do
     {raw_metadata, raw_content} = File.read!(path) |> extract_parts!()
 

lib/poly_post/scm/git.ex

@@ -0,0 +1,57 @@
+defmodule PolyPost.Scm.Git do
+  @moduledoc false
+
+  # API
+
+  def add!(path, specs) do
+    File.cd!(path, fn -> run!(["add", specs]) end)
+  end
+
+  def checkout!(path, reference) do
+    File.cd!(path, fn -> run!(["checkout", reference, "--quiet"]) end)
+  end
+
+  def clone!(repo, path) do
+    run!(["clone", repo, path])
+  end
+
+  def get_default_branch!(path) do
+    path
+    |> File.cd!(fn -> run!(["rev-parse", "--abbrev-ref", "HEAD"]) end)
+    |> String.trim()
+  end
+
+  def get_status!(path) do
+    File.cd!(path, fn -> run!(["status", "-u", "--porcelain"]) end)
+  end
+
+  def pull!(path) do
+    File.cd!(path, fn -> run!(["pull"]) end)
+  end
+
+  def stash!(path) do
+    File.cd!(path, fn ->
+      add!(path, ".")
+      run!(["stash"])
+    end)
+  end
+
+  # Private
+
+  defp run!(args) do
+    try do
+      System.cmd("git", args)
+    catch
+      :error, :enoent ->
+        raise "The git command was not found."
+    else
+      {response, 0} ->
+        response
+      {response, status} when is_binary(response) ->
+        reason = String.trim(response <> " " <> inspect(status))
+        raise "The git command failed with reason: #{reason}"
+      _ ->
+        raise "The git command failed with args: #{Enum.join(args, " ")}"
+    end
+  end
+end

lib/poly_post/scm/github.ex

@@ -0,0 +1,9 @@
+defmodule PolyPost.Scm.Github do
+  @moduledoc false
+
+  # API
+
+  def expand_repo(github) do
+    "https://github.com/#{github}.git"
+  end
+end

mix.exs

@@ -76,7 +76,7 @@ defmodule PolyPost.MixProject do
     ]
   end
 
-  defp elixirc_paths(:test), do: ["lib", "test/fixtures"]
+  defp elixirc_paths(:test), do: ["lib", "test/fixtures", "test/support"]
   defp elixirc_paths(_), do: ["lib"]
 
   defp package do

test/fixtures/test_articles/my_article1.html

@@ -0,0 +1,4 @@
+<h2>
+My Article 1</h2>
+<p>
+This is my first article</p>

test/fixtures/test_articles/my_article2.html

@@ -0,0 +1,5 @@
+<h2>
+My Article 2</h2>
+<p>
+This is my second article</p>
+<pre><code class="highlight"><span class="nc">Enum</span><span class="o">.</span><span class="n">map</span><span class="p" data-group-id="group-1">(</span><span class="p" data-group-id="group-2">[</span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span><span class="w"> </span><span class="mi">3</span><span class="p" data-group-id="group-2">]</span><span class="p">,</span><span class="w"> </span><span class="k" data-group-id="group-3">fn</span><span class="w"> </span><span class="n">x</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">x</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="w"> </span><span class="k" data-group-id="group-3">end</span><span class="p" data-group-id="group-1">)</span></code></pre>