diff --git a/README.md b/README.md index 97deb35..2728b32 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ - [Who's using?](#whos-using) - [Roadmap](#roadmap) - [Sponsors and Donations](#sponsors-and-donations) +- [Contributors](#contributors) - [Copyright License](#copyright-license) - [Additional notices](#additional-notices) @@ -48,6 +49,8 @@ PardallMarkdown is a reactive publishing framework and engine written in Elixir. **As opposed to static website generators** (such as Hugo, Docusaurs and others), with PardallMarkdown, **you don't need to recompile and republish your application every time you write or modify new content**. The application can be kept running indefinitely in production, while it **watches a content folder for changes** and **the new content re-actively gets available for consumption** by your application. +The **content folder can be any path in the application server** or it can be a Git repository, since **PardallMarkdown has support for automatically pulling changes from a Git repository**. + ### Looking for Contributors ⚠️ I'm looking for a contributor(s) to make a more elaborate HTML and CSS template for the demo project. Check the [current demo here](https://pardall.xyz/), including its inner sections. It's a mix of Blog + Documentation + Wiki website. @@ -63,9 +66,11 @@ See PardallMarkdown in action and learn how to use it by following this video: - Filesystem-based, with **Markdown** and static files support. - Markdown files are parsed as HTML. -- FileWatcher, that **detects new content and modification of existing content**, which then **automatically re-parses and rebuilds the content**. +- `FileWatcher`, that **detects new content and modification of existing content**, which then **automatically re-parses and rebuilds the content**. - There is **no need to recompile** and redeploy the application nor the website, the **new content is available immediately** (depends on the interval set via `:recheck_pending_file_events_interval`, see below). - Created with [Phoenix LiveView](https://hexdocs.pm/phoenix_live_view/Phoenix.LiveView.html) and Phoenix Channels in mind: **create or modify a post** or a whole new **set of posts** and they are **immediately published in a website**. Check out [the demo](https://github.com/alfredbaudisch/pardall-markdown-phoenix-demo) repository. +- `RepositoryWatcher`, that **detects changes from a Git repository** (optional) and automatically pulls the content into a PardallMarkdown's content folder. + - Since the content folder is watched by `FileWatcher`, PardallMarkdown the rebuilds the whole content everything there are changes from the Git repository. - Support for the content folders outside of the application, this way, **new content files can be synced immediately from a source location** (for example, your computer), and then picked up by the FileWatcher. - Automatic creation of **table of contents** from Markdown headers. - **Infinite content hierarchies** (categories and sub-categories, sections and sub-sections). @@ -91,11 +96,11 @@ See PardallMarkdown in action and learn how to use it by following this video: Add dependency and application into your `mix.exs`: ```elixir defp deps do -[{:pardall_markdown, "~> 0.3.3"} ...] + [{:pardall_markdown, "~> 0.4.0"} ...] end def application do -[extra_applications: [:pardall_markdown, ...], ...] + [extra_applications: [:pardall_markdown, ...], ...] end ``` @@ -123,6 +128,19 @@ config :pardall_markdown, PardallMarkdown.Content, # new or modified content in the `root_path:`? recheck_pending_file_events_interval: 10_000, + # Git repository to watch and automatically fetch content from, + # leave "" or nil to not get content from a repository. + # + # The repository is cloned into `:root_path`. + # + # By not setting a repository URL, you have to fill and modify + # content into the `:root_path` by other means (even manually). + remote_repository_url: "", + + # How often in ms the RepositoryWatcher should pool + # the `:remote_repository_url` Git repository and `fetch` changes? + recheck_pending_remote_events_interval: 15_000 + # Should the main content tree contain a link to the Home/Root page ("/")? content_tree_display_home: false, @@ -456,6 +474,11 @@ You can support my open-source contributions and this project on [Patreon](https - Mike King +## Contributors + +- [Alfred Reinold Baudisch](https://github.com/alfredbaudisch/): PardallMarkdown creator and maintainer. +- [Cody Brunner](https://github.com/rockchalkwushock): added the initial implementation to remote content support via automatic Git repository pooling and fetching - `RepositoryWatcher` ([#31](https://github.com/alfredbaudisch/pardall_markdown/issues/31)). + ## Copyright License Copyright 2021 Alfred Reinold Baudisch (alfredbaudisch, pardall) diff --git a/config/config.exs b/config/config.exs index a60db8d..9c49585 100644 --- a/config/config.exs +++ b/config/config.exs @@ -19,7 +19,12 @@ config :pardall_markdown, PardallMarkdown.Content, notify_content_reloaded: fn -> :ok end, is_markdown_metadata_required: true, is_content_draft_by_default: true, - metadata_parser: PardallMarkdown.MetadataParser.ElixirMap + metadata_parser: PardallMarkdown.MetadataParser.ElixirMap, + # Git repository to watch and automatically fetch content from, leave "" or nil to not + # get content from a repository. + # Available sample content repo: "https://github.com/alfredbaudisch/pardall_markdown_sample_content", + remote_repository_url: "", + recheck_pending_remote_events_interval: 15_000 config :pardall_markdown, PardallMarkdown.MetadataParser.JoplinNote, metadata_parser_after_title: PardallMarkdown.MetadataParser.ElixirMap diff --git a/config/test.exs b/config/test.exs index 7aef638..bcd0f9c 100644 --- a/config/test.exs +++ b/config/test.exs @@ -5,7 +5,8 @@ config :pardall_markdown, PardallMarkdown.Content, static_assets_path: "./test/content/static", cache_name: :content_cache, index_cache_name: :content_index_cache, - is_markdown_metadata_required: true + is_markdown_metadata_required: true, + remote_repository_url: "" # Print only warnings and errors during test config :logger, level: :warn diff --git a/lib/pardall_markdown/application.ex b/lib/pardall_markdown/application.ex index 6a4889a..a00cf45 100644 --- a/lib/pardall_markdown/application.ex +++ b/lib/pardall_markdown/application.ex @@ -28,13 +28,21 @@ defmodule PardallMarkdown.Application do PardallMarkdown.FileWatcher, name: PardallMarkdown.FileWatcher, dirs: [PardallMarkdown.Content.Utils.root_path()] } - # Start a worker by calling: PardallMarkdown.Worker.start_link(arg) - # {PardallMarkdown.Worker, arg} ] + |> maybe_append_repository_watcher(Application.get_env(:pardall_markdown, PardallMarkdown.Content)[:remote_repository_url]) # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: PardallMarkdown.Supervisor] Supervisor.start_link(children, opts) end + + def maybe_append_repository_watcher(children, url) when is_nil(url) or url == "", do: children + def maybe_append_repository_watcher(children, url) do + children ++ + [{ + PardallMarkdown.RepositoryWatcher, + name: PardallMarkdown.RepositoryWatcher, repo: url + }] + end end diff --git a/lib/pardall_markdown/content/utils.ex b/lib/pardall_markdown/content/utils.ex index 340559f..1e0ddc3 100644 --- a/lib/pardall_markdown/content/utils.ex +++ b/lib/pardall_markdown/content/utils.ex @@ -30,6 +30,13 @@ defmodule PardallMarkdown.Content.Utils do :is_content_draft_by_default ] + def is_path_hidden?(path) do + not is_nil(path + |> String.replace(root_path(), "") + |> String.split("/") + |> Enum.find(fn path -> String.starts_with?(path, ".") end)) + end + @doc """ Splits a path into a tree of categories, containing both readable category names and slugs for all categories in the hierarchy. The categories list is indexed from the diff --git a/lib/pardall_markdown/file_parser.ex b/lib/pardall_markdown/file_parser.ex index 7a22213..39cac9c 100644 --- a/lib/pardall_markdown/file_parser.ex +++ b/lib/pardall_markdown/file_parser.ex @@ -21,9 +21,7 @@ defmodule PardallMarkdown.FileParser do defp should_extract_path?(path), do: File.exists?(path) and not is_path_from_static_assets?(path) and - not is_file_hidden?(path) - - defp is_file_hidden?(path), do: String.starts_with?(".", Path.basename(path)) + not is_path_hidden?(path) defp extract_folder_if_valid!(path), do: if(should_extract_path?(path), do: extract_folder!(path)) diff --git a/lib/pardall_markdown/file_watcher.ex b/lib/pardall_markdown/file_watcher.ex index e5dfe85..8b78c3b 100755 --- a/lib/pardall_markdown/file_watcher.ex +++ b/lib/pardall_markdown/file_watcher.ex @@ -17,12 +17,14 @@ defmodule PardallMarkdown.FileWatcher do {:ok, %{watcher_pid: watcher_pid, pending_events: 1, processing_events: 1}} end - def handle_info({:file_event, _, {_path, _event} = data}, %{pending_events: pending} = state) do - Logger.info("Received file event: #{inspect(data)}") - pending = pending + 1 - Logger.info("Event is valid. Pending events: #{pending}.") - - {:noreply, put_in(state[:pending_events], pending)} + def handle_info({:file_event, _, {path, event} = data}, %{pending_events: pending} = state) do + if should_process_event?(path, event) do + pending = pending + 1 + Logger.info("Received valid file event: #{inspect(data)}. Pending events: #{pending}.") + {:noreply, put_in(state[:pending_events], pending)} + else + {:noreply, state} + end end def handle_info({:file_event, _, :stop}, state) do @@ -91,6 +93,8 @@ defmodule PardallMarkdown.FileWatcher do |> Map.put(:processing_events, 0)} end + defp should_process_event?(path, _event), do: not PardallMarkdown.Content.Utils.is_path_hidden?(path) + defp schedule_next_recheck, do: Process.send_after(self(), :check_pending_events, @recheck_interval) diff --git a/lib/pardall_markdown/repository_provider.ex b/lib/pardall_markdown/repository_provider.ex new file mode 100644 index 0000000..8cdb861 --- /dev/null +++ b/lib/pardall_markdown/repository_provider.ex @@ -0,0 +1,43 @@ +defmodule PardallMarkdown.RepositoryProvider do + @moduledoc """ + This implementation closely follows the implementation found in: + https://github.com/meddle0x53/blogit/blob/master/lib/blogit/repository_provider.ex + Many thanks to @meddle0x53 for this! + """ + + @type repository :: term + @type provider :: module + @type fetch_result :: {:no_updates} | {:updates, [String.t()]} + + @type t :: %__MODULE__{repo: repository, provider: provider} + @enforce_keys :provider + defstruct [:repo, :provider] + + @doc """ + Invoked to get a representation value of the repository the provider manages. + The actual data represented by this struct should be updated to its + newest version first. + If for example the repository is remote, all the files in it should be + downloaded so their most recent versions are accessible. + This structure can be passed to other callbacks in order to manage files + in the repository. + """ + @callback repository() :: repository + + @doc """ + Invoked to update the data represented by the given `repository` to its most + recent version. + If, for example the repository is remote, all the files in it should be + downloaded so their most recent versions are accessible. + Returns the path to the changed files in the form of the tuple + `{:updates, list-of-paths}`. These paths should be paths to deleted, updated + or newly created files. + """ + @callback fetch(repository) :: fetch_result + + @doc """ + Invoked to get the path to the locally downloaded data. If the repository + is remote, it should have local copy or something like that. + """ + @callback local_path() :: String.t() +end diff --git a/lib/pardall_markdown/repository_providers/git.ex b/lib/pardall_markdown/repository_providers/git.ex new file mode 100644 index 0000000..caa0dfc --- /dev/null +++ b/lib/pardall_markdown/repository_providers/git.ex @@ -0,0 +1,83 @@ +defmodule PardallMarkdown.RepositoryProviders.Git do + @moduledoc """ + This implementation is based on the implementation found in: + https://github.com/meddle0x53/blogit/blob/master/lib/blogit/settings.ex + Many thanks to @meddle0x53 for this! + """ + require Logger + + @behaviour PardallMarkdown.RepositoryProvider + + @repository_url Application.get_env(:pardall_markdown, PardallMarkdown.Content)[ + :remote_repository_url + ] + @local_path PardallMarkdown.Content.Utils.root_path() + + # Callbacks + @impl true + def repository do + repo = git_repository() + + case Git.pull(repo) do + {:ok, msg} -> + Logger.info("Pulling from git repository #{msg}") + + # Crash in case of unstaged changes + {_, %Git.Error{ + args: [], code: 128, command: "pull", + message: "error: cannot pull with rebase: You have unstaged changes.\nerror: please commit or stash them.\n" = message + }} -> + raise message + + {_, error} -> + Logger.error("Error while pulling from git repository #{inspect(error)}") + end + + repo + end + + @impl true + def fetch(repo) do + Logger.info("Fetching data from #{@repository_url}") + + case Git.fetch(repo) do + {:error, _} -> + {:no_updates} + + {:ok, ""} -> + {:no_updates} + + {:ok, _} -> + updates = + repo + |> Git.diff!(["--name-only", "HEAD", "origin/master"]) + |> String.split("\n", trim: true) + |> Enum.map(&String.trim/1) + + Logger.info("There are new updates, pulling them.") + Git.pull!(repo) + + {:updates, updates} + end + end + + @impl true + def local_path, do: @local_path + + # Private + defp git_repository do + if repository_exists?() do + Git.new(@local_path) + else + case Git.clone([@repository_url, @local_path]) do + {:ok, repo} -> + Logger.info("Cloning repository #{@repository_url}, into #{@local_path}") + repo + {:error, %Git.Error{}} -> + Git.new(@local_path) + end + end + end + + defp repository_exists?, do: File.exists?(Path.join(@local_path, ".git")) +end diff --git a/lib/repository_watcher.ex b/lib/repository_watcher.ex new file mode 100644 index 0000000..c1b4be4 --- /dev/null +++ b/lib/repository_watcher.ex @@ -0,0 +1,46 @@ +defmodule PardallMarkdown.RepositoryWatcher do + @moduledoc """ + + """ + use GenServer + require Logger + + alias PardallMarkdown.RepositoryProviders.{Git} + alias PardallMarkdown.RepositoryProvider, as: Repository + + @recheck_interval Application.get_env(:pardall_markdown, PardallMarkdown.Content)[ + :recheck_pending_remote_events_interval + ] + + def start_link(args) do + GenServer.start_link(__MODULE__, provider: args[:repo]) + end + + @impl true + def init(provider) do + repo = Git.repository() + state = %Repository{repo: repo, provider: provider} + send_next_recheck() + {:ok, state} + end + + @impl true + def handle_info(:check_pending_remote_events, %{repo: repo} = state) do + # Fetch any potential changes from remote. + Git.fetch(repo) + send_next_recheck() + {:noreply, state} + end + + # In eithe case of :no_updates or :updates we don't care about what happened. + # keep passing state. FileWatcher and FileParser will handle any changes at + # the local_path. + + @impl true + def handle_info({_, :no_updates}, state), do: {:noreply, state} + + @impl true + def handle_info({_, {:updates, _}}, state), do: {:noreply, state} + + defp send_next_recheck, do: Process.send_after(self(), :check_pending_remote_events, @recheck_interval) +end diff --git a/mix.exs b/mix.exs index 20838b8..6e2bb7b 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule PardallMarkdown.MixProject do use Mix.Project @url "https://github.com/alfredbaudisch/pardall_markdown" - @version "0.3.3" + @version "0.4.0" def project do [ @@ -36,7 +36,8 @@ defmodule PardallMarkdown.MixProject do {:slugify, "~> 1.3"}, {:html_entities, "~> 0.5"}, {:con_cache, "~> 0.13"}, - {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} + {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, + {:git_cli, "~> 0.3"}, ] end diff --git a/mix.lock b/mix.lock index f40362b..db2f25f 100644 --- a/mix.lock +++ b/mix.lock @@ -10,6 +10,7 @@ "ex_doc": {:hex, :ex_doc, "0.25.2", "4f1cae793c4d132e06674b282f1d9ea3bf409bcca027ddb2fe177c4eed6a253f", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5b0c172e87ac27f14dfd152d52a145238ec71a95efbf29849550278c58a393d6"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "floki": {:hex, :floki, "0.31.0", "f05ee8a8e6a3ced4e62beeb2c79a63bc8e12ab98fbaaf6e6a3d9b76b1278e23f", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "b05afa372f5c345a5bf240ac25ea1f0f3d5fcfd7490ac0beeb4a203f9444891e"}, + "git_cli": {:hex, :git_cli, "0.3.0", "a5422f9b95c99483385b976f5d43f7e8233283a47cda13533d7c16131cb14df5", [:mix], [], "hexpm", "78cb952f4c86a41f4d3511f1d3ecb28edb268e3a7df278de2faa1bd4672eaf9b"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"},