Manage project-credentials via provisioning API (#2372)

* ref #2367, make creds unique by user by name * update cl * use random cred name * load credentials in job * WIP * WIP 2 * fix ordering issues * more tests * fix failing tests * update test * fix tests * default keyword list * remove globals * update cli * integration tests * dializer * update changelog --------- Co-authored-by: Taylor Downs <downs.taylor@gmail.com> Co-authored-by: Taylor Downs <taylor@openfn.org>
OpenFn · Aug 16, 2024 · 8397e33 · 8397e33
1 parent b99e718
commit 8397e33
Show file tree

Hide file tree

Showing 14 changed files with 265 additions and 59 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -33,13 +33,17 @@ and this project adheres to
   script. (This is a tricky requirement for a distributed set of local
   deployments) [#2369](https://github.com/OpenFn/lightning/issues/2369) and
   [#2373](https://github.com/OpenFn/lightning/pull/2373)
+- Added support for _very basic_ project-credential management (add, associate
+  with job) via provisioning API.
+  [#2367](https://github.com/OpenFn/lightning/issues/2367)
 
 ### Changed
 
 - Enforced uniqueness on credential names _by user_.
   [#2371](https://github.com/OpenFn/lightning/pull/2371)
 - Use Swoosh to format User models into recipients
   [#2374](https://github.com/OpenFn/lightning/pull/2374)
+- Bump default CLI to `@openfn/cli@1.8.1`
 
 ### Fixed
 

diff --git a/lib/lightning/export_utils.ex b/lib/lightning/export_utils.ex
@@ -14,16 +14,21 @@ defmodule Lightning.ExportUtils do
 
   defp hyphenate(other), do: other
 
-  defp job_to_treenode(job) do
+  defp job_to_treenode(job, project_credentials) do
+    project_credential =
+      Enum.find(project_credentials, fn pc ->
+        pc.id == job.project_credential_id
+      end)
+
     %{
       # The identifier here for our YAML reducer will be the hyphenated name
       id: hyphenate(job.name),
       name: job.name,
       node_type: :job,
       adaptor: job.adaptor,
       body: job.body,
-      credential: nil,
-      globals: []
+      credential:
+        project_credential && project_credential_key(project_credential)
     }
   end
 
@@ -92,6 +97,7 @@ defmodule Lightning.ExportUtils do
   defp pick_and_sort(map) do
     ordering_map = %{
       project: [:name, :description, :credentials, :globals, :workflows],
+      credential: [:name, :owner],
       workflow: [:name, :jobs, :triggers, :edges],
       job: [:name, :adaptor, :credential, :globals, :body],
       trigger: [:type, :cron_expression, :enabled],
@@ -161,7 +167,7 @@ defmodule Lightning.ExportUtils do
   end
 
   defp handle_input(key, value, indentation) when value in [%{}, [], nil] do
-    "#{indentation}# #{key}:"
+    "#{indentation}#{key}: null"
   end
 
   defp handle_input(key, value, indentation) when is_map(value) do
@@ -197,25 +203,51 @@ defmodule Lightning.ExportUtils do
       workflows
       |> Enum.sort_by(& &1.inserted_at, NaiveDateTime)
       |> Enum.reduce(%{}, fn workflow, acc ->
-        ytree = build_workflow_yaml_tree(workflow)
+        ytree = build_workflow_yaml_tree(workflow, project.project_credentials)
         Map.put(acc, hyphenate(workflow.name), ytree)
       end)
 
+    credentials_map =
+      project.project_credentials
+      |> Enum.sort_by(& &1.inserted_at, NaiveDateTime)
+      |> Enum.reduce(%{}, fn project_credential, acc ->
+        ytree = build_project_credential_yaml_tree(project_credential)
+
+        Map.put(
+          acc,
+          project_credential_key(project_credential),
+          ytree
+        )
+      end)
+
     %{
       name: project.name,
       description: project.description,
       node_type: :project,
-      globals: [],
       workflows: workflows_map,
-      credentials: []
+      credentials: credentials_map
+    }
+  end
+
+  defp project_credential_key(project_credential) do
+    hyphenate(
+      "#{project_credential.credential.user.email} #{project_credential.credential.name}"
+    )
+  end
+
+  defp build_project_credential_yaml_tree(project_credential) do
+    %{
+      name: project_credential.credential.name,
+      node_type: :credential,
+      owner: project_credential.credential.user.email
     }
   end
 
-  defp build_workflow_yaml_tree(workflow) do
+  defp build_workflow_yaml_tree(workflow, project_credentials) do
     jobs =
       workflow.jobs
       |> Enum.sort_by(& &1.inserted_at, NaiveDateTime)
-      |> Enum.map(fn j -> job_to_treenode(j) end)
+      |> Enum.map(fn j -> job_to_treenode(j, project_credentials) end)
 
     triggers =
       workflow.triggers
@@ -240,6 +272,9 @@ defmodule Lightning.ExportUtils do
   def generate_new_yaml(project, snapshots \\ nil)
 
   def generate_new_yaml(project, nil) do
+    project =
+      Lightning.Repo.preload(project, project_credentials: [credential: :user])
+
     yaml =
       project
       |> Workflows.get_workflows_for()
@@ -250,6 +285,9 @@ defmodule Lightning.ExportUtils do
   end
 
   def generate_new_yaml(project, snapshots) when is_list(snapshots) do
+    project =
+      Lightning.Repo.preload(project, project_credentials: [credential: :user])
+
     yaml =
       snapshots
       |> Enum.sort_by(& &1.name)

diff --git a/lib/lightning/projects/project.ex b/lib/lightning/projects/project.ex
@@ -34,14 +34,14 @@ defmodule Lightning.Projects.Project do
 
     has_many :project_users, ProjectUser
     has_many :users, through: [:project_users, :user]
-    has_many :project_credentials, ProjectCredential
-    has_many :credentials, through: [:project_credentials, :credential]
     has_many :project_oauth_clients, ProjectOauthClient
     has_many :oauth_clients, through: [:project_oauth_clients, :oauth_client]
 
     has_many :workflows, Workflow
     has_many :jobs, through: [:workflows, :jobs]
 
+    has_many :project_credentials, ProjectCredential
+    has_many :credentials, through: [:project_credentials, :credential]
     timestamps()
   end
 

diff --git a/lib/lightning/projects/provisioner.ex b/lib/lightning/projects/provisioner.ex
@@ -14,6 +14,7 @@ defmodule Lightning.Projects.Provisioner do
   alias Ecto.Multi
   alias Lightning.Accounts.User
   alias Lightning.Projects.Project
+  alias Lightning.Projects.ProjectCredential
   alias Lightning.Projects.ProjectUser
   alias Lightning.Repo
   alias Lightning.VersionControl.ProjectRepoConnection
@@ -61,6 +62,7 @@ defmodule Lightning.Projects.Provisioner do
     |> preload_dependencies()
     |> parse_document(data)
     |> maybe_add_project_user(user_or_repo_connection)
+    |> maybe_add_project_credentials(user_or_repo_connection)
   end
 
   defp create_snapshots(project_changeset, inserted_workflows) do
@@ -143,13 +145,16 @@ defmodule Lightning.Projects.Provisioner do
       project,
       [
         :project_users,
+        project_credentials: [credential: [:user]],
         workflows: {w, [:jobs, :triggers, :edges]}
       ],
       force: true
     )
   end
 
   def preload_dependencies(project, snapshots) when is_list(snapshots) do
+    project = preload_dependencies(project)
+
     %{project | workflows: Snapshot.get_all_by_ids(snapshots)}
   end
 
@@ -176,7 +181,7 @@ defmodule Lightning.Projects.Provisioner do
 
   defp job_changeset(job, attrs) do
     job
-    |> cast(attrs, [:id, :name, :body, :adaptor, :delete])
+    |> cast(attrs, [:id, :name, :body, :adaptor, :delete, :project_credential_id])
     |> validate_required([:id])
     |> unique_constraint(:id, name: :jobs_pkey)
     |> Job.validate()
@@ -251,6 +256,60 @@ defmodule Lightning.Projects.Provisioner do
     end
   end
 
+  defp maybe_add_project_credentials(changeset, user_or_repo_connection) do
+    credentials_params = changeset.params["project_credentials"]
+
+    if is_struct(user_or_repo_connection, User) and is_list(credentials_params) do
+      user_credentials =
+        user_or_repo_connection
+        |> Ecto.assoc(:credentials)
+        |> Repo.all()
+
+      existing_project_credential_ids =
+        Enum.map(changeset.data.project_credentials, fn pc -> pc.id end)
+
+      new_credential_params =
+        Enum.filter(credentials_params, fn cred_params ->
+          cred_params["id"] not in existing_project_credential_ids and
+            cred_params["owner"] == user_or_repo_connection.email
+        end)
+
+      new_project_creds_to_add =
+        Enum.map(new_credential_params, fn cred_params ->
+          credential =
+            Enum.find(user_credentials, fn cred ->
+              cred.name == cred_params["name"]
+            end)
+
+          if credential do
+            change(%ProjectCredential{
+              id: cred_params["id"],
+              credential_id: credential.id
+            })
+          else
+            change(%ProjectCredential{
+              id: cred_params["id"]
+            })
+            |> add_error(
+              :credential,
+              "No credential found with name #{cred_params["name"]}"
+            )
+          end
+        end)
+
+      project_credentials =
+        Enum.map(changeset.data.project_credentials, &change/1)
+
+      put_assoc(
+        changeset,
+        :project_credentials,
+        new_project_creds_to_add ++ project_credentials
+      )
+    else
+      changeset
+    end
+  end
+
   @doc """
   Validate that there are no extraneous parameters in the changeset.
 
@@ -259,6 +318,7 @@ defmodule Lightning.Projects.Provisioner do
   """
   def validate_extraneous_params(changeset) do
     param_keys = changeset.params |> Map.keys() |> MapSet.new(&to_string/1)
+
     field_keys = changeset.types |> Map.keys() |> MapSet.new(&to_string/1)
 
     extraneous_params = MapSet.difference(param_keys, field_keys)

diff --git a/lib/lightning/workflows/job.ex b/lib/lightning/workflows/job.ex
@@ -39,6 +39,7 @@ defmodule Lightning.Workflows.Job do
 
     belongs_to :project_credential, ProjectCredential
     has_one :credential, through: [:project_credential, :credential]
+
     belongs_to :workflow, Workflow
     has_one :project, through: [:workflow, :project]
 
@@ -79,6 +80,9 @@ defmodule Lightning.Workflows.Job do
     |> validate_required(:name, message: "job name can't be blank")
     |> validate_required(:body, message: "job body can't be blank")
     |> validate_required(:adaptor, message: "job adaptor can't be blank")
+    |> foreign_key_constraint(:project_credential_id,
+      message: "credential doesn't exist or isn't available in this project"
+    )
     |> assoc_constraint(:workflow)
     |> validate_length(:name,
       max: 100,

diff --git a/lib/lightning_web/controllers/api/provisioning_json.ex b/lib/lightning_web/controllers/api/provisioning_json.ex
@@ -5,6 +5,7 @@ defmodule LightningWeb.API.ProvisioningJSON do
   import Ecto.Changeset
 
   alias Lightning.Projects.Project
+  alias Lightning.Projects.ProjectCredential
   alias Lightning.Workflows.Edge
   alias Lightning.Workflows.Job
   alias Lightning.Workflows.Snapshot
@@ -27,6 +28,12 @@ defmodule LightningWeb.API.ProvisioningJSON do
       |> Enum.sort_by(& &1.inserted_at, NaiveDateTime)
       |> Enum.map(&as_json/1)
     )
+    |> Map.put(
+      :project_credentials,
+      project.project_credentials
+      |> Enum.sort_by(& &1.inserted_at, NaiveDateTime)
+      |> Enum.map(&as_json/1)
+    )
   end
 
   def as_json(%module{} = workflow_or_snapshot)
@@ -63,7 +70,7 @@ defmodule LightningWeb.API.ProvisioningJSON do
 
   def as_json(%module{} = job) when module in [Job, Snapshot.Job] do
     Ecto.embedded_dump(job, :json)
-    |> Map.take(~w(id adaptor body name)a)
+    |> Map.take(~w(id adaptor body name project_credential_id)a)
   end
 
   def as_json(%module{} = trigger) when module in [Trigger, Snapshot.Trigger] do
@@ -81,6 +88,14 @@ defmodule LightningWeb.API.ProvisioningJSON do
     |> drop_keys_with_nil_value()
   end
 
+  def as_json(%ProjectCredential{} = project_credential) do
+    %{
+      id: project_credential.id,
+      name: project_credential.credential.name,
+      owner: project_credential.credential.user.email
+    }
+  end
+
   defp drop_keys_with_nil_value(map) do
     Map.reject(map, fn {_, v} -> is_nil(v) end)
   end

diff --git a/lib/mix/tasks/install_runtime.ex b/lib/mix/tasks/install_runtime.ex
@@ -43,7 +43,7 @@ defmodule Mix.Tasks.Lightning.InstallRuntime do
 
   def packages do
     ~W(
-      @openfn/cli@1.3.2
+      @openfn/cli@1.8.1
       @openfn/language-common@latest
     )
   end