Add new check_source check to check source code as a string (#189)

Co-authored-by: Tim Austin <tim@neenjaw.com>
Co-authored-by: Angelika Tyborska <angelikatyborska@fastmail.com>

lib/elixir_analyzer/constants.ex

@@ -32,6 +32,7 @@ defmodule ElixirAnalyzer.Constants do
       "elixir.solution.defmacro_with_is_and_question_mark",
     solution_same_as_exemplar: "elixir.solution.same_as_exemplar",
     solution_list_prepend_head: "elixir.solution.list_prepend_head",
+    solution_no_integer_literal: "elixir.solution.no_integer_literal",
 
     # Concept exercises
 

lib/elixir_analyzer/exercise_test.ex

@@ -3,6 +3,7 @@ defmodule ElixirAnalyzer.ExerciseTest do
 
   alias ElixirAnalyzer.ExerciseTest.Feature.Compiler, as: FeatureCompiler
   alias ElixirAnalyzer.ExerciseTest.AssertCall.Compiler, as: AssertCallCompiler
+  alias ElixirAnalyzer.ExerciseTest.CheckSource.Compiler, as: CheckSourceCompiler
   alias ElixirAnalyzer.ExerciseTest.CommonChecks
 
   alias ElixirAnalyzer.Submission
@@ -14,6 +15,7 @@ defmodule ElixirAnalyzer.ExerciseTest do
     quote do
       use ElixirAnalyzer.ExerciseTest.Feature
       use ElixirAnalyzer.ExerciseTest.AssertCall
+      use ElixirAnalyzer.ExerciseTest.CheckSource
       use ElixirAnalyzer.ExerciseTest.CommonChecks
       @suppress_tests unquote(opts)[:suppress_tests]
 
@@ -31,17 +33,23 @@ defmodule ElixirAnalyzer.ExerciseTest do
     # credo:disable-for-previous-line Credo.Check.Refactor.CyclomaticComplexity
     feature_test_data = Module.get_attribute(env.module, :feature_tests)
     assert_call_data = Module.get_attribute(env.module, :assert_call_tests)
+    check_source_data = Module.get_attribute(env.module, :check_source_tests)
     suppress_tests = Module.get_attribute(env.module, :suppress_tests, [])
 
-    # ast placeholder for the submission code ast
+    # placeholders for submission code
     code_ast = quote do: code_ast
+    code_as_string = quote do: code_as_string
 
     # compile each feature to a test
     feature_tests = Enum.map(feature_test_data, &FeatureCompiler.compile(&1, code_ast))
 
     # compile each assert_call to a test
     assert_call_tests = Enum.map(assert_call_data, &AssertCallCompiler.compile(&1, code_ast))
 
+    # compile each check_source to a test
+    check_source_tests =
+      Enum.map(check_source_data, &CheckSourceCompiler.compile(&1, code_as_string))
+
     quote do
       @spec analyze(Submission.t(), String.t(), nil | Macro.t()) :: Submission.t()
       def analyze(%Submission{} = submission, code_as_string, exemplar_ast) do
@@ -60,6 +68,7 @@ defmodule ElixirAnalyzer.ExerciseTest do
           Enum.concat([
             unquote(feature_tests),
             unquote(assert_call_tests),
+            unquote(check_source_tests),
             CommonChecks.run(code_ast, code_as_string, exemplar_ast)
           ])
           |> filter_suppressed_results()

lib/elixir_analyzer/exercise_test/check_source.ex

@@ -0,0 +1,101 @@
+defmodule ElixirAnalyzer.ExerciseTest.CheckSource do
+  @moduledoc """
+  Defines a `check_source` macro that allows checking the source code
+  """
+
+  @doc false
+  defmacro __using__(_opts) do
+    quote do
+      import unquote(__MODULE__)
+
+      @check_source_tests []
+    end
+  end
+
+  @doc """
+  Defines a macro which runs a boolean function on the source code.
+
+  This macro then collates the block into a map structure resembling:
+  test_data = %{
+    description: description,
+    type: :actionable,
+    comment: "message",
+    suppress_if: {"name of other test", :fail}
+  }
+  and an AST for the function
+  """
+  defmacro check_source(description, do: block) do
+    :ok = validate_check_block(block)
+
+    test_data =
+      block
+      |> walk_check_source_block()
+      |> Map.put(:description, description)
+      |> Map.put_new(:type, :informative)
+
+    check = test_data.check
+
+    test_data =
+      test_data
+      |> Map.delete(:check)
+      # made into a key-val list for better quoting
+      |> Map.to_list()
+
+    unless Keyword.has_key?(test_data, :comment) do
+      raise "Comment must be defined for each check_source test"
+    end
+
+    quote do
+      @check_source_tests [
+        {unquote(test_data), unquote(Macro.escape(check))} | @check_source_tests
+      ]
+    end
+  end
+
+  @supported_expressions [:comment, :type, :suppress_if, :check]
+  defp validate_check_block({:__block__, _, args}) do
+    Enum.each(args, fn {name, _, _} ->
+      if name not in @supported_expressions do
+        raise """
+        Unsupported expression `#{name}`.
+        The macro `check_source` supports expressions: #{Enum.join(@supported_expressions, ", ")}.
+        """
+      end
+    end)
+  end
+
+  defp walk_check_source_block(block, test_data \\ %{}) do
+    {_, test_data} = Macro.prewalk(block, test_data, &do_walk_check_source_block/2)
+    test_data
+  end
+
+  defp do_walk_check_source_block({:comment, _, [comment]} = node, test_data) do
+    {node, Map.put(test_data, :comment, comment)}
+  end
+
+  @supported_types ~w(essential actionable informative celebratory)a
+  defp do_walk_check_source_block({:type, _, [type]} = node, test_data) do
+    if type not in @supported_types do
+      raise """
+      Unsupported type `#{type}`.
+      The macro `check_source` supports the following types: #{Enum.join(@supported_types, ", ")}.
+      """
+    end
+
+    {node, Map.put(test_data, :type, type)}
+  end
+
+  defp do_walk_check_source_block({:suppress_if, _, [name, condition]} = node, test_data) do
+    {node, Map.put(test_data, :suppress_if, {name, condition})}
+  end
+
+  defp do_walk_check_source_block({:check, _, [source, [do: function]]} = node, test_data) do
+    function = {:fn, [], [{:->, [], [[source], function]}]}
+
+    {node, Map.put(test_data, :check, function)}
+  end
+
+  defp do_walk_check_source_block(node, test_data) do
+    {node, test_data}
+  end
+end

lib/elixir_analyzer/exercise_test/check_source/compiler.ex

@@ -0,0 +1,30 @@
+defmodule ElixirAnalyzer.ExerciseTest.CheckSource.Compiler do
+  @moduledoc false
+
+  alias ElixirAnalyzer.Comment
+
+  def compile({check_source_data, check_function}, code_string) do
+    name = Keyword.fetch!(check_source_data, :description)
+    comment = Keyword.fetch!(check_source_data, :comment)
+    type = Keyword.get(check_source_data, :type, :informative)
+    suppress_if = Keyword.get(check_source_data, :suppress_if, false)
+
+    test_description =
+      Macro.escape(%Comment{
+        name: name,
+        comment: comment,
+        type: type,
+        suppress_if: suppress_if
+      })
+
+    quote do
+      (fn string ->
+         if unquote(check_function).(string) do
+           {:pass, unquote(test_description)}
+         else
+           {:fail, unquote(test_description)}
+         end
+       end).(unquote(code_string))
+    end
+  end
+end

lib/elixir_analyzer/test_suite/german_sysadmin.ex

@@ -35,4 +35,14 @@ defmodule ElixirAnalyzer.TestSuite.GermanSysadmin do
     called_fn name: :case
     comment Constants.german_sysadmin_use_case()
   end
+
+  check_source "does not use integer literals for code points" do
+    type :actionable
+    comment Constants.solution_no_integer_literal()
+
+    check(source) do
+      integers = ["?ß", "?ä", "?ö", "?ü", "?_", "?a", "?z"]
+      Enum.all?(integers, &String.contains?(source, &1))
+    end
+  end
 end

test/elixir_analyzer/exercise_test/check_source_test.exs

@@ -0,0 +1,124 @@
+defmodule ElixirAnalyzer.ExerciseTest.CheckSourceTest do
+  use ElixirAnalyzer.ExerciseTestCase,
+    exercise_test_module: ElixirAnalyzer.Support.AnalyzerVerification.CheckSource
+
+  test_exercise_analysis "empty module",
+    comments: ["always return false", "didn't use multiline"] do
+    ~S"""
+    defmodule CheckSourceVerification do
+    end
+    """
+  end
+
+  test_exercise_analysis "contains integer literals",
+    comments: [
+      "always return false",
+      "used integer literal from ?a to ?z",
+      "didn't use multiline"
+    ] do
+    ~S"""
+    defmodule CheckSourceVerification do
+      def foo(x) do
+        case x do
+          97 -> "a"
+          98 -> "b"
+          _ -> "z"
+        end
+      end
+    end
+    """
+  end
+
+  test_exercise_analysis "contains integer but false positive",
+    comments: [
+      "always return false",
+      "used integer literal from ?a to ?z",
+      "didn't use multiline"
+    ] do
+    ~S"""
+    defmodule CheckSourceVerification do
+      def best_version(game) do
+        case game do
+          "fifa" -> "fifa 98"
+          "tomb raider" -> "tomb raider II (1997)"
+          _ -> "goat simulator"
+        end
+      end
+    end
+    """
+  end
+
+  test_exercise_analysis "uses multiline strings",
+    comments: ["always return false"] do
+    [
+      ~S'''
+      defmodule CheckSourceVerification do
+        @moduledoc """
+        this module doesn't do much
+        """
+      end
+      ''',
+      ~S'''
+      defmodule CheckSourceVerification do
+        def foo do
+          """
+          all
+          you
+          need
+          is
+          love
+          """
+        end
+      end
+      ''',
+      ~S"""
+      defmodule CheckSourceVerification do
+        def foo do
+          '''
+          love
+          is
+          all
+          you
+          need
+          '''
+        end
+      end
+      """
+    ]
+  end
+
+  test_exercise_analysis "short module",
+    comments: ["always return false", "module is too short", "didn't use multiline"] do
+    ~S"""
+    defmodule C do
+    end
+    """
+  end
+
+  test_exercise_analysis "badly formatted modules",
+    comments: ["always return false", "didn't use multiline", "module is not formatted"] do
+    [
+      ~S"""
+      defmodule CheckSourceVerification do
+                                    def foo(), do: :ok
+      end
+      """,
+      ~S"""
+      defmodule CheckSourceVerification do
+
+
+
+      def foo, do: :ok
+
+      end
+      """,
+      ~S"""
+      defmodule    CheckSourceVerification     do
+      end
+      """,
+      ~S"""
+      defmodule CheckSourceVerification do end
+      """
+    ]
+  end
+end