From 25028442416c459a95303a77da21809592e2114c Mon Sep 17 00:00:00 2001 From: Jie Date: Mon, 14 Feb 2022 16:38:15 +0900 Subject: [PATCH] 100% test coverage (#246) --- README.md | 8 - lib/elixir_analyzer.ex | 18 +- lib/elixir_analyzer/cli.ex | 51 ++-- lib/elixir_analyzer/exercise_test.ex | 11 +- .../exercise_test/assert_call/compiler.ex | 26 +- .../common_checks/function_capture.ex | 4 +- lib/elixir_analyzer/exercise_test/feature.ex | 10 +- lib/elixir_analyzer/quote_util.ex | 27 --- test/elixir_analyzer/cli_test.exs | 119 ++++++++++ test/elixir_analyzer/constants_test.exs | 5 + .../exercise_test/assert_call_test.exs | 121 ++++++++++ .../exercise_test/check_source_test.exs | 85 +++++++ .../common_checks/function_capture_test.exs | 25 ++ .../feature/duplicate_features_test.exs | 94 +++++--- .../exercise_test/feature_test.exs | 223 ++++++++++++++++++ test/elixir_analyzer/exercise_test_test.exs | 39 +++ test/elixir_analyzer/log_formatter_test.exs | 13 + test/elixir_analyzer_test.exs | 127 +++++++++- test/support/exercise_test_case.ex | 11 +- .../lasagna/wrong_config/.meta/config.json | 11 + .../lasagna/wrong_config2/.meta/config.json | 22 ++ .../informative_comments/.meta/config.json | 10 + .../informative_comments/.meta/example.ex | 7 + .../informative_comments/lib/two_fer.ex | 14 ++ 24 files changed, 923 insertions(+), 158 deletions(-) create mode 100644 test/elixir_analyzer/cli_test.exs create mode 100644 test/elixir_analyzer/log_formatter_test.exs create mode 100644 test_data/lasagna/wrong_config/.meta/config.json create mode 100644 test_data/lasagna/wrong_config2/.meta/config.json create mode 100644 test_data/two_fer/informative_comments/.meta/config.json create mode 100644 test_data/two_fer/informative_comments/.meta/example.ex create mode 100644 test_data/two_fer/informative_comments/lib/two_fer.ex diff --git a/README.md b/README.md index 4e741e46..b568e5aa 100644 --- a/README.md +++ b/README.md @@ -29,14 +29,6 @@ Running `bin/elixir_analyzer` on a system with Elixir/Erlang/OTP installed ```text Usage: $ elixir_analyzer [options] - - You may also pass the following options: - --skip-analysis flag skips running the static analysis - --output-file - - You may also test only individual files : - (assuming analyzer tests are compiled for the named module) - $ exercism_analyzer --analyze-file : ``` ### via IEX diff --git a/lib/elixir_analyzer.ex b/lib/elixir_analyzer.ex index 2428b77d..95786767 100644 --- a/lib/elixir_analyzer.ex +++ b/lib/elixir_analyzer.ex @@ -29,18 +29,14 @@ defmodule ElixirAnalyzer do * `exercise` is which exercise is submitted to determine proper analysis - * `path` is the path (ending with a '/') to the submitted solution + * `input_path` is the path to the submitted solution + + * `output_path` is the path to the output folder * `opts` is a Keyword List of options, see **options** ## Options - * `:exercise` - name of the exercise, defaults to the `exercise` parameter - - * `:path` - path to the submitted solution, defaults to the `path` parameter - - * `:output_path` - path to write file output, defaults to the `path` parameter - * `:output_file`, - specifies the name of the output_file, defaults to `@output_file` (`analysis.json`) @@ -52,8 +48,6 @@ defmodule ElixirAnalyzer do * `:puts_summary` - boolean flag if an analysis should print the summary of the analysis to stdio, defaults to `true` - - Any arbitrary keyword-value pair can be passed to `analyze_exercise/3` and these options may be used the other consuming code. """ @spec analyze_exercise(String.t(), String.t(), String.t(), keyword()) :: Submission.t() def analyze_exercise(exercise, input_path, output_path, opts \\ []) do @@ -147,14 +141,14 @@ defmodule ElixirAnalyzer do } rescue e in File.Error -> - Logger.warning("Unable to decode 'config.json'", error_message: e.message) + Logger.warning("Unable to read config file #{e.path}", error_message: e.reason) submission |> Submission.halt() |> Submission.set_halt_reason("Analysis skipped, not able to read solution config.") e in Jason.DecodeError -> - Logger.warning("Unable to decode 'config.json'", error_message: e.message) + Logger.warning("Unable to decode 'config.json'", data: e.data) submission |> Submission.halt() @@ -256,7 +250,7 @@ defmodule ElixirAnalyzer do submission = submission - |> submission.analysis_module.analyze(submission.source) + |> submission.analysis_module.analyze() |> Submission.set_analyzed(true) Logger.info("Analyzing code complete") diff --git a/lib/elixir_analyzer/cli.ex b/lib/elixir_analyzer/cli.ex index 44068de4..568e6758 100644 --- a/lib/elixir_analyzer/cli.ex +++ b/lib/elixir_analyzer/cli.ex @@ -6,22 +6,19 @@ defmodule ElixirAnalyzer.CLI do @usage """ Usage: - $ elixir_analyzer [options] + $ elixir_analyzer [options] You may also pass the following options: - --skip-analysis flag skips running the static analysis - --output-file - - You may also test only individual files : - (assuming analyzer tests are compiled for the named module) - - $ exercism_analyzer --analyze-file : + --help see this message + --output-file output file name (default: analysis.json) + --no-write-results doesn't write to JSON file + --no-puts-summary doesn't print summary to stdio """ @options [ - {{:skip_analyze, :boolean}, false}, {{:output_file, :string}, "analysis.json"}, - {{:analyze_file, :string}, nil}, + {{:write_results, :boolean}, true}, + {{:puts_summary, :boolean}, true}, {{:help, :boolean}, false} ] @@ -30,45 +27,27 @@ defmodule ElixirAnalyzer.CLI do args |> parse_args() |> process() end - def parse_args(args) do - options = %{ - :output_file => "analysis.json" - } + defp parse_args(args) do + default_ops = for({{key, _}, val} <- @options, do: {key, val}, into: %{}) - cmd_opts = - OptionParser.parse(args, - strict: for({o, _} <- @options, do: o) - ) + cmd_opts = OptionParser.parse(args, strict: for({o, _} <- @options, do: o)) case cmd_opts do {[help: true], _, _} -> :help - {[analyze_file: target], _, _} -> - [full_path, module] = String.split(target, ":", trim: true) - path = Path.dirname(full_path) - file = Path.basename(full_path) - {Enum.into([module: module, file: file], options), "undefined", path} - {opts, [exercise, input_path, output_path], _} -> - {Enum.into(opts, options), exercise, input_path, output_path} + {Enum.into(opts, default_ops), exercise, input_path, output_path} end rescue _ -> :help end - def process(:help), do: IO.puts(@usage) + defp process(:help), do: IO.puts(@usage) - def process({options, exercise, input_path, output_path}) do - opts = get_default_options(options) - ElixirAnalyzer.analyze_exercise(exercise, input_path, output_path, opts) - end + defp process({options, exercise, input_path, output_path}) do + opts = Map.to_list(options) - defp get_default_options(options) do - @options - |> Enum.reduce(options, fn {{option, _}, default}, acc -> - Map.put_new(acc, option, default) - end) - |> Map.to_list() + ElixirAnalyzer.analyze_exercise(exercise, input_path, output_path, opts) end end diff --git a/lib/elixir_analyzer/exercise_test.ex b/lib/elixir_analyzer/exercise_test.ex index 514cbec8..72fab337 100644 --- a/lib/elixir_analyzer/exercise_test.ex +++ b/lib/elixir_analyzer/exercise_test.ex @@ -22,7 +22,7 @@ defmodule ElixirAnalyzer.ExerciseTest do import unquote(__MODULE__) @before_compile unquote(__MODULE__) - @dialyzer no_match: {:do_analyze, 2} + @dialyzer no_match: {:do_analyze, 1} end end @@ -51,19 +51,20 @@ defmodule ElixirAnalyzer.ExerciseTest do check_source_tests = Enum.map(check_source_data, &CheckSourceCompiler.compile(&1, source)) quote do - @spec analyze(Submission.t(), Source.t()) :: Submission.t() - def analyze(%Submission{} = submission, %Source{code_string: code_string} = source) do + @spec analyze(Submission.t()) :: Submission.t() + def analyze(%Submission{source: %Source{code_string: code_string} = source} = submission) do case Code.string_to_quoted(code_string) do {:ok, code_ast} -> source = %{source | code_ast: code_ast} - do_analyze(submission, source) + submission = %{submission | source: source} + do_analyze(submission) {:error, e} -> append_analysis_failure(submission, e) end end - defp do_analyze(%Submission{} = submission, %Source{code_ast: code_ast} = source) do + defp do_analyze(%Submission{source: %Source{code_ast: code_ast} = source} = submission) do results = Enum.concat([ unquote(feature_tests), diff --git a/lib/elixir_analyzer/exercise_test/assert_call/compiler.ex b/lib/elixir_analyzer/exercise_test/assert_call/compiler.ex index f2ea84e0..f70f8331 100644 --- a/lib/elixir_analyzer/exercise_test/assert_call/compiler.ex +++ b/lib/elixir_analyzer/exercise_test/assert_call/compiler.ex @@ -150,10 +150,9 @@ defmodule ElixirAnalyzer.ExerciseTest.AssertCall.Compiler do """ @spec matching_function_call?( Macro.t(), - nil | AssertCall.function_signature(), + AssertCall.function_signature(), %{[atom] => [atom] | keyword()} ) :: boolean() - def matching_function_call?(_node, nil, _), do: false # For erlang libraries: :math._ or :math.pow def matching_function_call?( @@ -209,22 +208,6 @@ defmodule ElixirAnalyzer.ExerciseTest.AssertCall.Compiler do def matching_function_call?(_, _, _), do: false - @doc """ - compare a node to the function_signature, looking for a match for a called function - """ - @spec matching_function_def?(Macro.t(), AssertCall.function_signature()) :: boolean() - def matching_function_def?(_node, nil), do: false - - def matching_function_def?( - {def_type, _, [{name, _, _args}, [do: {:__block__, _, [_ | _]}]]}, - {_module_path, name} - ) - when def_type in ~w[def defp]a do - true - end - - def matching_function_def?(_, _), do: false - @doc """ node is a module definition """ @@ -237,13 +220,10 @@ defmodule ElixirAnalyzer.ExerciseTest.AssertCall.Compiler do def extract_module_name({:defmodule, _, [{:__aliases__, _, name}, [do: _]]}), do: name - def extract_module_name(_), do: nil - @doc """ node is a function definition """ - def function_def?({def_type, _, [{name, _, _}, [do: _]]}) - when is_atom(name) and def_type in ~w[def defp]a do + def function_def?({def_type, _, [_, [do: _]]}) when def_type in ~w[def defp]a do true end @@ -260,8 +240,6 @@ defmodule ElixirAnalyzer.ExerciseTest.AssertCall.Compiler do when is_atom(name) and def_type in ~w[def defp]a, do: name - def extract_function_name(_), do: nil - @doc """ compare the name of the function to the function signature, if they match return true """ diff --git a/lib/elixir_analyzer/exercise_test/common_checks/function_capture.ex b/lib/elixir_analyzer/exercise_test/common_checks/function_capture.ex index 46ef6e2f..3c90f85b 100644 --- a/lib/elixir_analyzer/exercise_test/common_checks/function_capture.ex +++ b/lib/elixir_analyzer/exercise_test/common_checks/function_capture.ex @@ -54,13 +54,13 @@ defmodule ElixirAnalyzer.ExerciseTest.CommonChecks.FunctionCapture do depth = depth - 1 functions = - if depth == 0 and wrong_use? and actual_function?(name) and name not in @exceptions do + if depth <= 0 and wrong_use? and actual_function?(name) and name not in @exceptions do [{:&, name, length(args)} | functions] else functions end - {node, %{capture_depth: depth - 1, functions: functions}} + {node, %{capture_depth: depth, functions: functions}} end # fn -> foo end diff --git a/lib/elixir_analyzer/exercise_test/feature.ex b/lib/elixir_analyzer/exercise_test/feature.ex index 51a7b240..fdee9a73 100644 --- a/lib/elixir_analyzer/exercise_test/feature.ex +++ b/lib/elixir_analyzer/exercise_test/feature.ex @@ -53,10 +53,16 @@ defmodule ElixirAnalyzer.ExerciseTest.Feature do feature_data = %{feature_data | meta: Map.to_list(feature_data.meta)} feature_data = Map.to_list(feature_data) + unless Keyword.has_key?(feature_data, :comment) do + raise "Comment must be defined for each feature test" + end + quote do # Check if the feature is unique - case Enum.filter(@feature_tests, fn {_data, forms} -> - forms == unquote(Macro.escape(feature_forms)) + case Enum.filter(@feature_tests, fn {data, forms} -> + {Keyword.get(data, :find), Keyword.get(data, :depth), forms} == + {Keyword.get(unquote(feature_data), :find), + Keyword.get(unquote(feature_data), :depth), unquote(Macro.escape(feature_forms))} end) do [{data, _forms} | _] -> raise FeatureError, diff --git a/lib/elixir_analyzer/quote_util.ex b/lib/elixir_analyzer/quote_util.ex index 605dec93..22bdbcd8 100644 --- a/lib/elixir_analyzer/quote_util.ex +++ b/lib/elixir_analyzer/quote_util.ex @@ -68,15 +68,6 @@ defmodule ElixirAnalyzer.QuoteUtil do end) end - @doc """ - Performs a depth-first, pre-order traversal of quoted expressions. - With depth provided to a function - """ - @spec prewalk(Macro.t(), (Macro.t(), non_neg_integer -> Macro.t())) :: Macro.t() - def prewalk(ast, fun) when is_function(fun, 2) do - elem(prewalk(ast, nil, fn x, nil, d -> {fun.(x, d), nil} end), 0) - end - @doc """ Performs a depth-first, pre-order traversal of quoted expressions using an accumulator. @@ -86,22 +77,4 @@ defmodule ElixirAnalyzer.QuoteUtil do def prewalk(ast, acc, fun) when is_function(fun, 3) do traverse_with_depth(ast, acc, fun, fn x, a, _d -> {x, a} end) end - - @doc """ - Performs a depth-first, post-order traversal of quoted expressions. - """ - @spec postwalk(Macro.t(), (Macro.t(), non_neg_integer -> Macro.t())) :: Macro.t() - def postwalk(ast, fun) when is_function(fun, 2) do - elem(postwalk(ast, nil, fn x, nil, d -> {fun.(x, d), nil} end), 0) - end - - @doc """ - Performs a depth-first, post-order traversal of quoted expressions - using an accumulator. - """ - @spec postwalk(Macro.t(), any, (Macro.t(), any, non_neg_integer -> {Macro.t(), any})) :: - {Macro.t(), any} - def postwalk(ast, acc, fun) when is_function(fun, 3) do - traverse_with_depth(ast, acc, fn x, a, _d -> {x, a} end, fun) - end end diff --git a/test/elixir_analyzer/cli_test.exs b/test/elixir_analyzer/cli_test.exs new file mode 100644 index 00000000..a313e07f --- /dev/null +++ b/test/elixir_analyzer/cli_test.exs @@ -0,0 +1,119 @@ +defmodule ElixirAnalyzer.CLITest do + use ExUnit.Case + import ExUnit.CaptureIO + + alias ElixirAnalyzer.CLI + alias ElixirAnalyzer.{Source, Submission} + + @lasagna_path "test_data/lasagna/perfect_solution" + @help """ + Usage: + + $ elixir_analyzer [options] + + You may also pass the following options: + --help see this message + --output-file output file name (default: analysis.json) + --no-write-results doesn't write to JSON file + --no-puts-summary doesn't print summary to stdio + """ + + defp match_submission(%Submission{ + analysis_module: ElixirAnalyzer.TestSuite.Lasagna, + analyzed: true, + comments: [ + %{comment: "elixir.solution.same_as_exemplar", type: :celebratory} + ], + halt_reason: nil, + halted: false, + source: %Source{ + code_ast: {:defmodule, _, _}, + code_string: "defmodule Lasagna" <> _, + code_path: @lasagna_path <> "/lib/lasagna.ex", + exemploid_ast: {:defmodule, _, _}, + exemploid_string: "defmodule Lasagna" <> _, + exemploid_path: @lasagna_path <> "/.meta/exemplar.ex", + exercise_type: :concept, + path: @lasagna_path, + slug: "lasagna" + } + }) do + true + end + + defp match_submission(_), do: false + + @lasagna_result "{\"comments\":[{\"comment\":\"elixir.solution.same_as_exemplar\",\"type\":\"celebratory\"}],\"summary\":\"You're doing something right. 🎉\"}" + + setup do + on_exit(fn -> + File.ls!(@lasagna_path) + |> Enum.filter(&String.ends_with?(&1, ".json")) + |> Enum.each(&File.rm(Path.join(@lasagna_path, &1))) + end) + end + + test "getting help" do + assert capture_io(fn -> CLI.main(["--help"]) end) =~ @help + end + + test "incorrect arguments" do + assert capture_io(fn -> CLI.main(["--hello"]) end) =~ @help + end + + test "analyze a file with default values" do + summary = """ + ElixirAnalyzer Report + --------------------- + + Exercise: lasagna + Status: Analysis Complete + Output written to ... test_data/lasagna/perfect_solution/analysis.json + """ + + assert capture_io(fn -> + assert CLI.main(["lasagna", @lasagna_path, @lasagna_path]) |> match_submission + end) =~ summary + + assert File.read!(Path.join(@lasagna_path, "analysis.json")) == @lasagna_result + end + + test "analyze a file with different output path" do + summary = """ + ElixirAnalyzer Report + --------------------- + + Exercise: lasagna + Status: Analysis Complete + Output written to ... test_data/lasagna/perfect_solution/output.json + """ + + assert capture_io(fn -> + assert CLI.main([ + "lasagna", + @lasagna_path, + @lasagna_path, + "--output-file", + "output.json" + ]) + |> match_submission + end) =~ summary + + assert File.read!(Path.join(@lasagna_path, "output.json")) == @lasagna_result + end + + test "analyze a file, no output" do + assert capture_io(fn -> + assert CLI.main([ + "lasagna", + @lasagna_path, + @lasagna_path, + "--no-write-results", + "--no-puts-summary" + ]) + |> match_submission + end) == "" + + assert {:error, :enoent} = File.read(Path.join(@lasagna_path, "analysis.json")) + end +end diff --git a/test/elixir_analyzer/constants_test.exs b/test/elixir_analyzer/constants_test.exs index a6fb9d0a..b8dbf48c 100644 --- a/test/elixir_analyzer/constants_test.exs +++ b/test/elixir_analyzer/constants_test.exs @@ -3,6 +3,7 @@ defmodule ElixirAnalyzer.ConstantsTest do doctest ElixirAnalyzer alias ElixirAnalyzer.Constants + alias ElixirAnalyzer.Support setup_all do Application.ensure_all_started(:inets) @@ -11,6 +12,10 @@ defmodule ElixirAnalyzer.ConstantsTest do :ok end + test "check mock constant" do + assert Support.Constants.mock_constant() == "mock.constant" + end + describe "if comment exists at exercism/website-copy" do @comments Constants.list_of_all_comments() @website_copy_url "https://github.com/exercism/website-copy/blob/master/analyzer-comments/" diff --git a/test/elixir_analyzer/exercise_test/assert_call_test.exs b/test/elixir_analyzer/exercise_test/assert_call_test.exs index 1c8aca9e..1b26f0fa 100644 --- a/test/elixir_analyzer/exercise_test/assert_call_test.exs +++ b/test/elixir_analyzer/exercise_test/assert_call_test.exs @@ -250,4 +250,125 @@ defmodule ElixirAnalyzer.ExerciseTest.AssertCallTest do end end end + + describe "test errors" do + test "undefined comment" do + assert_raise RuntimeError, "Comment must be defined for each assert_call test", fn -> + defmodule AssertFail do + use ElixirAnalyzer.ExerciseTest + + assert_call "some assert_call" do + end + end + end + end + + test "unsupported expression" do + message = + "Unsupported expression `unsupported`.\nThe macro `assert_call` supports expressions: comment, type, calling_fn, called_fn, suppress_if.\n" + + assert_raise RuntimeError, message, fn -> + defmodule AssertFail do + use ElixirAnalyzer.ExerciseTest + + assert_call "some assert_call" do + comment "some comment" + unsupported(true) + end + end + end + end + + test "unsupported type" do + message = + "Unsupported type `unsupported`.\nThe macro `assert_call` supports the following types: essential, actionable, informative, celebratory.\n" + + assert_raise RuntimeError, message, fn -> + defmodule AssertFail do + use ElixirAnalyzer.ExerciseTest + + assert_call "some assert_call" do + comment "some comment" + type :unsupported + end + end + end + end + + test "non-atomic called function module" do + message = "calling function signature requires :module to be nil or a module atom, got: 42" + + assert_raise ArgumentError, message, fn -> + defmodule AssertFail do + use ElixirAnalyzer.ExerciseTest + + assert_call "some assert_call" do + comment "some comment" + called_fn module: 42, name: :floor + end + end + end + end + + test "non-atomic calling function module" do + message = "calling function signature requires :module to be nil or a module atom, got: 42" + + assert_raise ArgumentError, message, fn -> + defmodule AssertFail do + use ElixirAnalyzer.ExerciseTest + + assert_call "some assert_call" do + comment "some comment" + called_fn name: :floor + calling_fn module: 42, name: :fourty_two + end + end + end + end + + test "non-atomic called function name" do + message = "calling function signature requires :name to be an atom, got: 42" + + assert_raise ArgumentError, message, fn -> + defmodule AssertFail do + use ElixirAnalyzer.ExerciseTest + + assert_call "some assert_call" do + comment "some comment" + called_fn module: Enum, name: 42 + end + end + end + end + + test "non-atomic calling function name" do + message = "calling function signature requires :name to be an atom, got: 42" + + assert_raise ArgumentError, message, fn -> + defmodule AssertFail do + use ElixirAnalyzer.ExerciseTest + + assert_call "some assert_call" do + comment "some comment" + calling_fn module: Enum, name: 42 + end + end + end + end + + test "nil calling function name" do + message = "calling function signature requires :module to be an atom" + + assert_raise ArgumentError, message, fn -> + defmodule AssertFail do + use ElixirAnalyzer.ExerciseTest + + assert_call "some assert_call" do + comment "some comment" + calling_fn name: nil + end + end + end + end + end end diff --git a/test/elixir_analyzer/exercise_test/check_source_test.exs b/test/elixir_analyzer/exercise_test/check_source_test.exs index 1d34b711..a896dfaf 100644 --- a/test/elixir_analyzer/exercise_test/check_source_test.exs +++ b/test/elixir_analyzer/exercise_test/check_source_test.exs @@ -121,4 +121,89 @@ defmodule ElixirAnalyzer.ExerciseTest.CheckSourceTest do """ ] end + + test "full check_source definition in a test file for coverage" do + defmodule Coverage do + use ElixirAnalyzer.ExerciseTest + + check_source "check" do + comment "this is a comment" + + check(_) do + true + end + end + + check_source "some other check" do + comment "this is another comment" + suppress_if "check", :pass + + check(_) do + false + end + end + end + + defmodule CoverageTest do + use ElixirAnalyzer.ExerciseTestCase, + exercise_test_module: Coverage + + test_exercise_analysis "test check", + comments: ["this is a comment"] do + defmodule Module do + end + end + end + end + + describe "incorrect use raises errors" do + unsupported = + "Unsupported expression `unsupported`.\nThe macro `check_source` supports expressions: comment, type, suppress_if, check.\n" + + assert_raise RuntimeError, unsupported, fn -> + defmodule Failure do + use ElixirAnalyzer.ExerciseTest + + check_source "check" do + comment "this will fail" + unsupported(:woops) + + check(_) do + true + end + end + end + end + + assert_raise RuntimeError, "Comment must be defined for each check_source test", fn -> + defmodule Failure do + use ElixirAnalyzer.ExerciseTest + + check_source "check" do + type :informative + + check(_) do + true + end + end + end + end + + wrong_type = + "Unsupported type `unsupported`.\nThe macro `check_source` supports the following types: essential, actionable, informative, celebratory.\n" + + assert_raise RuntimeError, wrong_type, fn -> + defmodule Failure do + use ElixirAnalyzer.ExerciseTest + + check_source "check" do + type :unsupported + + check(_) do + true + end + end + end + end + end end diff --git a/test/elixir_analyzer/exercise_test/common_checks/function_capture_test.exs b/test/elixir_analyzer/exercise_test/common_checks/function_capture_test.exs index 9c2e2c8a..19c12320 100644 --- a/test/elixir_analyzer/exercise_test/common_checks/function_capture_test.exs +++ b/test/elixir_analyzer/exercise_test/common_checks/function_capture_test.exs @@ -243,6 +243,31 @@ defmodule ElixirAnalyzer.ExerciseTest.CommonChecks.FunctionCaptureTest do assert FunctionCapture.run(code) == [{:fail, comment}] end + + test "Function of arity 0, no parentheses, after a function with correct use" do + code = + quote do + defmodule Capture do + def doing_it_right() do + Enum.map(input, &exports/0) + end + + def capture(input) do + Enum.map(input, fn -> exports end) + end + end + end + + comment = %{ + @comment + | params: %{ + actual: "fn -> exports end", + expected: "&exports/0" + } + } + + assert FunctionCapture.run(code) == [{:fail, comment}] + end end describe "catches & notation" do diff --git a/test/elixir_analyzer/exercise_test/feature/duplicate_features_test.exs b/test/elixir_analyzer/exercise_test/feature/duplicate_features_test.exs index ad2e0661..6e474802 100644 --- a/test/elixir_analyzer/exercise_test/feature/duplicate_features_test.exs +++ b/test/elixir_analyzer/exercise_test/feature/duplicate_features_test.exs @@ -164,40 +164,6 @@ defmodule Feature.DuplicateFeaturesTest do end end - test "Same feature with different metadata" do - assert_raise FeatureError, - "Features \"feature 1\" and \"feature 2\" compile to the same value.", - fn -> - defmodule FeatureFail do - use ElixirAnalyzer.ExerciseTest - - feature "feature 1" do - type :essential - find :one - comment "feature 1 failed" - - form do - def add_player(_ignore, _ignore, _ignore \\ @_ignore) do - _ignore - end - end - end - - feature "feature 2" do - find :any - type :actionable - comment "feature 2 failed" - - form do - def add_player(_ignore, _ignore, _ignore \\ @_ignore) do - _ignore - end - end - end - end - end - end - test "Features with more than one form" do assert_raise FeatureError, "Features \"feature 1\" and \"feature 2\" compile to the same value.", @@ -281,5 +247,65 @@ defmodule Feature.DuplicateFeaturesTest do end end end + + test "Features with different types are distinct" do + defmodule FeatureFind do + use ElixirAnalyzer.ExerciseTest + + feature "feature 1" do + find :any + comment "feature 1 failed" + + form do + @_ignore + end + + form do + def add_player(_ignore, _ignore, _ignore \\ @_ignore) do + _ignore + end + end + end + + feature "feature 2" do + find :all + comment "feature 2 failed" + + form do + @_ignore + end + + form do + def add_player(_ignore, _ignore, _ignore \\ @_ignore) do + _ignore + end + end + end + end + end + + test "Features with different depths are distinct" do + defmodule FeatureDepth do + use ElixirAnalyzer.ExerciseTest + + feature "feature 1" do + comment "feature 1 failed" + depth 1 + + form do + @_ignore + end + end + + feature "feature 2" do + comment "feature 2 failed" + depth 2 + + form do + @_ignore + end + end + end + end end end diff --git a/test/elixir_analyzer/exercise_test/feature_test.exs b/test/elixir_analyzer/exercise_test/feature_test.exs index 48ef3221..2eb2323c 100644 --- a/test/elixir_analyzer/exercise_test/feature_test.exs +++ b/test/elixir_analyzer/exercise_test/feature_test.exs @@ -382,4 +382,227 @@ defmodule ElixirAnalyzer.ExerciseTest.FeatureTest do ] end end + + describe "types of blocks" do + defmodule Blocks do + use ElixirAnalyzer.ExerciseTest + + feature "single line" do + comment "single line" + + form do + :a + end + end + + feature "two lines" do + comment "two lines" + + form do + :a + :b + end + end + + feature "single line block" do + comment "single line block" + + form do + !:a + end + end + end + + defmodule BlocksTest do + use ElixirAnalyzer.ExerciseTestCase, + exercise_test_module: Blocks + + test_exercise_analysis "empty", + comments: ["single line block", "two lines", "single line"] do + defmodule Module do + end + end + + test_exercise_analysis "two lines", + comments: ["single line block"], + comments_exclude: ["single line", "two lines"] do + defmodule Module do + def foo do + :a + :b + end + end + end + + test_exercise_analysis "single line block", + comments: ["two lines"], + comments_exclude: ["single line", "single line block"] do + defmodule Module do + def foo do + !:a + end + end + end + end + end + + describe "other features" do + defmodule Coverage do + use ElixirAnalyzer.ExerciseTest + + feature "skip" do + comment "skip" + status :skip + + form do + :ok + end + end + + feature "any" do + comment "any" + find :any + + form do + :ok + end + + form do + :error + end + end + + feature "all" do + comment "all" + find :all + + form do + :ok + end + + form do + :error + end + end + + feature "suppress_if" do + comment "suppress_if" + suppress_if "all", :pass + + form do + :error + end + end + + feature "depth 2" do + comment "depth 2" + depth 2 + + form do + :ok + end + end + + feature "depth 3" do + comment "depth 3" + depth 3 + + form do + :ok + end + end + + feature "meta" do + comment "meta" + meta(keep_meta(true)) + + form do + def foo, do: :ok + end + end + end + + defmodule CoverageTest do + use ElixirAnalyzer.ExerciseTestCase, + exercise_test_module: Coverage + + test_exercise_analysis "has :ok at depth 2", + comments: ["meta", "depth 3", "all", "suppress_if"], + comments_exclude: ["any", "depth 2", "skip"] do + defmodule Module do + def foo, do: :ok + end + end + + test_exercise_analysis "has :error, and :ok at depth 3", + comments: ["meta", "depth 2"], + comments_exclude: ["any", "all", "depth 3", "skip", "suppress_if"] do + defmodule Module do + defmodule SubModule do + def foo, do: :ok + end + + def bar, do: :error + end + end + + test_exercise_analysis "empty", + comments: ["meta", "depth 2", "depth 3", "any", "all", "suppress_if"], + comments_exclude: ["skip"] do + defmodule Module do + end + end + end + end + + describe "incorrect use raises errors" do + unsupported = + "Unsupported expression `unsupported`.\nThe macro `feature` supports expressions: comment, type, find, status, suppress_if, depth, meta, form.\n" + + assert_raise RuntimeError, unsupported, fn -> + defmodule Failure do + use ElixirAnalyzer.ExerciseTest + + feature "check" do + comment "this will fail" + unsupported(:woops) + + form do + _ignore + end + end + end + end + + assert_raise RuntimeError, "Comment must be defined for each feature test", fn -> + defmodule Failure do + use ElixirAnalyzer.ExerciseTest + + feature "check" do + type :informative + + form do + true + end + end + end + end + + wrong_type = + "Unsupported type `unsupported`.\nThe macro `feature` supports the following types: essential, actionable, informative, celebratory.\n" + + assert_raise RuntimeError, wrong_type, fn -> + defmodule Failure do + use ElixirAnalyzer.ExerciseTest + + feature "check" do + type :unsupported + + form do + true + end + end + end + end + end end diff --git a/test/elixir_analyzer/exercise_test_test.exs b/test/elixir_analyzer/exercise_test_test.exs index c5679df1..4b718709 100644 --- a/test/elixir_analyzer/exercise_test_test.exs +++ b/test/elixir_analyzer/exercise_test_test.exs @@ -3,8 +3,10 @@ defmodule ElixirAnalyzer.ExerciseTestTest.SameComment do assert_no_call "essential comment for helper1_essential" do type :essential + calling_fn module: SomeModule, name: :function called_fn name: :helper1_essential comment "the same comment" + suppress_if "some other check", :pass end assert_no_call "essential comment for helper2_essential" do @@ -125,4 +127,41 @@ defmodule ElixirAnalyzer.ExerciseTestTest do ] end end + + describe "test_exercise_analysis exceptions" do + assert_raise RuntimeError, + "Expected to receive at least one of the supported assertions: comments, comments_include, comments_exclude", + fn -> + defmodule Failing do + use ElixirAnalyzer.ExerciseTestCase, + exercise_test_module: ElixirAnalyzer.ExerciseTestTest.SameComment + + test_exercise_analysis "doesn't have any extra key", [] do + [] + end + end + end + + assert_raise RuntimeError, "Unsupported assertions received: whoops", fn -> + defmodule Failing do + use ElixirAnalyzer.ExerciseTestCase, + exercise_test_module: ElixirAnalyzer.ExerciseTestTest.SameComment + + test_exercise_analysis "doesn't have a comment key", whoops: nil do + [] + end + end + end + + assert_raise RuntimeError, "Unsupported assertions received: whoops, again", fn -> + defmodule Failing do + use ElixirAnalyzer.ExerciseTestCase, + exercise_test_module: ElixirAnalyzer.ExerciseTestTest.SameComment + + test_exercise_analysis "doesn't have a comment key", whoops: nil, again: nil do + [] + end + end + end + end end diff --git a/test/elixir_analyzer/log_formatter_test.exs b/test/elixir_analyzer/log_formatter_test.exs new file mode 100644 index 00000000..283ffece --- /dev/null +++ b/test/elixir_analyzer/log_formatter_test.exs @@ -0,0 +1,13 @@ +defmodule ElixirAnalyzer.LogFormatterTest do + use ExUnit.Case + + test "formatting a message" do + assert ElixirAnalyzer.LogFormatter.format(:warn, "hi", {{2021, 12, 4}, {11, 59, 12, 0}}, []) == + "# 2021-12-04T11:59:12.000Z [] [warn] hi\n" + end + + test "formatting failure" do + assert ElixirAnalyzer.LogFormatter.format(:warn, "hi", :not_a_timestamp, []) == + "could not format message: {:warn, \"hi\", :not_a_timestamp, []}\n" + end +end diff --git a/test/elixir_analyzer_test.exs b/test/elixir_analyzer_test.exs index af0489a9..9662021c 100644 --- a/test/elixir_analyzer_test.exs +++ b/test/elixir_analyzer_test.exs @@ -4,7 +4,7 @@ defmodule ElixirAnalyzerTest do import ExUnit.CaptureLog - alias ElixirAnalyzer.Submission + alias ElixirAnalyzer.{Submission, Source, Summary} describe "ElixirAnalyzer for practice exercise" do @options [puts_summary: false, write_results: false] @@ -33,6 +33,18 @@ defmodule ElixirAnalyzerTest do assert Submission.to_json(analyzed_exercise) == expected_output end + test "solution with informative comments only" do + exercise = "two-fer" + path = "./test_data/two_fer/informative_comments/" + + analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options) + + expected_output = + "{\"comments\":[{\"comment\":\"elixir.solution.use_module_doc\",\"type\":\"informative\"}],\"summary\":\"Check the comments for some things to learn. 📖\"}" + + assert Submission.to_json(analyzed_exercise) == expected_output + end + test "error solution" do exercise = "two-fer" path = "./test_data/two_fer/error_solution/" @@ -91,6 +103,14 @@ defmodule ElixirAnalyzerTest do test "perfect solution" do exercise = "lasagna" path = "./test_data/lasagna/perfect_solution/" + + Logger.configure(level: :debug) + + assert capture_log(fn -> ElixirAnalyzer.analyze_exercise(exercise, path, path, @options) end) =~ + "Initialization successful" + + Logger.configure(level: :warn) + analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options) expected_output = @@ -150,6 +170,111 @@ defmodule ElixirAnalyzerTest do end end + describe "different failures" do + test "summary for a submission that did not run" do + submission = %Submission{source: %Source{}, analysis_module: nil} + params = %{exercise: "lasagna", output_path: "a", output_file: "b"} + + assert Summary.summary(submission, params) == + """ + ElixirAnalyzer Report + --------------------- + + Exercise: lasagna + Status: Analysis Incomplete + Output written to ... a/b + """ + + assert Submission.to_json(submission) == + "{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}" + + assert Submission.to_json(%{submission | halted: true}) == + "{\"comments\":[],\"summary\":\"Analysis was halted.\"}" + end + + test "solution with wrong analysis module" do + exercise = "lasagna" + path = "./test_data/lasagna/perfect_solution/" + + option = + Keyword.put(@options, :exercise_config, %{"lasagna" => %{analyzer_module: NonSense}}) + + assert capture_log(fn -> + analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, option) + + assert %Submission{ + halted: true, + halt_reason: "Analysis skipped, unexpected error Elixir.ArgumentError" + } = analyzed_exercise + + assert Summary.summary(analyzed_exercise, %{ + exercise: exercise, + output_path: "a", + output_file: "b" + }) == """ + ElixirAnalyzer Report + --------------------- + + Exercise: lasagna + Status: Halted + Output written to ... a/b + """ + + assert Submission.to_json(analyzed_exercise) == + "{\"comments\":[],\"summary\":\"Analysis was halted. Analysis skipped, unexpected error Elixir.ArgumentError\"}" + end) =~ "[error] Loading exercise test suite 'Elixir.NonSense' failed" + end + + test "solution with missing config" do + exercise = "lasagna" + path = "./test_data/lasagna/missing_config" + + log = + capture_log(fn -> + analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options) + + assert %Submission{ + halted: true, + halt_reason: "Analysis skipped, not able to read solution config." + } = analyzed_exercise + end) + + assert log =~ + "[error_message: :enoent] [warning] Unable to read config file ./test_data/lasagna/missing_config/.meta/config.json" + + assert log =~ "[warning] Check not performed, halted previously" + end + + test "solution with wrong config" do + exercise = "lasagna" + path = "./test_data/lasagna/wrong_config" + + assert capture_log(fn -> + analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options) + + assert %Submission{ + halted: true, + halt_reason: "Analysis skipped, not able to decode solution config." + } = analyzed_exercise + end) =~ "[warning] Unable to decode 'config.json'" + end + + test "solution with no solution in config" do + exercise = "lasagna" + path = "./test_data/lasagna/wrong_config2" + + assert capture_log(fn -> + analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options) + + assert %Submission{ + halted: true, + halt_reason: "Analysis skipped, unexpected error Elixir.ArgumentError" + } = analyzed_exercise + end) =~ + "[error_message: \"errors were found at the given arguments:\\n\\n * 1st argument: not a nonempty list\\n\"] [warning] TestSuite halted, Elixir.ArgumentError" + end + end + describe "config" do test "every available exercise test suite assigned to an exercise slug in the config" do {:ok, modules} = :application.get_key(:elixir_analyzer, :modules) diff --git a/test/support/exercise_test_case.ex b/test/support/exercise_test_case.ex index f9b35e99..7a6468cf 100644 --- a/test/support/exercise_test_case.ex +++ b/test/support/exercise_test_case.ex @@ -64,11 +64,12 @@ defmodule ElixirAnalyzer.ExerciseTestCase do assertions_key_diff = assertions_keys -- supported_assertions_keys if assertions_keys == [] do - raise "Expected to receive at least one of the supported assertions: #{Enum.join(supported_assertions_keys)}" + supported = Enum.join(supported_assertions_keys, ", ") + raise "Expected to receive at least one of the supported assertions: #{supported}" end if assertions_key_diff != [] do - raise "Unsupported assertion received: #{Enum.join(assertions_key_diff)}" + raise "Unsupported assertions received: #{Enum.join(assertions_key_diff, ", ")}" end test_cases = List.wrap(test_cases) @@ -97,7 +98,7 @@ defmodule ElixirAnalyzer.ExerciseTestCase do analysis_module: "" } - result = @exercise_test_module.analyze(empty_submission, source) + result = @exercise_test_module.analyze(empty_submission) comments = result.comments @@ -151,10 +152,6 @@ defmodule ElixirAnalyzer.ExerciseTestCase do end end - def assert_comments(_, _, _) do - :noop - end - # Return as much of the source data as can be found @concept_exercise_path "elixir/exercises/concept" diff --git a/test_data/lasagna/wrong_config/.meta/config.json b/test_data/lasagna/wrong_config/.meta/config.json new file mode 100644 index 00000000..52691bbc --- /dev/null +++ b/test_data/lasagna/wrong_config/.meta/config.json @@ -0,0 +1,11 @@ +{ + "blurb": "Learn about the basics of Elixir by following a lasagna recipe.", + "authors": [ + "neenjaw" + ], + "contributors": [ + "angelikatyborska" + ], + "files": { + "solution": [ + "lib/lasagna.ex" diff --git a/test_data/lasagna/wrong_config2/.meta/config.json b/test_data/lasagna/wrong_config2/.meta/config.json new file mode 100644 index 00000000..8a2d6815 --- /dev/null +++ b/test_data/lasagna/wrong_config2/.meta/config.json @@ -0,0 +1,22 @@ +{ + "blurb": "Learn about the basics of Elixir by following a lasagna recipe.", + "authors": [ + "neenjaw" + ], + "contributors": [ + "angelikatyborska" + ], + "files": { + "solution": [], + "test": [ + "test/lasagna_test.exs" + ], + "exemplar": [ + ".meta/exemplar.ex" + ] + }, + "forked_from": [ + "csharp/lucians-luscious-lasagna" + ], + "language_versions": ">=1.10" +} diff --git a/test_data/two_fer/informative_comments/.meta/config.json b/test_data/two_fer/informative_comments/.meta/config.json new file mode 100644 index 00000000..ed319ba5 --- /dev/null +++ b/test_data/two_fer/informative_comments/.meta/config.json @@ -0,0 +1,10 @@ +{ + "blurb": "Create a sentence of the form \"One for X, one for me.\"", + "authors": [], + "files": { + "solution": ["lib/two_fer.ex"], + "test": ["test/two_fer_test.exs"], + "example": [".meta/example.ex"] + }, + "source_url": "https://github.com/exercism/problem-specifications/issues/757" +} diff --git a/test_data/two_fer/informative_comments/.meta/example.ex b/test_data/two_fer/informative_comments/.meta/example.ex new file mode 100644 index 00000000..0485a716 --- /dev/null +++ b/test_data/two_fer/informative_comments/.meta/example.ex @@ -0,0 +1,7 @@ +defmodule TwoFer do + @doc """ + Two-fer or 2-fer is short for two for one. One for you and one for me. + """ + @spec two_fer(String.t()) :: String.t() + def two_fer(name \\ "you") when is_binary(name), do: "One for #{name}, one for me." +end diff --git a/test_data/two_fer/informative_comments/lib/two_fer.ex b/test_data/two_fer/informative_comments/lib/two_fer.ex new file mode 100644 index 00000000..4f293fdd --- /dev/null +++ b/test_data/two_fer/informative_comments/lib/two_fer.ex @@ -0,0 +1,14 @@ +defmodule TwoFer do + # missing moduledoc + # @moduledoc false + + @doc """ + Two-fer or 2-fer is short for two for one. One for you and one for me. + + Using a tab like this or like this \t in a @doc is allowed. + """ + @spec two_fer(String.t()) :: String.t() + def two_fer(name \\ "you") when is_binary(name) do + "One for #{name}, one for me." + end +end