fix: counter examples in failing tests [Closes parroty#36]

triq_reporter.report is now arity 2 and 3

commit 5988da597d439befe08eb1c1474e68ab7335ae06
Author: Adam Rutkowski <[email protected]>
Date:   Tue Jul 4 11:11:18 2017 +0200

Suppress failure print outs
These should fail exactly like ExUnit
Properly start the application
Remove unnecessary IO capture
Reimplement check/2 and add check!/2
triq agent fixup
IO Server fixup
Remove IO Server fixup
Ensure triq_rnd is loaded
Sometimes a race condition occurs when relying on -on_load alone.
Add triq reporter module
Configure triq reporter module
Remove Error Agent

Fix reporting failed cases

Since 0.5.2 the data for failed cases is not shown in the test output.
This fixes it while also making the failed cases display only once in
umbrella projects. It seems `:reset_test_count` is called before
generating the output, so cleaning `ErrAgent` there leads to surpressing
the output.

The approach taken here is to make `ErrAgent` clean itself
when you read from it. That's fine because it's only ever read once (per
project in case of umbrella) to generate the output.

README.md

@@ -12,7 +12,7 @@ First add ExCheck and [triq][triq] to your project's dependencies in `mix.exs`.
 ```Elixir
   defp deps do
     [
-      {:excheck, git: "https://github.com/devstopfix/excheck.git", tag: "0.7.3", only: :test},
+      {:excheck, git: "https://github.com/devstopfix/excheck.git", tag: "0.7.4", only: :test},
       {:triq, "~> 1.3"}
     ]
   end
@@ -156,13 +156,14 @@ The following generators defined in :triq are imported through "use ExCheck" sta
 
 Issues closed:
 
+* [36](https://github.com/parroty/excheck/issues/36) Missing counter examples after 0.5.2
 * [43](https://github.com/parroty/excheck/issues/43)
 
 ### Credits
 
 - [parroty](https://github.com/parroty) for authoring this library
 - [triq developers](https://gitlab.com/triq/triq/-/project_members) for the core library
-- [luc-tielen](https://github.com/luc-tielen) for bug fixes
+- [luc-tielen](https://github.com/luc-tielen) and [aerosol](https://github.com/aerosol) for bug fixes
 
 
 [triq]: https://hex.pm/packages/triq

config/config.exs

@@ -1,3 +1,4 @@
 use Mix.Config
 
 config :excheck, :number_iterations, 100
+config :triq, :reporter_module, ExCheck.TriqReporter

lib/excheck.ex

@@ -14,25 +14,23 @@ defmodule ExCheck do
       use ExUnit.Callbacks
 
       setup(context) do
-        # Redirect all output first to IOServer process before test starts
-        ExCheck.IOServer.redirect(self())
         {:ok, context}
       end
     end
   end
 
   @doc "Starts the ExCheck application."
   def start do
+    # {:ok, :triq_rnd} = :triq_rand_compat.init('triq_rnd')
     ExUnit.configure(formatters: [ExCheck.Formatter])
-    Application.ensure_all_started(:excheck)
   end
 
   @doc "Starts the ExCheck application."
   def start(_app, _type) do
     import Supervisor.Spec, warn: false
 
     children = [
-      worker(ExCheck.IOServer, [])
+      worker(ExCheck.TriqAgent, [])
     ]
 
     Supervisor.start_link(children, strategy: :one_for_one)
@@ -47,19 +45,23 @@ defmodule ExCheck do
 
     case :triq.check(target, iterations || default_iterations) do
       true ->
-        true
-
+        :ok
       false ->
-        false
-
-      {:EXIT, %{__struct__: ExUnit.AssertionError} = error} ->
-        raise ExUnit.AssertionError, message: error.message
-
-      {:EXIT, %{__struct__: type, message: msg}} ->
-        raise ExCheck.Error, message: "error raised: (#{type}) #{msg}"
-
+        example = :triq.counterexample()
+        {:error, %ExCheck.Error{message: "check failed: Counterexample: #{inspect example}"}}
+      {:EXIT, %{__struct__: _, message: _} = e} ->
+        example = :triq.counterexample()
+        {:error, e}
       error ->
-        raise ExCheck.Error, message: "check failed: #{inspect(error)}"
+        example = :triq.counterexample()
+        {:error, %ExCheck.Error{message: "check failed: #{inspect error}. Counterexample: #{inspect example}"}}
+    end
+  end
+
+  def check!(target, iterations \\ nil) do
+    case check(target, iterations) do
+      :ok -> :ok
+      {:error, e} -> raise e
     end
   end
 end

lib/excheck/error_agent.ex

@@ -3,9 +3,11 @@ defmodule ExCheck.ErrorAgent do
   Agent which stores errors untill all tests are finished.
   """
 
+  @initial_state %{}
+
   @doc "Start the agent and link it to the calling process."
   def start_link do
-    Agent.start_link(fn -> %{} end)
+    Agent.start_link fn -> @initial_state end
   end
 
   @doc "Saves an error message."
@@ -18,22 +20,17 @@ defmodule ExCheck.ErrorAgent do
   end
 
   @doc "Return all errors stored by this agent."
-  def errors(agent) do
-    agent
-    |> Agent.get(fn state ->
-      state
-      |> Enum.map(fn {_pid, errors} ->
-        # Errors are stored in reverse
-        Enum.reverse(errors)
-      end)
+  def flush_errors(agent) do
+    agent |> Agent.get_and_update(fn(state) ->
+      {
+        state |> Enum.map(fn {_pid, errors} ->
+          Enum.reverse(errors)  # Errors are stored in reverse
+        end),
+        @initial_state
+      }
     end)
   end
 
-  @doc "Clears all errors stored by this agent."
-  def clear_errors(agent) do
-    agent |> Agent.update(fn _ -> %{} end)
-  end
-
   @doc "Stops the agent."
   def stop(agent) do
     Agent.stop(agent)

lib/excheck/formatter.ex

@@ -1,5 +1,5 @@
 defmodule ExCheck.Formatter do
-  @version140_or_later Version.compare(System.version(), "1.4.0") in [:gt, :eq]
+  @version140_or_later Version.compare(System.version, "1.4.0") in [:gt, :eq]
 
   if @version140_or_later do
     use GenServer
@@ -11,7 +11,6 @@ defmodule ExCheck.Formatter do
 
   @moduledoc """
   Helper module for properly formatting test output.
-
   This formatter implements a GenEvent based ExUnit formatter when an Elixir version prior to 1.4.0
   is used, and otherwise implements a GenServer based formatter.
   """
@@ -22,50 +21,36 @@ defmodule ExCheck.Formatter do
   end
 
   if @version140_or_later do
+
     @doc false
     def handle_cast(event = {:suite_finished, _run_us, _load_us}, config) do
       updated_tests_count = update_tests_counter(config.test_counter)
       new_cfg = %{config | test_counter: updated_tests_count}
-      print_property_test_errors()
       CF.handle_cast(event, new_cfg)
     end
-
     def handle_cast(event, config) do
       CF.handle_cast(event, config)
     end
+
   else
+
     @doc false
     def handle_event(event = {:suite_finished, _run_us, _load_us}, config) do
       updated_tests_count = update_tests_counter(config.tests_counter)
       new_cfg = %{config | tests_counter: updated_tests_count}
-      print_property_test_errors()
       CF.handle_event(event, new_cfg)
     end
-
     def handle_event(event, config) do
       CF.handle_event(event, config)
     end
-  end
 
-  defp print_property_test_errors do
-    ExCheck.IOServer.errors()
-    |> List.flatten()
-    |> Enum.map(fn {msg, value_list} ->
-      :io.format(msg, value_list)
-    end)
   end
 
   defp update_tests_counter(tests_counter) when is_integer(tests_counter) do
-    total_tests = tests_counter + ExCheck.IOServer.total_tests()
-    ExCheck.IOServer.reset_test_count()
-    total_tests
+    tests_counter + ExCheck.TriqAgent.get_tests_count
   end
 
-  defp update_tests_counter(%{test: _} = tests_counter) when is_map(tests_counter) do
-    total_tests = %{tests_counter | test: tests_counter.test + ExCheck.IOServer.total_tests()}
-    ExCheck.IOServer.reset_test_count()
-    total_tests
+  defp update_tests_counter(tests_counter) when is_map(tests_counter) do
+    %{tests_counter | test: tests_counter.test + ExCheck.TriqAgent.get_tests_count}
   end
-
-  defp update_tests_counter(%{}), do: %{}
 end