Specs: Formatter for JUnit XML format

Reporting in this format allows interoperability with CI tools such as Circle

spec/std/spec/junit_formatter_spec.cr

@@ -0,0 +1,127 @@
+require "spec"
+
+describe "JUnit Formatter" do
+  it "reports succesful results" do
+    output = build_report do |f|
+      f.report Spec::Result.new(:success, "should do something", "spec/some_spec.cr", 33, nil)
+      f.report Spec::Result.new(:success, "should do something else", "spec/some_spec.cr", 50, nil)
+    end
+
+    expected = <<-XML
+                 <testsuite tests="2" errors="0" failed="0">
+                 <testcase file=\"spec/some_spec.cr\" classname=\"spec.some_spec\" name="should do something">
+                 </testcase>
+                 <testcase file=\"spec/some_spec.cr\" classname=\"spec.some_spec\" name="should do something else">
+                 </testcase>
+                 </testsuite>
+                 XML
+
+    output.should eq(expected)
+  end
+
+  it "reports failures" do
+    output = build_report do |f|
+      f.report Spec::Result.new(:fail, "should do something", "spec/some_spec.cr", 33, nil)
+    end
+
+    expected = <<-XML
+                 <testsuite tests="1" errors="0" failed="1">
+                 <testcase file=\"spec/some_spec.cr\" classname=\"spec.some_spec\" name="should do something">
+                 <failure />
+                 </testcase>
+                 </testsuite>
+                 XML
+
+    output.should eq(expected)
+  end
+
+  it "reports errors" do
+    output = build_report do |f|
+      f.report Spec::Result.new(:error, "should do something", "spec/some_spec.cr", 33, nil)
+    end
+
+    expected = <<-XML
+                 <testsuite tests="1" errors="1" failed="0">
+                 <testcase file=\"spec/some_spec.cr\" classname=\"spec.some_spec\" name="should do something">
+                 <error />
+                 </testcase>
+                 </testsuite>
+                 XML
+
+    output.should eq(expected)
+  end
+
+  it "reports mixed results" do
+    output = build_report do |f|
+      f.report Spec::Result.new(:success, "should do something1", "spec/some_spec.cr", 33, nil)
+      f.report Spec::Result.new(:fail, "should do something2", "spec/some_spec.cr", 50, nil)
+      f.report Spec::Result.new(:error, "should do something3", "spec/some_spec.cr", 65, nil)
+      f.report Spec::Result.new(:error, "should do something4", "spec/some_spec.cr", 80, nil)
+    end
+
+    expected = <<-XML
+                 <testsuite tests="4" errors="2" failed="1">
+                 <testcase file=\"spec/some_spec.cr\" classname=\"spec.some_spec\" name="should do something1">
+                 </testcase>
+                 <testcase file=\"spec/some_spec.cr\" classname=\"spec.some_spec\" name="should do something2">
+                 <failure />
+                 </testcase>
+                 <testcase file=\"spec/some_spec.cr\" classname=\"spec.some_spec\" name="should do something3">
+                 <error />
+                 </testcase>
+                 <testcase file=\"spec/some_spec.cr\" classname=\"spec.some_spec\" name="should do something4">
+                 <error />
+                 </testcase>
+                 </testsuite>
+                 XML
+
+    output.should eq(expected)
+  end
+
+  it "escapes spec names" do
+    output = build_report do |f|
+      f.report Spec::Result.new(:success, "complicated \" <n>'&ame", __FILE__, __LINE__, nil)
+    end
+
+    name = XML.parse(output).xpath_string("string(//testsuite/testcase[1]/@name)")
+    name.should eq("complicated \" <n>'&ame")
+  end
+
+  it "report failure stacktrace if present" do
+    cause = Exception.new("Something happened")
+
+    output = build_report do |f|
+      f.report Spec::Result.new(:fail, "foo", __FILE__, __LINE__, cause)
+    end
+
+    xml = XML.parse(output)
+    name = xml.xpath_string("string(//testsuite/testcase[1]/failure/@message)")
+    name.should eq("Something happened")
+
+    backtrace = xml.xpath_string("string(//testsuite/testcase[1]/failure/text())")
+    backtrace.should eq(cause.backtrace.join("\n"))
+  end
+
+  it "report error stacktrace if present" do
+    cause = Exception.new("Something happened")
+
+    output = build_report do |f|
+      f.report Spec::Result.new(:error, "foo", __FILE__, __LINE__, cause)
+    end
+
+    xml = XML.parse(output)
+    name = xml.xpath_string("string(//testsuite/testcase[1]/error/@message)")
+    name.should eq("Something happened")
+
+    backtrace = xml.xpath_string("string(//testsuite/testcase[1]/error/text())")
+    backtrace.should eq(cause.backtrace.join("\n"))
+  end
+end
+
+def build_report
+  output = String::Builder.new
+  formatter = Spec::JUnitFormatter.new(output)
+  yield formatter
+  formatter.finish
+  output.to_s
+end

src/spec/formatter.cr

@@ -88,4 +88,12 @@ module Spec
   def self.formatters
     @@formatters
   end
+
+  def self.override_default_formatter(formatter)
+    @@formatters[0] = formatter
+  end
+
+  def self.add_formatter(formatter)
+    @@formatters << formatter
+  end
 end

src/spec/junit_formatter.cr

@@ -0,0 +1,92 @@
+require "xml"
+
+module Spec
+  # :nodoc:
+  class JUnitFormatter < Formatter
+    @output : IO
+    @results = [] of Spec::Result
+    @summary = {} of Symbol => Int32
+
+    def initialize(@output)
+    end
+
+    def push(context)
+    end
+
+    def pop
+    end
+
+    def before_example(description)
+    end
+
+    def report(result)
+      current = @summary[result.kind]? || 0
+      @summary[result.kind] = current + 1
+      @results << result
+    end
+
+    def finish
+      io = @output
+      io << "<testsuite tests=\"#{@results.size}\" \
+                        errors=\"#{@summary[:error]? || 0}\" \
+                        failed=\"#{@summary[:fail]? || 0}\">\n"
+
+      @results.each { |r| write_report(r, io) }
+
+      io << "</testsuite>"
+      io.close
+    end
+
+    def self.file(output_dir)
+      Dir.mkdir_p(output_dir)
+      output_file_path = File.join(output_dir, "output.xml")
+      file = File.new(output_file_path, "w")
+      JUnitFormatter.new(file)
+    end
+
+    # -------- private utility methods
+    private def write_report(result, io)
+      io << "<testcase file=\"#{result.file}\" classname=\"#{classname(result)}\" name=\"#{XML.escape(result.description)}\">\n"
+
+      if (has_inner_content(result.kind))
+        tag = inner_content_tag(result.kind)
+        ex = result.exception
+        if ex
+          write_inner_content(tag, ex, io)
+        else
+          io << "<#{tag} />\n"
+        end
+      end
+
+      io << "</testcase>\n"
+    end
+
+    private def has_inner_content(kind)
+      kind == :fail || kind == :error
+    end
+
+    private def inner_content_tag(kind)
+      case kind
+      when :error
+        :error
+      when :fail
+        :failure
+      end
+    end
+
+    private def write_inner_content(tag, exception, io)
+      m = exception.message
+      if m
+        io << "<#{tag} message=\"#{XML.escape(m)}\">"
+      else
+        io << "<#{tag}>"
+      end
+      io << XML.escape(exception.backtrace.join("\n"))
+      io << "</#{tag}>\n"
+    end
+
+    private def classname(result)
+      result.file.sub(%r{\.[^/.]+\Z}, "").gsub("/", ".").gsub(/\A\.+|\.+\Z/, "")
+    end
+  end
+end

src/spec/spec.cr

@@ -221,12 +221,16 @@ OptionParser.parse! do |opts|
       exit
     end
   end
+  opts.on("--junit_output OUTPUT_DIR", "generate JUnit XML output") do |output_dir|
+    junit_formatter = Spec::JUnitFormatter.file(output_dir)
+    Spec.add_formatter(junit_formatter)
+  end
   opts.on("--help", "show this help") do |pattern|
     puts opts
     exit
   end
   opts.on("-v", "--verbose", "verbose output") do
-    Spec.formatters.replace([Spec::VerboseFormatter.new])
+    Spec.override_default_formatter(Spec::VerboseFormatter.new)
   end
   opts.on("--no-color", "Disable colored output") do
     Spec.use_colors = false