Merge pull request #1102 from bjarthur/bja/regress

don't automatically call compare_examples; add flags

.gitignore

@@ -3,4 +3,5 @@ test/gennedoutput.html
 test/cachedoutput.html
 test/gennedoutput
 test/cachedoutput
+test/diffedoutput
 doc/html

docs/src/dev/regression.md

@@ -8,14 +8,15 @@ Running `Pkg.test("Gadfly")` evaluates all of the files in
 addition, the figures that are produced are put into either the `gennedoutput/`
 or `cachedoutput/` sub-directories.  Nominally, the former represents the
 changes in a pull request while the latter are used for comparison.
-Specifically, `runtests.jl` examines the currently checkout out git commit, and
+Specifically, `runtests.jl` examines the currently checked out git commit, and
 sets the output directory to `cachedoutput/` if it is the HEAD of the master
 branch or if it is detached.  Otherwise, it assumes you are at the tip of a
-development branch and saves the figures to `gennedoutput/`.  After evaluating
-all the test scripts, `runtests.jl` checks to see if both of the output
-directories are not empty.  If so, `compare_examples.jl` is called, and any
-differences between the new and old figures will be displayed in the REPL and
-the browser.
+development branch and saves the figures to `gennedoutput/`.  After running the
+tests on both of these branches, executing `compare_examples.jl` displays
+differences between the new and old figures.  This script can dump a diff of
+the files to the REPL, open both figures for manual comparison, and/or, for SVG
+and PNG files, display a black and white figure highlighting the spatial
+location of the differences.
 
 So the automated regression analysis workflow is then as follows:
 
@@ -25,4 +26,5 @@ So the automated regression analysis workflow is then as follows:
 4. `Pkg.test("Gadfly")`,
 5. checkout master,
 6. `Pkg.test` again,
-7. check that any of the reported differences are as intended.
+7. `Pkg.add("ArgParse")` and, for B&W images, Cairo, Fontconfig, Rsvg, and Images as well,
+8. check for differences with `julia test/compare_examples.jl [--diff] [--two] [--bw] [-h] [filter]`

test/REQUIRE

@@ -1,5 +1,3 @@
 RDatasets
 Cairo
 Fontconfig
-Rsvg
-Images

test/compare_examples.jl

@@ -1,29 +1,49 @@
-using Base.Test, Rsvg, Cairo, Images
+using ArgParse
 
-if !isempty(ARGS)
-    regex_filter = Regex(ARGS[1])
+s = ArgParseSettings()
+@add_arg_table s begin
+    "--diff"
+        help = "print to STDOUT the output of `diff`"
+        action = :store_true
+    "--two"
+        help = "open and display both files"
+        action = :store_true
+    "--bw"
+        help = "generate, save and display a B&W difference image for PNG and SVG files.  requires Rsvg, Cairo, and Images"
+        action = :store_true
+    "filter"
+        help = "a regular expression describing the filenames to compare"
+        default=""
 end
 
-function svg2img(filename)
-  r = Rsvg.handle_new_from_file(filename)
-  d = Rsvg.handle_get_dimensions(r)
-  cs = Cairo.CairoImageSurface(d.width,d.height,Cairo.FORMAT_ARGB32)
-  c = Cairo.CairoContext(cs)
-  Rsvg.handle_render_cairo(c,r)
-  fout = tempname()
-  Cairo.write_to_png(cs,fout)
-  png = load(fout)
-  rm(fout)
-  png
+args = parse_args(s)
+
+if args["bw"]
+    using Rsvg, Cairo, Images
+
+    function svg2img(filename)
+        r = Rsvg.handle_new_from_file(filename)
+        d = Rsvg.handle_get_dimensions(r)
+        cs = Cairo.CairoImageSurface(d.width,d.height,Cairo.FORMAT_ARGB32)
+        c = Cairo.CairoContext(cs)
+        Rsvg.handle_render_cairo(c,r)
+        fout = tempname()
+        Cairo.write_to_png(cs,fout)
+        png = load(fout)
+        rm(fout)
+        png
+    end
 end
 
 # Compare with cached output
 cachedout = joinpath((@__DIR__), "cachedoutput")
 gennedout = joinpath((@__DIR__), "gennedoutput")
+diffedout = joinpath((@__DIR__), "diffedoutput")
 ndifferentfiles = 0
 const creator_producer = r"(Creator|Producer)"
-for file in filter(x->!startswith(x,"git."), readdir(cachedout))
-    isdefined(:regex_filter) && !ismatch(regex_filter,file) && continue
+filter_mkdir_git(x) = !mapreduce(y->x==y,|,[".mkdir","git.log","git.status"])
+filter_regex(x) = ismatch(Regex(args["filter"]), x)
+for file in filter(x->filter_mkdir_git(x) && filter_regex(x), readdir(cachedout))
     print("Comparing ", file, " ... ")
     cached = open(readlines, joinpath(cachedout, file))
     genned = open(readlines, joinpath(gennedout, file))
@@ -46,25 +66,32 @@ for file in filter(x->!startswith(x,"git."), readdir(cachedout))
     else
         ndifferentfiles +=1
         println("different :(")
-        run(ignorestatus(`diff $(joinpath(cachedout, file)) $(joinpath(gennedout, file))`))
-        run(`open $(joinpath(cachedout,file))`)
-        run(`open $(joinpath(gennedout,file))`)
-        if endswith(file,".svg")
-            gimg = svg2img(joinpath(gennedout,file));
-            cimg = svg2img(joinpath(cachedout,file));
-        elseif endswith(file,".png")
-            gimg = load(joinpath(gennedout,file));
-            cimg = load(joinpath(cachedout,file));
+        if args["diff"]
+            diffcmd = `diff $(joinpath(cachedout, file)) $(joinpath(gennedout, file))`
+            run(ignorestatus(diffcmd))
         end
-        if endswith(file,".svg") || endswith(file,".png")
-            dimg = convert(Matrix{Gray}, gimg.==cimg)
-            fout = joinpath(tempdir(),file*".png")
-            Images.save(fout, dimg)
-            run(`open $fout`)
+        args["two"] && run(`open $(joinpath(cachedout,file))`)
+        args["two"] && run(`open $(joinpath(gennedout,file))`)
+        if args["bw"]
+          if endswith(file,".svg")
+              gimg = svg2img(joinpath(gennedout,file));
+              cimg = svg2img(joinpath(cachedout,file));
+          elseif endswith(file,".png")
+              gimg = load(joinpath(gennedout,file));
+              cimg = load(joinpath(cachedout,file));
+          end
+          if endswith(file,".svg") || endswith(file,".png")
+              dimg = convert(Matrix{Gray}, gimg.==cimg)
+              fout = joinpath(diffedout,file*".png")
+              Images.save(fout, dimg)
+              args["bw"] && run(`open $fout`)
+          end
         end
-        println("Press the return/enter key to continue")
+        args["diff"] || args["two"] || args["bw"] || continue
+        println("Press ENTER to continue, CTRL-C to quit")
         readline()
-        (endswith(file,".svg") || endswith(file,".png")) && rm(fout)
     end
 end
-@test ndifferentfiles==0
+
+infoorwarn = ndifferentfiles==0 ? info : warn
+infoorwarn("# different images = ",ndifferentfiles)

test/runtests.jl

@@ -108,9 +108,3 @@ close(output)
 if prev_theme !== nothing
     ENV["GADFLY_THEME"] = prev_theme
 end
-
-if !haskey(ENV, "TRAVIS") && isinteractive() &&
-            length(readdir(joinpath((@__DIR__),"cachedoutput")))>1 &&
-            length(readdir(joinpath((@__DIR__),"gennedoutput")))>1
-    run(`$(Base.julia_cmd()) compare_examples.jl`)
-end