Merge pull request #1120 from bjarthur/bja/regress3

more refinements to regression testing
GiovineItalia · Mar 30, 2018 · 51fbc2c · 51fbc2c
2 parents a111e38 + a74da8f
commit 51fbc2c
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 20 deletions.
diff --git a/docs/src/dev/regression.md b/docs/src/dev/regression.md
@@ -27,4 +27,7 @@ So the automated regression analysis workflow is then as follows:
 5. checkout master,
 6. `Pkg.test` again,
 7. `Pkg.add("ArgParse")` and, for B&W images, Cairo, Fontconfig, Rsvg, and Images as well,
-8. check for differences with `julia test/compare_examples.jl [--diff] [--two] [--bw] [-h] [filter]`
+8. check for differences with `julia test/compare_examples.jl [--diff] [--two]
+[--bw] [-h] [filter]`.  For example, `julia test/compare_examples.jl -bw
+.js.svg` will show black and white images hightlighting the differences between
+the svg test images.
diff --git a/test/compare_examples.jl b/test/compare_examples.jl
@@ -35,6 +35,21 @@ if args["bw"]
     end
 end
 
+# delete diffedoutput/
+repo = LibGit2.GitRepo(dirname(@__DIR__))
+options = LibGit2.StatusOptions(flags=LibGit2.Consts.STATUS_OPT_INCLUDE_IGNORED |
+                                      LibGit2.Consts.STATUS_OPT_RECURSE_IGNORED_DIRS)
+status = LibGit2.GitStatus(repo, status_opts=options)
+for i in 1:length(status)
+    entry = status[i]
+    index_to_workdir = unsafe_load(entry.index_to_workdir)
+    if index_to_workdir.status == Int(LibGit2.Consts.DELTA_IGNORED)
+        filepath = unsafe_string(index_to_workdir.new_file.path)
+        startswith(filepath,joinpath("test/diffedoutput")) || continue
+        rm(filepath)
+    end
+end
+
 # Compare with cached output
 cachedout = joinpath((@__DIR__), "cachedoutput")
 gennedout = joinpath((@__DIR__), "gennedoutput")
@@ -47,10 +62,10 @@ cached_files = filter(x->filter_mkdir_git(x) && filter_regex(x), readdir(cachedo
 genned_files = filter(x->filter_mkdir_git(x) && filter_regex(x), readdir(gennedout))
 cached_notin_genned = setdiff(cached_files, genned_files)
 isempty(cached_notin_genned) ||
-      warn("files in cachedoutput/ but not in gennedoutput/: ", cached_notin_genned...)
+      warn("files in cachedoutput/ but not in gennedoutput/: ", join(cached_notin_genned,", "))
 genned_notin_cached = setdiff(genned_files, cached_files)
 isempty(genned_notin_cached) ||
-      warn("files in gennedoutput/ but not in cachedoutput/: ", genned_notin_cached...)
+      warn("files in gennedoutput/ but not in cachedoutput/: ", join(genned_notin_cached,", "))
 for file in intersect(cached_files,genned_files)
     print("Comparing ", file, " ... ")
     cached = open(readlines, joinpath(cachedout, file))
@@ -78,24 +93,35 @@ for file in intersect(cached_files,genned_files)
             diffcmd = `diff $(joinpath(cachedout, file)) $(joinpath(gennedout, file))`
             run(ignorestatus(diffcmd))
         end
-        args["two"] && run(`open $(joinpath(cachedout,file))`)
-        args["two"] && run(`open $(joinpath(gennedout,file))`)
-        if args["bw"]
-          if endswith(file,".svg")
-              gimg = svg2img(joinpath(gennedout,file));
-              cimg = svg2img(joinpath(cachedout,file));
-          elseif endswith(file,".png")
-              gimg = load(joinpath(gennedout,file));
-              cimg = load(joinpath(cachedout,file));
-          end
-          if endswith(file,".svg") || endswith(file,".png")
-              dimg = convert(Matrix{Gray}, gimg.==cimg)
-              fout = joinpath(diffedout,file*".png")
-              Images.save(fout, dimg)
-              args["bw"] && run(`open $fout`)
-          end
+        if args["two"]
+            run(`open $(joinpath(cachedout,file))`)
+            run(`open $(joinpath(gennedout,file))`)
+        end
+        if args["bw"] && (endswith(file,".svg") || endswith(file,".png"))
+            wait_for_user = false
+            if endswith(file,".svg")
+                gimg = svg2img(joinpath(gennedout,file));
+                cimg = svg2img(joinpath(cachedout,file));
+            elseif endswith(file,".png")
+                gimg = load(joinpath(gennedout,file));
+                cimg = load(joinpath(cachedout,file));
+            end
+            if size(gimg)==size(cimg)
+                dimg = convert(Matrix{Gray}, gimg.==cimg)
+                if any(dimg.==0)
+                    fout = joinpath(diffedout,file*".png")
+                    Images.save(fout, dimg)
+                    wait_for_user = true
+                    run(`open $fout`)
+                else
+                    println("files are different but PNGs are the same")
+                end
+            else
+                wait_for_user = true
+                println("PNGs are different sizes :(")
+            end
         end
-        args["diff"] || args["two"] || args["bw"] || continue
+        args["diff"] || args["two"] || (args["bw"] && wait_for_user) || continue
         println("Press ENTER to continue, CTRL-C to quit")
         readline()
     end

diff --git a/test/runtests.jl b/test/runtests.jl
@@ -13,6 +13,20 @@ outputdir = mapreduce(x->startswith(branch,x), |, ["master","(detac"]) ?
         "cachedoutput" : "gennedoutput"
 
 if VERSION>=v"0.6"
+    # delete outputdir/
+    options = LibGit2.StatusOptions(flags=LibGit2.Consts.STATUS_OPT_INCLUDE_IGNORED |
+                                          LibGit2.Consts.STATUS_OPT_RECURSE_IGNORED_DIRS)
+    status = LibGit2.GitStatus(repo, status_opts=options)
+    for i in 1:length(status)
+        entry = status[i]
+        index_to_workdir = unsafe_load(entry.index_to_workdir)
+        if index_to_workdir.status == Int(LibGit2.Consts.DELTA_IGNORED)
+            filepath = unsafe_string(index_to_workdir.new_file.path)
+            startswith(filepath,joinpath("test",outputdir)) || continue
+            rm(joinpath(dirname(@__DIR__),filepath))
+        end
+    end
+
     function mimic_git_log_n1(io::IO, head)
         hash = LibGit2.GitHash(head)
         println(io, "commit ",string(hash))