diff --git a/Project.toml b/Project.toml index cad0ea3..102409c 100644 --- a/Project.toml +++ b/Project.toml @@ -22,6 +22,7 @@ julia = "1" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" [targets] -test = ["Test"] +test = ["Test", "Logging"] diff --git a/doc/user.rst b/doc/user.rst index cc88082..d08b43b 100644 --- a/doc/user.rst +++ b/doc/user.rst @@ -12,11 +12,15 @@ be able to use them from Matrix Depot. Declaring Generators -------------------- -When Matrix Depot is first loaded, a new directory ``myMatrixDepot`` -will be created. Matrix Depot automatically includes all Julia files -in this directory. Hence, all we need to do is to copy -the generator files to ``path/to/MatrixDepot/myMatrixDepot`` and use -the function ``include_generator`` to declare them. +Users may use the ``include_generator`` function to declare new +generators. Note that ``include_generator`` must be called at runtime, so +users creating packages which define custom matrix test suites should place +their calls to ``include_generator`` in the ``__init__`` function of their +module to avoid +`precompilation issues `_. +A complete working example of how to create a Julia package with custom matrices is provided as a +`template https://github.com/KlausC/MatrixDepotTemplate.jl`_. + .. function:: include_generator(Stuff_To_Be_Included, Stuff, f) @@ -32,9 +36,8 @@ the function ``include_generator`` to declare them. Examples --------- -To get a feel of how it works, let's see an example. -Suppose we have a file ``myrand.jl`` which contains two -matrix generators ``randsym`` and ``randorth``:: +To get a feel of how it works, let's see an example. Suppose we have defined two +matrix generators ``randsym`` and ``randorth`` in our own module outside of MatrixDepot:: """ random symmetric matrix @@ -65,35 +68,14 @@ matrix generators ``randsym`` and ``randorth``:: """ randorth(n) = Matrix(qr(randn(n,n)).Q) -We first need to find out where the user directory of Matrix Depot is installed. This -can be done by:: - - julia> MatrixDepot.user_dir() - "/home/.../.julia/dev/MatrixDepot/myMatrixDepot" - -That points to the default user directory which can be changed by an environment variable:: - - MATRIXDEPOT_USERDIR=/... - -The data directory is queried by:: - - julia> MatrixDepot.data_dir() - "/home/.../.julia/scratchspaces/b51810bb-c9f3-55da-ae3c-350fc1fbce05/data" - -By default, the data directory is managed by ``Scratch.jl``, but can be changed by another environment variable:: +We can then write:: - MATRIXDEPOT_DATA=/... + MatrixDepot.include_generator(MatrixDepot.FunctionName, "randsym", randsym) + MatrixDepot.include_generator(MatrixDepot.FunctionName, "randorth", randorth) -For me, the package user data are installed at -``/home/.../.julia/dev/MatrixDepot/myMatrixDepot``. We can copy ``myrand.jl`` to this directory. -Now we open the file -``myMatrixDepot/generator.jl`` and write:: +and when we are done including generators, we need to update the database:: - include_generator(FunctionName, "randsym", randsym) - include_generator(FunctionName, "randorth", randorth) - -The changes are activated by re-initializing:: - julia> MatrixDepot.init() + MatrixDepot.publish_user_generators() This is it. We can now use them from Matrix Depot:: @@ -163,13 +145,14 @@ This is it. We can now use them from Matrix Depot:: 5.55112e-17 -2.77556e-17 1.94289e-16 1.0 1.38778e-16 -6.93889e-17 -5.55112e-17 -1.66533e-16 1.38778e-16 1.0 -We can also add group information in generator.jl:: +We can also add group information with:: - include_generator(Group, :random, randsym) - include_generator(Group, :symmetric, randsym) - include_generator(Group, :random, randorth) + MatrixDepot.include_generator(MatrixDepot.Group, :random, randsym) + MatrixDepot.include_generator(MatrixDepot.Group, :symmetric, randsym) + MatrixDepot.include_generator(MatrixDepot.Group, :random, randorth) + MatrixDepot.publish_user_generators() -After re-initializing ``MatrixDepot`` we can do for example:: +For example:: julia> mdlist(:symmetric) 22-element Array{String,1}: @@ -205,15 +188,14 @@ After re-initializing ``MatrixDepot`` we can do for example:: the function ``randsym`` will be part of the groups ``:symmetric`` and ``:random`` while ``randorth`` is in group ``:random``. +If we put our code in a package called `MatrixDepotTemplate` and our calls to +`include_generator` and `publish_user_generators` inside the `__init__` function, +we could use our new generators by simply importing the package.:: -It is a good idea to back up your changes. For example, we -could save it on GitHub by creating a new repository named ``myMatrixDepot``. -(See https://help.github.com/articles/create-a-repo/ for details of creating a new repository on GitHub.) -Then we go to the directory ``.../myMatrixDepot`` and type:: - - git init - git add *.jl - git commit -m "first commit" - git remote add origin https://github.com/your-user-name/myMatrixDepot.git - git push -u origin master - + julia> import MatrixDepot + julia> import MatrixDepotTemplate + julia> listnames(:random) + list(13) + –––––––– ––––––––– –––––––– –––––––– ––––––– –––––––––– –––––– + erdrey golub randcorr randorth randsym rosser wathen + gilbert oscillate rando randsvd rohess smallworld \ No newline at end of file diff --git a/src/MatrixDepot.jl b/src/MatrixDepot.jl index 3a20ab1..00d3c60 100644 --- a/src/MatrixDepot.jl +++ b/src/MatrixDepot.jl @@ -105,6 +105,9 @@ include("markdown.jl") # construct MD objects include("downloadmm.jl") # read metatdata from MM database include("downloadsp.jl") # read metatdata from SS database +#Once we no longer include code from MY_DEPOT_DIR, we can make this declaration const. +usermatrixclass = Dict() + function init(;ignoredb::Bool=false) GROUP = "group.jl" GENERATOR = "generator.jl" @@ -115,24 +118,24 @@ function init(;ignoredb::Bool=false) mkpath(data_dir()) end - if !isdir(MYDEP) - mkpath(MYDEP) - open(joinpath(MYDEP, GROUP), "w") do f - write(f, "usermatrixclass = Dict(\n);") - end - open(joinpath(MYDEP, GENERATOR), "w") do f - write(f, "# include your matrix generators below \n") + if isdir(MYDEP) && readdir(MYDEP) != [] #Backward compatibility check deprecation. Delete eventually. + if sort(readdir(MYDEP)) != sort([GROUP, GENERATOR]) || + read(joinpath(MYDEP, GROUP), String) != "usermatrixclass = Dict(\n);" || + read(joinpath(MYDEP, GENERATOR), String) != "# include your matrix generators below \n" + + @warn "MY_DEPOT_DIR custom code inclusion is deprecated: load custom generators by calling include_generator and reinitializing matrix depot at runtime. For more information, see: https://matrixdepotjl.readthedocs.io/en/latest/user.html. Duplicate warnings will be suppressed." + for file in readdir(MYDEP) + if endswith(file, ".jl") && file != GENERATOR + println("include $file for user defined matrix generators") + include(joinpath(MYDEP, file)) + end + end + if isfile(joinpath(MYDEP, GENERATOR)) + include(joinpath(MYDEP, GENERATOR)) + end end - @info("created dir $(MYDEP)") end - for file in readdir(MYDEP) - if endswith(file, ".jl") && file != GENERATOR - @info("include $file for user defined matrix generators") - include(joinpath(MYDEP, file)) - end - end - include(joinpath(MYDEP, GENERATOR)) @info("verify download of index files...") downloadindices(MATRIX_DB, ignoredb=ignoredb) @info("used remote sites are $(remote_name(preferred(TURemoteType))) and $(remote_name(preferred(MMRemoteType)))") diff --git a/src/common.jl b/src/common.jl index 48aa437..ac61508 100644 --- a/src/common.jl +++ b/src/common.jl @@ -116,6 +116,12 @@ function include_generator(::Type{Group}, groupname::Symbol, f::Function) @addgroup to add this group") end +#a more lightweight alternative to calling `init` again after adding user-defined matrices. +function publish_user_generators() + insertlocal(MATRIX_DB, GeneratedMatrixData{:U}, USERMATRIXDICT) + #note that we do not call writedb because we don't serialize user matrix generators +end + "return the name of the function `f` as a string." function fname(f::Function) for (key, value) in MATRIXDICT diff --git a/src/download.jl b/src/download.jl index 5266650..f1736f5 100644 --- a/src/download.jl +++ b/src/download.jl @@ -31,8 +31,12 @@ end function writedb(db::MatrixDatabase) cachedb = dbpath(db) + #Avoid serializing user matrices, since we don't know which ones will be loaded on deserialization. + dbx_data = filter(((key, val),) -> !(val isa GeneratedMatrixData{:U}), db.data) + dbx_aliases = filter(((_, key),) -> !(db.data[key] isa GeneratedMatrixData{:U}), db.aliases) + dbx = MatrixDatabase(dbx_data, dbx_aliases) open(cachedb, "w") do io - serialize(io, db) + serialize(io, dbx) end end diff --git a/src/markdown.jl b/src/markdown.jl index 69ba17a..6b5db29 100644 --- a/src/markdown.jl +++ b/src/markdown.jl @@ -27,7 +27,7 @@ _mdheader(md::Markdown.Header, p, o) = (md, p) _mdheader(md, p, o) = (nothing, o) function mdinfo(data::GeneratedMatrixData) - md = eval(Meta.parse("Docs.@doc $(data.func)", raise = false)) + md = Docs.doc(data.func) # As md is cached internally, need to make copies mdh, md = _mdheader(md, nothing, md) if mdh !== nothing diff --git a/src/types.jl b/src/types.jl index 4282f44..4710821 100644 --- a/src/types.jl +++ b/src/types.jl @@ -139,6 +139,7 @@ struct MatrixDatabase aliases::Dict{AbstractString,AbstractString} MatrixDatabase() = new(Dict{AbstractString,MatrixData}(), Dict{AbstractString,AbstractString}()) + MatrixDatabase(data, aliases) = new(data, aliases) end Base.show(io::IO, db::MatrixDatabase) = print(io, "MatrixDatabase(", length(db.data), ")") diff --git a/test/clean.jl b/test/clean.jl index bdadeda..74f6a36 100644 --- a/test/clean.jl +++ b/test/clean.jl @@ -1,8 +1,5 @@ # setup clean and empty directories -user_dir = abspath(dirname(@__FILE__), "..", "myMatrixDepot") -data_dir = abspath(dirname(@__FILE__),"..","data") - function save_target(path::AbstractString) base = basename(path) dir = dirname(path) diff --git a/test/include_generator.jl b/test/include_generator.jl index 73e4740..99e8b43 100644 --- a/test/include_generator.jl +++ b/test/include_generator.jl @@ -1,5 +1,6 @@ -matrixdata = -""" +import MatrixDepot: publish_user_generators, include_generator, Group, FunctionName +using Logging + "random symmetric matrix" function randsym(::Type{T}, n) where T A = zeros(T, n, n) @@ -15,13 +16,9 @@ randsym(n) = randsym(Float64, n) include_generator(FunctionName, "randsym", randsym) include_generator(Group, :random, randsym) include_generator(Group, :symmetric, randsym) -""" -open(joinpath(MatrixDepot.user_dir(), "generator.jl"), "w") do f - write(f, matrixdata) -end -# include the just written user file -MatrixDepot.init(ignoredb=true) +# update the database +MatrixDepot.publish_user_generators() n = rand(1:8) @test matrixdepot("randsym", n) !== nothing @@ -29,6 +26,49 @@ n = rand(1:8) @test "randsym" in MatrixDepot.mdlist(:random) @test "randsym" in MatrixDepot.mdlist(:symmetric) -import MatrixDepot: include_generator, Group -@test_throws ArgumentError include_generator(Group, :lkjasj, sin) +@test_logs min_level=Logging.Warn MatrixDepot.init() +n = rand(1:8) +@test matrixdepot("randsym", n) !== nothing +@test mdinfo("randsym") !== nothing +@test mdinfo("randsym") == Base.Docs.doc(randsym) +@test "randsym" in MatrixDepot.mdlist(:random) +@test "randsym" in MatrixDepot.mdlist(:symmetric) + +begin #Testing backward compatibility deprecation. Delete eventually. + mydepot_warning = "MY_DEPOT_DIR custom code inclusion is deprecated: load custom generators by calling include_generator and reinitializing matrix depot at runtime. For more information, see: https://matrixdepotjl.readthedocs.io/en/latest/user.html. Duplicate warnings will be suppressed." + + mkpath(MatrixDepot.user_dir()) + open(joinpath(MatrixDepot.user_dir(), "group.jl"), "w") do f + write(f, "usermatrixclass = Dict(\n);") + end + matrixgenerator = + """ + randorth(n) = Matrix(qr(randn(n,n)).Q) + include_generator(FunctionName, "randorth", randorth) + include_generator(Group, :random, randorth) + """ + open(joinpath(MatrixDepot.user_dir(), "generator.jl"), "w") do f + write(f, matrixgenerator) + end + + @test_logs (:warn, mydepot_warning) min_level=Logging.Warn match_mode=:any MatrixDepot.init() + n = rand(1:8) + + @test matrixdepot("randorth", n) !== nothing + @test mdinfo("randorth") != nothing + @test "randorth" in MatrixDepot.mdlist(:random) + + open(joinpath(MatrixDepot.user_dir(), "generator.jl"), "w") do f + write(f, "# include your matrix generators below \n") + end + + @test_logs min_level=Logging.Warn MatrixDepot.init() + + rm(joinpath(MatrixDepot.user_dir(), "generator.jl")) + rm(joinpath(MatrixDepot.user_dir(), "group.jl")) + + @test_logs min_level=Logging.Warn MatrixDepot.init() +end + +@test_throws ArgumentError include_generator(Group, :lkjasj, sin) diff --git a/test/runtests.jl b/test/runtests.jl index 1837a4b..ac3a299 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,7 +4,7 @@ const MD_TOGGLE = true ENV["MATRIXDEPOT_URL_REDIRECT"] = length(VERSION.prerelease) == 0 ? "1" : "1" const basedir = tempname() ENV["MATRIXDEPOT_DATA"] = abspath(basedir, "data") -ENV["MATRIXDEPOT_MYDEPOT"] = abspath(basedir, "myMatrixDepot") +ENV["MATRIXDEPOT_MYDEPOT"] = abspath(basedir, "myMatrixDepot") #Delete when MYDEPOT functionality is deleted using MatrixDepot using Test