add gitignore file; add function to create layer

seung-lab · Dec 19, 2018 · db1a4d2 · db1a4d2
1 parent 1058406
commit db1a4d2
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 29 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+*.ipynb
+
diff --git a/src/Infos.jl b/src/Infos.jl
@@ -69,13 +69,13 @@ function InfoScale(d::Dict{Symbol, Any})
     InfoScale(key, chunkSizes, encoding, resolution, volumeSize, voxelOffset)
 end
 
-function Base.show(self::InfoScale)
-    println("\nkey:         ", get_key(key))
-    println("chunk size:    ", get_chunk_size(self))
-    println("encoding:      ", get_encoding(self))
-    println("resolution:    ", get_resolution(self))
-    println("volume size:   ", get_volume_size(self))
-    println("voxel offset:  ", get_voxel_offset(self))
+function Base.show(io::IO, self::InfoScale)
+    show(io, get_key(key))
+#    println("chunk size:    ", get_chunk_size(self))
+#    println("encoding:      ", get_encoding(self))
+#    println("resolution:    ", get_resolution(self))
+#    println("volume size:   ", get_volume_size(self))
+#    println("voxel offset:  ", get_voxel_offset(self))
 end 
 
 
@@ -87,7 +87,7 @@ function Base.Dict(self::InfoScale)
         if v == get_encoding(self)
             d[:encoding] = string(k)
         end 
-    end 
+    end
     d[:size] = [get_volume_size(self)...]
     d[:voxelOffset] = [Tuple(get_voxel_offset(self))...]
     return d
@@ -278,19 +278,23 @@ end
 @inline function get_mesh(self::Info) self.mesh end 
 @inline function set_mesh!(self::Info, mesh::String) self.mesh=mesh end 
 
-@inline function get_chunk_size(self::Info, mip::Integer=0) 
-    InfoScales.get_chunk_size( self.scales[mip+1] ) 
+@inline function get_key(self::Info, mip::Integer=1)
+    InfoScales.get_key( get_scales(self)[mip] )
+end 
+
+@inline function get_chunk_size(self::Info, mip::Integer=1) 
+    InfoScales.get_chunk_size( self.scales[mip] ) 
 end
 @inline function set_chunk_size!(self::Info, chunkSize::NTuple{3,Int}, mip::Integer)
-    InfoScales.set_chunk_size!( self.scales[mip+1], chunkSize) 
+    InfoScales.set_chunk_size!( self.scales[mip], chunkSize) 
 end 
 @inline function set_chunk_size!(self::Info, chunkSize::NTuple{3,Int})
     for infoScale in self.scales 
         InfoScales.set_chunk_size!( infoScale )
     end
 end 
-@inline function get_encoding(self::Info, mip::Integer=0)
-    InfoScales.get_encoding( get_scales(self)[mip+1] )
+@inline function get_encoding(self::Info, mip::Integer=1)
+    InfoScales.get_encoding( get_scales(self)[mip] )
 end 
 @inline function set_encoding!(self::Info, encoding::DataType)
     for scale in get_scales(self)
@@ -312,11 +316,11 @@ end
 @inline function set_layer_type(self::Info, layerType::Symbol) self.layerType=layerType end 
 
 """
-    get_properties_in_mip_level(self::Info, mip::Integer=0)
+    get_properties_in_mip_level(self::Info, mip::Integer=1)
 """
-function get_properties_in_mip_level(self::Info, mip::Integer=0)
+function get_properties_in_mip_level(self::Info, mip::Integer=1)
     numChannels = get_num_channels(self)
-    infoScale = self.scales[mip+1]
+    infoScale = self.scales[mip]
     chunkSize, encoding, resolution, voxelOffset, volumeSize = 
                                         InfoScales.get_properties(infoScale)
     if numChannels > 1
@@ -333,8 +337,7 @@ end
 function get_properties_in_mip_level(self::Info, key::Symbol)
     for (i, infoScale) in self.scales |> enumerate 
         if key == InfoScales.get_key(infoScale)
-            mip = i - 1
-            return get_properties_in_mip_level(self, mip) 
+            return get_properties_in_mip_level(self, i) 
         end 
     end
     @error "did not find any corresponding mip level with key: " key

diff --git a/src/type.jl b/src/type.jl
@@ -1,5 +1,6 @@
 
 const DEFAULT_MODE = :multithreads 
+const DEFAULT_FILL_MISSING = true 
 
 """
     BigArray
@@ -29,7 +30,7 @@ end
 """
     BigArray(layerPath::AbstractString; fillMissing=true, mod::Symbol=DEFAULT_MODE)
 """
-function BigArray(layerPath::AbstractString; fillMissing=true, mode::Symbol=DEFAULT_MODE)
+function BigArray(layerPath::AbstractString; fillMissing=DEFAULT_FILL_MISSING, mode::Symbol=DEFAULT_MODE)
     if isdir(layerPath) || startswith(layerPath, "file://")
         d = BinDict(layerPath)
     elseif startswith(layerPath, "gs://")
@@ -42,34 +43,35 @@ function BigArray(layerPath::AbstractString; fillMissing=true, mode::Symbol=DEFA
     BigArray(d; fillMissing=fillMissing, mode=mode)
 end 
 
-@inline function BigArray(d::AbstractBigArrayBackend; fillMissing::Bool=true, mode::Symbol=DEFAULT_MODE)  
+@inline function BigArray(d::AbstractBigArrayBackend; fillMissing::Bool=DEFAULT_FILL_MISSING, 
+                          mode::Symbol=DEFAULT_MODE)  
     info = get_info(d) |> Info 
     return BigArray(d, info; fillMissing=fillMissing, mode=mode)
 end
 
 @inline function BigArray( d::AbstractBigArrayBackend, info::Vector{UInt8};
-                  fillMissing::Bool=true,
+                  fillMissing::Bool=DEFAULT_FILL_MISSING,
                   mode::Symbol=DEFAULT_MODE) 
     info = Codings.decode(info, GzipCoding) |> Info 
     BigArray(d, String(info); fillMissing=fillMissing, mode=mode)
 end 
 
 @inline function BigArray( d::AbstractBigArrayBackend, info::AbstractString;
-                fillMissing::Bool=true,
+                fillMissing::Bool=DEFAULT_FILL_MISSING,
                 mode::Symbol=DEFAULT_MODE)  
     BigArray(d, Info(info); fillMissing=fillMissing, mode=mode)
 end 
 
 @inline function BigArray( d::AbstractBigArrayBackend, infoConfig::Dict{Symbol, Any};
-                    fillMissing::Bool=fillMissing, mode::Symbol=DEFAULT_MODE) 
+                    fillMissing::Bool=DEFAULT_FILL_MISSING, mode::Symbol=DEFAULT_MODE) 
     info = Info(infoConfig)
     BigArray(d, info; mip=mip, fillMissing=fillMissing, mode=mode) 
 end
 
 """
     BigArray( d::AbstractBigArrayBackend, info::Info;
                   mip::Int = 0,
-                  fillMissing::Bool=fillMissing,
+                  fillMissing::Bool=DEFAULT_FILL_MISSING,
                   mode::Symbol=DEFAULT_MODE)
 
 Parameters:
@@ -80,8 +82,8 @@ Parameters:
     fillMissing: whether fill the missing blocks in the storage backend with zeros or not. 
     mode: the io mode with options in {multithreading, sequential, multiprocesses, sharedarray}
 """
-@inline function BigArray( d::AbstractBigArrayBackend, info::Info;
-                  fillMissing::Bool=fillMissing,
+function BigArray( d::AbstractBigArrayBackend, info::Info;
+                  fillMissing::Bool=DEFAULT_FILL_MISSING,
                   mode::Symbol=DEFAULT_MODE)
     dataType = Infos.get_data_type(info) 
     key = get_scale_name(d) |> Symbol
@@ -92,6 +94,27 @@ Parameters:
              offset=voxelOffset, fillMissing=fillMissing, mode=mode) 
 end
 
+"""
+    BigArray(info::Info; fillMissing::Bool=DEFAULT_FILL_MISSING, mode=DEFAULT_MODE)
+    
+create a new directory with random name. 
+this function was designed for test and benchmark.
+we need another function the clear the whole array 
+"""
+function BigArray(info::Info; mip::Integer=1,
+                  fillMissing::Bool=DEFAULT_FILL_MISSING, mode=DEFAULT_MODE)
+    # prepare directory
+    layerDir = tempname()
+    datasetDir = joinpath(layerDir, string(Infos.get_key(info, 1))) 
+    mkdir(layerDir)
+    mkdir(datasetDir)
+    d = BinDict(datasetDir)
+
+    # write the info as file 
+    write(joinpath(layerDir, "info"), JSON.json(Dict(info)))
+
+    return BigArray(d, info; fillMissing=fillMissing, mode=mode)
+end
 
 ######################### base functions #######################
 
@@ -113,8 +136,8 @@ function Base.size(ba::BigArray, i::Int)
 end
 
 @inline function Base.show(io::IO, ba::BigArray)
-    write("key-value store: ", ba.kvStore)
-    write("block size: ", ba.chunkSize) 
+    #show(io, ba.kvStore)
+    show(io, ba.chunkSize) 
 end
 
 function Base.display(ba::BigArray)

diff --git a/test/BinDicts.jl b/test/BinDicts.jl
@@ -1,5 +1,6 @@
 using BigArrays
 using BigArrays.BinDicts
+using BigArrays.Infos
 using Test
 using OffsetArrays 
 
@@ -18,6 +19,11 @@ infoString = """
 
 write( joinpath(tempDir, "info"), infoString )
 
+@testset "test bigarray construction" begin 
+    info = Info()
+    ba = BigArray(info)
+end 
+
 @testset "test BinDict" begin 
     h = BinDict(datasetDir)
     a = rand(UInt8, 20)

diff --git a/test/Infos.jl b/test/Infos.jl
@@ -14,7 +14,7 @@ using JSON
     @test Info(d) != nothing
 
     info = Info(d)
-    Infos.get_properties_in_mip_level(info, 0)
+    Infos.get_properties_in_mip_level(info, 1)
 
     d = Dict(info)
     @test isa(d, Dict{Symbol, Any})