Change unixfs empty folder to dynamic folder

README.md

@@ -23,14 +23,14 @@ GLOBAL OPTIONS:
    --help, -h                    show help (default: false)
 ```
 
-The input file can be a text file that contains a list of file infomation. i.e.
+The input file can be a text file that contains a list of file information SORTED by the path. i.e.
 ```json
 [
   {
     "Path": "test/test.txt",
     "Size": 4038,
-    "Start": 1000,
-    "End": 2000
+    "Start": 1000, # Inclusive
+    "End": 2000 # Exclusive
   },
   {
     "Path": "test/test2.txt",

generate-car.go

@@ -29,6 +29,7 @@ type Result struct {
 	DataCid   string
 	PieceCid  string
 	PieceSize uint64
+	CidMap    map[string]util.CidMapValue
 }
 
 type Input []util.Finfo
@@ -117,7 +118,7 @@ func main() {
 			}
 			cp := new(commp.Calc)
 			writer := bufio.NewWriterSize(io.MultiWriter(carF, cp), BufSize)
-			ipld, cid, err := util.GenerateCar(ctx, input, parent, tmpDir, writer)
+			ipld, cid, cidMap, err := util.GenerateCar(ctx, input, parent, tmpDir, writer)
 			if err != nil {
 				return err
 			}
@@ -157,6 +158,7 @@ func main() {
 				DataCid:   cid,
 				PieceCid:  commCid.String(),
 				PieceSize: pieceSize,
+				CidMap:    cidMap,
 			})
 			if err != nil {
 				return err

go.mod

@@ -26,11 +26,13 @@ require (
 )
 
 require (
+	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-bitfield v1.0.0 // indirect
 	github.com/ipfs/go-block-format v0.0.3 // indirect
 	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
 	github.com/ipfs/go-ipfs-exchange-interface v0.1.0 // indirect
@@ -64,6 +66,7 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
 	golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	lukechampine.com/blake3 v1.1.6 // indirect

spec/generate-car_spec.rb

@@ -21,6 +21,7 @@
   ]
 }
     }
+    expectedCidMap = {""=>{"Cid"=>"bafybeiceqv4l23zs2766j3i2ros3zvatxmanelmnzk753ue525d6j4azgy", "IsDir"=>true}, "test.txt"=>{"Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq", "IsDir"=>false}}
     stdout = `./generate-car -i test/test.json  -o test -p test -t tmpdir`
     result = JSON.parse(stdout)
     expectDataCid = JSON.parse(expectIpld)['Hash']
@@ -30,8 +31,8 @@
     expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
     expect(result['PieceCid']).to eq(expectCommp)
     expect(result['PieceSize']).to eq(expectPieceSize)
-    puts JSON.pretty_generate(result['Ipld'])
     expect(result['Ipld']).to eq(JSON.parse(expectIpld))
+    expect(result['CidMap']).to eq(expectedCidMap)
   end
 
   it 'should work for files with parent path' do
@@ -57,6 +58,7 @@
   ]
 }
     }
+    expectedCidMap = {""=>{"IsDir"=>true, "Cid"=>"bafybeigivse44aebx2ipou7fvvejjighko4yvtcbrhcuoc4wx6xr2gtdwe"}, "test"=>{"IsDir"=>true, "Cid"=>"bafybeiceqv4l23zs2766j3i2ros3zvatxmanelmnzk753ue525d6j4azgy"}, "test/test.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}}
     stdout = `./generate-car -i test/test.json -o test -p . -t tmpdir`
     result = JSON.parse(stdout)
     expectDataCid = JSON.parse(expectIpld)['Hash']
@@ -66,8 +68,8 @@
     expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
     expect(result['PieceCid']).to eq(expectCommp)
     expect(result['PieceSize']).to eq(expectPieceSize)
-    puts JSON.pretty_generate(result['Ipld'])
     expect(result['Ipld']).to eq(JSON.parse(expectIpld))
+    expect(result['CidMap']).to eq(expectedCidMap)
   end
 
   it 'should work with partial file' do
@@ -86,6 +88,7 @@
   ]
 }
     }
+    expectedCidMap = {""=>{"Cid"=>"bafybeiejb5tmssizbrxv2p5q5tx34g4d424zylpp6fucp5m7bwiyhqxnxa", "IsDir"=>true}, "test.txt"=>{"Cid"=>"bafkreihgspm7pi3bgf44lag72wmqkms27t2et7kbmkvt537p5h4drdgzse", "IsDir"=>false}}
     stdout = `./generate-car -i test/test-partial.json  -o test -p test -t tmpdir`
     result = JSON.parse(stdout)
     expectDataCid = JSON.parse(expectIpld)['Hash']
@@ -95,8 +98,8 @@
     expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
     expect(result['PieceCid']).to eq(expectCommp)
     expect(result['PieceSize']).to eq(expectPieceSize)
-    puts JSON.pretty_generate(result['Ipld'])
     expect(result['Ipld']).to eq(JSON.parse(expectIpld))
+    expect(result['CidMap']).to eq(expectedCidMap)
   end
 
   it 'should work with multiple files' do
@@ -121,6 +124,7 @@
   ]
 }
     }
+    expectedCidMap = {""=>{"IsDir"=>true, "Cid"=>"bafybeige75p3h2b72aufmsjgtwqppiuf67nvd25vzugpc3xs2ab5gmsfwq"}, "test.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "test2.txt"=>{"IsDir"=>false, "Cid"=>"bafkreihrzwmwtzh7ax25ue4txzeuhrr77a3okuqagez243z4dcd2wlz4my"}}
     stdout = `./generate-car -i test/test-multiple.json  -o test -p test -t tmpdir`
     result = JSON.parse(stdout)
     expectDataCid = JSON.parse(expectIpld)['Hash']
@@ -130,8 +134,8 @@
     expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
     expect(result['PieceCid']).to eq(expectCommp)
     expect(result['PieceSize']).to eq(expectPieceSize)
-    puts JSON.pretty_generate(result['Ipld'])
     expect(result['Ipld']).to eq(JSON.parse(expectIpld))
+    expect(result['CidMap']).to eq(expectedCidMap)
   end
 
   it 'should work with file with wrong but larger size' do
@@ -159,7 +163,6 @@
     expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
     expect(result['PieceCid']).to eq(expectCommp)
     expect(result['PieceSize']).to eq(expectPieceSize)
-    puts JSON.pretty_generate(result['Ipld'])
     expect(result['Ipld']).to eq(JSON.parse(expectIpld))
   end
 
@@ -189,7 +192,6 @@
     expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
     expect(result['PieceCid']).to eq(expectCommp)
     expect(result['PieceSize']).to eq(expectPieceSize)
-    puts JSON.pretty_generate(result['Ipld'])
     expect(result['Ipld']).to eq(JSON.parse(expectIpld))
   end
   it 'should work with symbolic link' do
@@ -217,7 +219,6 @@
     expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
     expect(result['PieceCid']).to eq(expectCommp)
     expect(result['PieceSize']).to eq(expectPieceSize)
-    puts JSON.pretty_generate(result['Ipld'])
     expect(result['Ipld']).to eq(JSON.parse(expectIpld))
   end
 
@@ -233,6 +234,23 @@
     expect($?.exitstatus).to eq(1)
   end
 
+  it 'should work with large number of sub files' do
+    base = 'subfiles_test'
+    FileUtils.mkdir_p(File.join(base, 'subfolders'))
+    json = (10000..20000).map do |i|
+        path = File.join(base, "subfolders/#{i}.txt")
+        File.write(path, "Hello World #{i}")
+        {
+            "Path" => path,
+            "Size" => File.size(path)
+        }
+    end
+    File.write('subfiles_test/test.json', JSON.generate(json))
+    stdout = `./generate-car -i subfiles_test/test.json  -o test -p subfiles_test`
+    result = JSON.parse(stdout)
+    expect(JSON.generate(result)).to eq(JSON.generate(JSON.parse(File.read('test/test-dynamic-folder.json'))))
+  end
+
   it 'should handle complicated file structure' do
     base = 'generated_test'
     FileUtils.mkdir_p(base)
@@ -356,6 +374,7 @@
   ]
 }
         }
+    expectedCidMap = {""=>{"IsDir"=>true, "Cid"=>"bafybeib3roh4zlejbijx3ub2pkvko6szvo3yncqpxh53jlcwabsscvntgy"}, "1"=>{"IsDir"=>true, "Cid"=>"bafybeihul4g4is36a2lvqwd4osksjzcyqr6q3kj6kmlycwgsvrqwvkpuzq"}, "1/1"=>{"IsDir"=>true, "Cid"=>"bafybeidyv33hivd5ll27m6y7caaskhurr6twoobkl57xrugwpwmr47rcea"}, "1/1/1.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "1/1/2.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "1/2.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "1/3.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "2"=>{"IsDir"=>true, "Cid"=>"bafybeigjw26ddnnmqunl4ezbypr7gr55wo7uznlcfh6yp6dt3jmcvnagbi"}, "2/1"=>{"IsDir"=>true, "Cid"=>"bafybeibue5osgywpuwy5waov7cjrdlxrxps3m2znjznm7l73x362attysq"}, "2/1/1.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "2/1/2.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "2/1/3"=>{"IsDir"=>true, "Cid"=>"bafybeiavjj7lncrdsu2d5rndsvsqfl6lg4p7hzyisb32il4qoybdtmirwi"}, "2/1/3/1"=>{"IsDir"=>true, "Cid"=>"bafybeib6gazg7coviekkoakcefjl7wachgqztr3utmw7uohg635ixthhmm"}, "2/1/3/1/1.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "3.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}}
     stdout = `./generate-car -i generated_test/test.json  -o test -p generated_test -t tmpdir`
     result = JSON.parse(stdout)
     expectDataCid = JSON.parse(expectIpld)['Hash']
@@ -365,7 +384,7 @@
     expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
     expect(result['PieceCid']).to eq(expectCommp)
     expect(result['PieceSize']).to eq(expectPieceSize)
-    puts JSON.pretty_generate(result['Ipld'])
     expect(result['Ipld']).to eq(JSON.parse(expectIpld))
+    expect(result['CidMap']).to eq(expectedCidMap)
   end
 end