Skip to content

Commit

Permalink
add bufreader to compression module
Browse files Browse the repository at this point in the history
  • Loading branch information
digizeph committed Feb 23, 2024
1 parent 1c21d13 commit 9ae889c
Show file tree
Hide file tree
Showing 6 changed files with 16 additions and 20 deletions.
4 changes: 2 additions & 2 deletions src/oneio/compressions/bzip2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ use bzip2::read::BzDecoder;
use bzip2::write::BzEncoder;
use bzip2::Compression;
use std::fs::File;
use std::io::{BufWriter, Read, Write};
use std::io::{BufReader, BufWriter, Read, Write};

pub(crate) struct OneIOBzip2;

impl OneIOCompression for OneIOBzip2 {
fn get_reader(raw_reader: Box<dyn Read + Send>) -> Result<Box<dyn Read + Send>, OneIoError> {
Ok(Box::new(BzDecoder::new(raw_reader)))
Ok(Box::new(BufReader::new(BzDecoder::new(raw_reader))))
}

fn get_writer(raw_writer: BufWriter<File>) -> Result<Box<dyn Write>, OneIoError> {
Expand Down
4 changes: 2 additions & 2 deletions src/oneio/compressions/gzip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ use libflate::finish::AutoFinishUnchecked;
use libflate::gzip::Decoder;
use libflate::gzip::Encoder;
use std::fs::File;
use std::io::{BufWriter, Read, Write};
use std::io::{BufReader, BufWriter, Read, Write};

pub(crate) struct OneIOGzip;

impl OneIOCompression for OneIOGzip {
fn get_reader(raw_reader: Box<dyn Read + Send>) -> Result<Box<dyn Read + Send>, OneIoError> {
Ok(Box::new(Decoder::new(raw_reader)?))
Ok(Box::new(BufReader::new(Decoder::new(raw_reader)?)))
}

fn get_writer(raw_writer: BufWriter<File>) -> Result<Box<dyn Write>, OneIoError> {
Expand Down
4 changes: 2 additions & 2 deletions src/oneio/compressions/lz4.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ use crate::oneio::compressions::OneIOCompression;
use crate::OneIoError;
use lz4::Decoder;
use std::fs::File;
use std::io::{BufWriter, Read, Write};
use std::io::{BufReader, BufWriter, Read, Write};

pub(crate) struct OneIOLz4;

impl OneIOCompression for OneIOLz4 {
fn get_reader(raw_reader: Box<dyn Read + Send>) -> Result<Box<dyn Read + Send>, OneIoError> {
Ok(Box::new(Decoder::new(raw_reader).unwrap()))
Ok(Box::new(BufReader::new(Decoder::new(raw_reader).unwrap())))
}

fn get_writer(_raw_writer: BufWriter<File>) -> Result<Box<dyn Write>, OneIoError> {
Expand Down
4 changes: 2 additions & 2 deletions src/oneio/compressions/xz.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
use crate::oneio::compressions::OneIOCompression;
use crate::OneIoError;
use std::fs::File;
use std::io::{BufWriter, Read, Write};
use std::io::{BufReader, BufWriter, Read, Write};
use xz2::read::XzDecoder;
use xz2::write::XzEncoder;

pub(crate) struct OneIOXz;

impl OneIOCompression for OneIOXz {
fn get_reader(raw_reader: Box<dyn Read + Send>) -> Result<Box<dyn Read + Send>, OneIoError> {
Ok(Box::new(XzDecoder::new(raw_reader)))
Ok(Box::new(BufReader::new(XzDecoder::new(raw_reader))))
}

fn get_writer(raw_writer: BufWriter<File>) -> Result<Box<dyn Write>, OneIoError> {
Expand Down
7 changes: 3 additions & 4 deletions src/oneio/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ pub use utils::*;
use crate::OneIoError;

use std::fs::File;
use std::io::{BufWriter, Read, Write};
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::Path;

fn get_writer_raw(path: &str) -> Result<BufWriter<File>, OneIoError> {
Expand All @@ -32,8 +32,7 @@ fn get_reader_raw(path: &str) -> Result<Box<dyn Read + Send>, OneIoError> {
#[cfg(feature = "remote")]
let raw_reader: Box<dyn Read + Send> = remote::get_reader_raw_remote(path)?;
#[cfg(not(feature = "remote"))]
let raw_reader: Box<dyn Read + Send> =
Box::new(std::io::BufReader::new(std::fs::File::open(path)?));
let raw_reader: Box<dyn Read + Send> = Box::new(std::fs::File::open(path)?);
Ok(raw_reader)
}

Expand Down Expand Up @@ -74,7 +73,7 @@ pub fn get_reader(path: &str) -> Result<Box<dyn Read + Send>, OneIoError> {
"xz" | "xz2" | "lzma" => compressions::xz::OneIOXz::get_reader(raw_reader),
_ => {
// unknown file type of file {}. try to read as uncompressed file
Ok(Box::new(raw_reader))
Ok(Box::new(BufReader::new(raw_reader)))
}
}
}
Expand Down
13 changes: 5 additions & 8 deletions src/oneio/remote.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use crate::oneio::compressions::OneIOCompression;
use crate::oneio::{compressions, get_writer_raw};
use crate::OneIoError;
use std::collections::HashMap;
use std::io::{BufReader, Read};
use std::io::Read;

fn get_protocol(path: &str) -> Option<String> {
let parts = path.split("://").collect::<Vec<&str>>();
Expand Down Expand Up @@ -209,25 +209,22 @@ pub(crate) fn get_reader_raw_remote(path: &str) -> Result<Box<dyn Read + Send>,
Some(protocol) => match protocol.as_str() {
"http" | "https" => {
let response = get_remote_http_raw(path, HashMap::new())?;
Box::new(BufReader::new(response))
Box::new(response)
}
"ftp" => {
let response = get_remote_ftp_raw(path)?;
Box::new(BufReader::new(response))
Box::new(response)
}
#[cfg(feature = "s3")]
"s3" => {
let (bucket, path) = crate::oneio::s3::s3_url_parse(path)?;
Box::new(BufReader::new(crate::oneio::s3::s3_reader(
bucket.as_str(),
path.as_str(),
)?))
Box::new(crate::oneio::s3::s3_reader(bucket.as_str(), path.as_str())?)
}
_ => {
return Err(OneIoError::NotSupported(path.to_string()));
}
},
None => Box::new(BufReader::new(std::fs::File::open(path)?)),
None => Box::new(std::fs::File::open(path)?),
};

Ok(raw_reader)
Expand Down

0 comments on commit 9ae889c

Please sign in to comment.