diff --git a/Cargo.lock b/Cargo.lock index 6ba9ca6..a0b14bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -239,6 +239,17 @@ dependencies = [ "syn", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -287,6 +298,24 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + [[package]] name = "bytes" version = "1.1.0" @@ -302,6 +331,12 @@ dependencies = [ "bytes", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.73" @@ -317,6 +352,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "bitflags", + "textwrap", + "unicode-width", +] + [[package]] name = "clap" version = "4.0.30" @@ -403,6 +449,85 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +dependencies = [ + "atty", + "cast", + "clap 2.34.0", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.3" @@ -413,6 +538,28 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -442,6 +589,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + [[package]] name = "encoding_rs" version = "0.8.30" @@ -635,6 +788,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.11.2" @@ -737,6 +896,15 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -758,12 +926,22 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kiryuu" version = "0.1.1" dependencies = [ "actix-web", - "clap", + "clap 4.0.30", + "criterion", "hex", "rand", "redis", @@ -843,6 +1021,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -891,6 +1078,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.13.1" @@ -916,6 +1112,12 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "os_str_bytes" version = "6.4.1" @@ -969,6 +1171,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plotters" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" + +[[package]] +name = "plotters-svg" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -1047,6 +1277,28 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redis" version = "0.21.5" @@ -1089,6 +1341,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" version = "0.6.25" @@ -1124,6 +1382,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -1145,6 +1412,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.136" @@ -1273,6 +1550,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.30" @@ -1311,6 +1597,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.5.1" @@ -1420,6 +1716,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "unicode-xid" version = "0.2.2" @@ -1444,6 +1746,17 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" @@ -1456,6 +1769,70 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 7ef3f7a..652b100 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,3 +18,14 @@ clap = { version = "4.0.30", features = ["derive"] } [[example]] name = "redis_zscore" path = "examples/redis_zscore.rs" + +[[example]] +name = "redis_zset" +path = "examples/redis_zset.rs" + +[dev-dependencies] +criterion = "0.3" + +[[bench]] +name = "my_benchmark" +harness = false diff --git a/Makefile b/Makefile index 4d87f81..7170e2b 100644 --- a/Makefile +++ b/Makefile @@ -5,4 +5,8 @@ static: cargo build --target=x86_64-unknown-linux-musl --release test-announce: - curl -v "localhost:6969/announce?info_hash=AAAAAAAAAAAAAAAAAAAB&port=3333&left=0" \ No newline at end of file + curl -v "localhost:6969/announce?info_hash=AAAAAAAAAAAAAAAAAAAB&port=3333&left=0" + +# kiryuu & redis should be running +gauge: + docker run -e KIRYUU_HOST=http://172.17.0.1:6969 -e REDIS_HOST=redis://172.17.0.1:6379 ghcr.io/ckcr4lyf/kiryuu-gauge:master \ No newline at end of file diff --git a/benches/bruvva.rs b/benches/bruvva.rs new file mode 100644 index 0000000..1f1c0a7 --- /dev/null +++ b/benches/bruvva.rs @@ -0,0 +1,3 @@ +pub fn xd() { + let x = 1; +} \ No newline at end of file diff --git a/benches/my_benchmark.rs b/benches/my_benchmark.rs new file mode 100644 index 0000000..495ee51 --- /dev/null +++ b/benches/my_benchmark.rs @@ -0,0 +1,19 @@ +mod bruvva; + +#[inline] +fn fibonacci(n: u64) -> u64 { + match n { + 0 => 1, + 1 => 1, + n => fibonacci(n-1) + fibonacci(n-2), + } +} + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; + +pub fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("fib 20", |b| b.iter(|| fibonacci(black_box(20)))); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); \ No newline at end of file diff --git a/examples/redis_zscore.rs b/examples/redis_zscore.rs index 0afa47c..ef359e0 100644 --- a/examples/redis_zscore.rs +++ b/examples/redis_zscore.rs @@ -30,5 +30,8 @@ fn main(){ _ => true, }; - println!("exist_highlevel is {:?}, exist_lowlevel is {:?}", exist_highlevel, exist_lowlevel) + println!("exist_highlevel is {:?}, exist_lowlevel is {:?}", exist_highlevel, exist_lowlevel); + + // let exist_bool: bool = r_client.zscore("thezset", "KEY2").unwrap(); + // println!("bool coerced is {:?}", exist_bool); } \ No newline at end of file diff --git a/examples/redis_zset.rs b/examples/redis_zset.rs new file mode 100644 index 0000000..c0895db --- /dev/null +++ b/examples/redis_zset.rs @@ -0,0 +1,79 @@ +use redis::{self, Commands}; + +#[derive(Debug)] +enum Exists { + Yes, + No, +} + +impl redis::FromRedisValue for Exists { + fn from_redis_value(v: &redis::Value) -> redis::RedisResult { + match *v { + redis::Value::Nil => Ok(Exists::No), + _ => Ok(Exists::Yes), + } + } +} + +fn main(){ + let mut r_client = redis::Client::open("redis://127.0.0.1:6379").unwrap(); + + // Work with low val + let zrange_result = match r_client.zrangebyscore::<_, _, _, redis::Value>("MYKEY", "0", "101").unwrap() { + redis::Value::Bulk(v1) => v1, + _ => vec![] + }; + + let mut seeders = [0_u8; 50 * 6]; // We will store max 50 guys + let mut pos = 0; + + for element in &zrange_result { + if let redis::Value::Data(xd) = element { + for i in 0..6 { + seeders[pos + i] = xd[i]; + } + pos += 6; + } + + if pos >= 300 { + break; + } + } + + println!("Manual seeders are {:?}", seeders); + + + let dummy: Vec = vec![0u8; 1]; + let mut seeders_v2: [&Vec; 50] = [&dummy; 50]; + + pos = 0; + + for element in &zrange_result { + if let redis::Value::Data(xd) = element { + seeders_v2[pos] = xd; + pos += 1; + } + } + + println!("Manual seeders 2 are {:?}", seeders_v2); + + + + // let mut body: [u8; 10] = [0; 10]; + // if let redis::Value::Data(xd) = zrange_result.get(0).unwrap() { + // body[0] = xd[0]; + // } + + // // I Know I have a Vec now + // println!("zrange_result is {:?}", zrange_result); + // println!("body is {:?}", body); + + let og: Vec> = r_client.zrangebyscore("MYKEY", "0", "101").unwrap(); + println!("og is {:?}", og) +} + +/* Manual testing: +FLUSHDB +ZADD MYKEY 10 ABCDEF +ZADD MYKEY 10 XXXXXX +*/ diff --git a/src/constants/mod.rs b/src/constants/mod.rs index e9a9fbb..beff710 100644 --- a/src/constants/mod.rs +++ b/src/constants/mod.rs @@ -1,4 +1,5 @@ pub const ANNOUNCE_COUNT_KEY: &str = "kiryuu_http_announce_count"; pub const NOCHANGE_ANNOUNCE_COUNT_KEY: &str = "kiryuu_http_nochange_announce_count"; // If no change to seeder_count / leecher_count +pub const CACHE_HIT_ANNOUNCE_COUNT_KEY: &str = "kiryuu_http_cache_hit_announce_count"; pub const REQ_DURATION_KEY: &str = "kiryuu_http_req_seconds_sum"; pub const TORRENTS_KEY: &str = "TORRENTS"; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index a8451a3..77e618d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,6 @@ mod constants; mod req_log; use actix_web::{get, App, HttpServer, web, HttpRequest, HttpResponse, http::header, http::StatusCode}; -use rand::{thread_rng, prelude::SliceRandom, Rng}; use std::time::{SystemTime, UNIX_EPOCH}; use clap::Parser; @@ -29,6 +28,21 @@ struct Args { // So dont waste bandwidth on redis query etc. const THIRTY_ONE_MINUTES: i64 = 60 * 31 * 1000; +#[derive(Debug)] +enum Exists { + Yes, + No, +} + +impl redis::FromRedisValue for Exists { + fn from_redis_value(v: &redis::Value) -> redis::RedisResult { + match *v { + redis::Value::Nil => Ok(Exists::No), + _ => Ok(Exists::Yes), + } + } +} + #[get("/announce")] async fn announce(req: HttpRequest, data: web::Data) -> HttpResponse { @@ -64,26 +78,15 @@ async fn announce(req: HttpRequest, data: web::Data) -> HttpResponse { // Get seeders & leechers let mut rc = data.redis_connection.clone(); - let seeders_key = parsed.info_hash.clone() + "_seeders"; let leechers_key = parsed.info_hash.clone() + "_leechers"; + let cache_key = parsed.info_hash.clone() + "_cache"; - let (seeders, mut leechers) : (Vec>, Vec>) = redis::pipe() - .cmd("ZRANGEBYSCORE").arg(&seeders_key).arg(max_limit).arg(time_now_ms) - .cmd("ZRANGEBYSCORE").arg(&leechers_key).arg(max_limit).arg(time_now_ms) - .query_async(&mut rc).await.unwrap(); - - let is_seeder = seeders.contains(&parsed.ip_port); - - let is_leecher = match is_seeder { - true => false, // If it's a seeder, leecher must be false - false => leechers.contains(&parsed.ip_port), // otherwise we will search the leecher vector as well - }; - - // Don't shuffle the seeders, for leechers shuffle so that the older ones also get a shot - // e.g. if there are 1000 leechers, the one whom announced 29 minutes ago also has a fair - // change of being announced to a peer, to help swarm - leechers.shuffle(&mut thread_rng()); + let (is_seeder_v2, is_leecher_v2, cached_reply) : (Exists, Exists, Vec) = redis::pipe() + .cmd("ZSCORE").arg(&seeders_key).arg(&parsed.ip_port) + .cmd("ZSCORE").arg(&leechers_key).arg(&parsed.ip_port) + .cmd("GET").arg(&cache_key) + .query_async(&mut rc).await.unwrap(); let mut post_announce_pipeline = redis::pipe(); post_announce_pipeline.cmd("ZADD").arg(constants::TORRENTS_KEY).arg(time_now_ms).arg(&parsed.info_hash).ignore(); // To "update" the torrent @@ -97,28 +100,26 @@ async fn announce(req: HttpRequest, data: web::Data) -> HttpResponse { None => "unknown", }; - // let bg_rc = data.redis_connection.clone(); - if event == "stopped" { - if is_seeder { + if let Exists::Yes = is_seeder_v2 { seed_count_mod -= 1; post_announce_pipeline.cmd("ZREM").arg(&seeders_key).arg(&parsed.ip_port).ignore(); // We dont care about the return value - } else if is_leecher { + } else if let Exists::Yes = is_leecher_v2 { leech_count_mod -= 1; post_announce_pipeline.cmd("ZREM").arg(&leechers_key).arg(&parsed.ip_port).ignore(); // We dont care about the return value } } else if parsed.is_seeding { // New seeder - if is_seeder == false { + if let Exists::No = is_seeder_v2 { post_announce_pipeline.cmd("ZADD").arg(&seeders_key).arg(time_now_ms).arg(&parsed.ip_port).ignore(); - seed_count_mod += 1 + seed_count_mod += 1; } // They just completed if event == "completed" { // If they were previously leecher, remove from that pool - if is_leecher { + if let Exists::Yes = is_leecher_v2 { post_announce_pipeline.cmd("ZREM").arg(&leechers_key).arg(&parsed.ip_port).ignore(); leech_count_mod -= 1 } @@ -126,38 +127,61 @@ async fn announce(req: HttpRequest, data: web::Data) -> HttpResponse { // Increment the downloaded count for the infohash stats post_announce_pipeline.cmd("HINCRBY").arg(&parsed.info_hash).arg("downloaded").arg(1u32).ignore(); } - } else if is_leecher == false { + } else if let Exists::No = is_leecher_v2 { post_announce_pipeline.cmd("ZADD").arg(&leechers_key).arg(time_now_ms).arg(&parsed.ip_port).ignore(); - leech_count_mod += 1 + leech_count_mod += 1; }; - // Update seeder & leecher count, if required - if seed_count_mod != 0 { - post_announce_pipeline.cmd("HINCRBY").arg(&parsed.info_hash).arg("seeders").arg(seed_count_mod).ignore(); - } + // Cache miss = query redis + // no change = update cache + // change = clear cache + + let final_res = match cached_reply.len() { + 0 => { + // Cache miss. Lookup from redis + let (seeders, leechers) : (Vec>, Vec>) = redis::pipe() + .cmd("ZRANGEBYSCORE").arg(&seeders_key).arg(max_limit).arg(time_now_ms) + .cmd("ZRANGEBYSCORE").arg(&leechers_key).arg(max_limit).arg(time_now_ms) + .query_async(&mut rc).await.unwrap(); + + // endex = end index XD. seems in rust cannot select first 50 elements, or limit to less if vector doesnt have 50 + // e.g. &seeders[0..50] is panicking when seeders len is < 50. Oh well. + let seeder_endex = std::cmp::min(seeders.len(), 50); + let leecher_endex = std::cmp::min(leechers.len(), 50); + + query::announce_reply(seeders.len() as i64 + seed_count_mod, leechers.len() as i64 + leech_count_mod, &seeders[0..seeder_endex], &leechers[0..leecher_endex]) + }, + _ => { + post_announce_pipeline.cmd("INCR").arg(constants::CACHE_HIT_ANNOUNCE_COUNT_KEY).ignore(); + cached_reply + } + }; - if leech_count_mod != 0 { - post_announce_pipeline.cmd("HINCRBY").arg(&parsed.info_hash).arg("leechers").arg(leech_count_mod).ignore(); - } + // Is there a change in seeders / leechers + if seed_count_mod != 0 || leech_count_mod != 0 { + // TBD: Maybe we can issue the HINCRBY anyway, it is: + // Pipelined + // In background (not .awaited for announce reply) + // O(1) in redis + // Can clean up this branching crap + if seed_count_mod != 0 { + post_announce_pipeline.cmd("HINCRBY").arg(&parsed.info_hash).arg("seeders").arg(seed_count_mod).ignore(); + } - // endex = end index XD. seems in rust cannot select first 50 elements, or limit to less if vector doesnt have 50 - // e.g. &seeders[0..50] is panicking when seeders len is < 50. Oh well. - let seeder_endex = if seeders.len() >= 50 { - 50 - } else { - seeders.len() - }; + if leech_count_mod != 0 { + post_announce_pipeline.cmd("HINCRBY").arg(&parsed.info_hash).arg("leechers").arg(leech_count_mod).ignore(); + } - let leecher_endex = if leechers.len() >= 50 { - 50 + // TODO: Patch cached reply with the count mods? + // Also invalidate existing cache + post_announce_pipeline.cmd("DEL").arg(&cache_key).ignore(); } else { - leechers.len() - }; - - let scount: i64 = seeders.len().try_into().expect("fucky wucky"); - let lcount: i64 = leechers.len().try_into().expect("fucky wucky"); + post_announce_pipeline.cmd("INCR").arg(constants::NOCHANGE_ANNOUNCE_COUNT_KEY).ignore(); + // TBD: If we had a cache hit, any point to set it again? + // For now we are ok, since background pipeline, O(1) in redis. + post_announce_pipeline.cmd("SET").arg(&cache_key).arg(&final_res).arg("EX").arg(60 * 30).ignore(); + } - let bruvva_res = query::announce_reply(scount + seed_count_mod, lcount + leech_count_mod, &seeders[0..seeder_endex], &leechers[0..leecher_endex]); let time_end = SystemTime::now().duration_since(UNIX_EPOCH).expect("fucked up"); let time_end_ms: i64 = i64::try_from(time_end.as_millis()).expect("fucc"); @@ -167,20 +191,8 @@ async fn announce(req: HttpRequest, data: web::Data) -> HttpResponse { post_announce_pipeline.cmd("INCR").arg(constants::ANNOUNCE_COUNT_KEY).ignore(); post_announce_pipeline.cmd("INCRBY").arg(constants::REQ_DURATION_KEY).arg(req_duration).ignore(); - // If neither seeder count changed, neither leech count - // In future, we will use this to determine if we need to update the cache or not - if seed_count_mod == 0 && leech_count_mod == 0 { - post_announce_pipeline.cmd("INCR").arg(constants::NOCHANGE_ANNOUNCE_COUNT_KEY).ignore(); - } actix_web::rt::spawn(async move { - // 0.1% chance to trigger a clean, - let chance = rand::thread_rng().gen_range(0..1000); - if chance == 0 { - post_announce_pipeline.cmd("ZREMRANGEBYSCORE").arg(&seeders_key).arg(0).arg(max_limit).ignore(); - post_announce_pipeline.cmd("ZREMRANGEBYSCORE").arg(&leechers_key).arg(0).arg(max_limit).ignore(); - } - // log the summary post_announce_pipeline.cmd("PUBLISH").arg("reqlog").arg(req_log::generate_csv(&user_ip_owned, &parsed.info_hash)).ignore(); @@ -194,7 +206,7 @@ async fn announce(req: HttpRequest, data: web::Data) -> HttpResponse { }; }); - return HttpResponse::build(StatusCode::OK).append_header(header::ContentType::plaintext()).body(bruvva_res); + return HttpResponse::build(StatusCode::OK).append_header(header::ContentType::plaintext()).body(final_res); } #[get("/healthz")]