perf(cli): add page links direct return

spider-rs · Jan 11, 2025 · 8e9812f · 8e9812f
1 parent 2eacb11
commit 8e9812f
Show file tree

Hide file tree

Showing 8 changed files with 18 additions and 23 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.25.4"
+version = "2.26.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_chrome/Cargo.toml b/spider_chrome/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.25.4"
+version = "2.26.0"
 rust-version = "1.70"
 authors = [
     "j-mendez <jeff@spider.cloud>"

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.25.4"
+version = "2.26.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_cli/src/main.rs b/spider_cli/src/main.rs
@@ -10,7 +10,6 @@ use options::{Cli, Commands};
 use serde_json::json;
 use spider::features::chrome_common::RequestInterceptConfiguration;
 use spider::hashbrown::HashMap;
-use spider::page::get_page_selectors;
 use spider::string_concat::string_concat;
 use spider::string_concat::string_concat_impl;
 use spider::tokio;
@@ -169,28 +168,24 @@ async fn main() {
                 }) => {
                     let mut stdout = tokio::io::stdout();
 
-                    let selectors: Option<spider::RelativeSelectors> = if output_links {
-                        get_page_selectors(&url, cli.subdomains, cli.tld)
-                    } else {
-                        None
-                    };
-
-                    let base = website.get_url_parsed().clone();
+                    if output_links {
+                        website.configuration.return_page_links = true;
+                    }
 
                     tokio::spawn(async move {
                         website.crawl().await;
                     });
 
-                    while let Ok(mut res) = rx2.recv().await {
+                    while let Ok(res) = rx2.recv().await {
                         let page_json = json!({
                             "url": res.get_url(),
                             "html": if output_html {
                                 res.get_html()
                             } else {
                                 Default::default()
                             },
-                            "links": match selectors {
-                                Some(ref s) => res.links(s, &base).await.iter().map(|i| i.inner().to_string()).collect::<serde_json::Value>(),
+                            "links": match res.page_links {
+                                Some(ref s) => s.iter().map(|i| i.inner().to_string()).collect::<serde_json::Value>(),
                                 _ => Default::default()
                             }
                         });

diff --git a/spider_transformations/Cargo.toml b/spider_transformations/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.25.4"
+version = "2.26.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.25.4"
+version = "2.26.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.25.4"
+version = "2.26.0"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]