[furaffinity] add 'search' extractor (closes #915)

mikf · Aug 18, 2020 · 063c71c · 063c71c
1 parent dbbbb21
commit 063c71c
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 1 deletion.
diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst
@@ -34,7 +34,7 @@ Fallen Angels Scans  https://www.fascans.com/            Chapters, Manga
 Fashion Nova         https://www.fashionnova.com/        Collections, Products
 Fireden              https://boards.fireden.net/         Threads
 Flickr               https://www.flickr.com/             |flickr-C|                                         Optional (`OAuth <https://github.com/mikf/gallery-dl#oauth>`__)
-Fur Affinity         https://www.furaffinity.net/        Favorites, Galleries, Posts, Scraps, User Profiles Optional (`Cookies <https://github.com/mikf/gallery-dl#cookies>`__)
+Fur Affinity         https://www.furaffinity.net/        |furaffinity-C|                                    Optional (`Cookies <https://github.com/mikf/gallery-dl#cookies>`__)
 Fuskator             https://fuskator.com/               Galleries, Search Results
 Futaba Channel       https://www.2chan.net/              Threads
 Gelbooru             https://gelbooru.com/               Pools, Posts, Tag Searches
@@ -154,6 +154,7 @@ Turboimagehost       https://www.turboimagehost.com/     individual Images
 .. |artstation-C| replace:: Albums, Artwork Listings, Challenges, individual Images, Likes, Search Results, User Profiles
 .. |deviantart-C| replace:: Collections, Deviations, Favorites, Folders, Galleries, Journals, Popular Images, Scraps, Sta.sh, User Profiles
 .. |flickr-C| replace:: Albums, Favorites, Galleries, Groups, individual Images, Search Results, User Profiles
+.. |furaffinity-C| replace:: Favorites, Galleries, Posts, Scraps, Search Results, User Profiles
 .. |hentaifoundry-C| replace:: Favorites, individual Images, Popular Images, Recent Images, Scraps, User Profiles
 .. |imgur-C| replace:: Albums, Favorites, Galleries, individual Images, Subreddits, User Profiles
 .. |instagram-C| replace:: Channels, individual Images, Saved Posts, Stories, Tag Searches, User Profiles

diff --git a/gallery_dl/extractor/furaffinity.py b/gallery_dl/extractor/furaffinity.py
@@ -30,15 +30,21 @@ def __init__(self, match):
         self.offset = 0
 
     def items(self):
+        metadata = self.metadata()
         for post_id in util.advance(self.posts(), self.offset):
             post = self._parse_post(post_id)
             if post:
+                if metadata:
+                    post.update(metadata)
                 yield Message.Directory, post
                 yield Message.Url, post["url"], post
 
     def posts(self):
         return self._pagination()
 
+    def metadata(self):
+        return None
+
     def skip(self, num):
         self.offset += num
         return num
@@ -133,6 +139,40 @@ def _pagination_favorites(self):
             yield from text.extract_iter(page, 'id="sid-', '"')
             path = text.extract(page, 'right" href="', '"')[0]
 
+    def _pagination_search(self, query):
+        url = self.root + "/search/"
+        data = {
+            "page"           : 0,
+            "next_page"      : "Next",
+            "order-by"       : "relevancy",
+            "order-direction": "desc",
+            "range"          : "all",
+            "rating-general" : "on",
+            "rating-mature"  : "on",
+            "rating-adult"   : "on",
+            "type-art"       : "on",
+            "type-music"     : "on",
+            "type-flash"     : "on",
+            "type-story"     : "on",
+            "type-photo"     : "on",
+            "type-poetry"    : "on",
+            "mode"           : "extended",
+        }
+        data.update(query)
+        if "page" in query:
+            data["page"] = text.parse_int(query["page"])
+
+        while True:
+            page = self.request(url, method="POST", data=data).text
+            post_id = None
+
+            for post_id in text.extract_iter(page, 'id="sid-', '"'):
+                yield post_id
+
+            if not post_id:
+                return
+            data["page"] += 1
+
 
 class FuraffinityGalleryExtractor(FuraffinityExtractor):
     """Extractor for a furaffinity user's gallery"""
@@ -171,6 +211,25 @@ def posts(self):
         return self._pagination_favorites()
 
 
+class FuraffinitySearchExtractor(FuraffinityExtractor):
+    """Extractor for furaffinity search results"""
+    subcategory = "search"
+    directory_fmt = ("{category}", "Search", "{search}")
+    pattern = BASE_PATTERN + r"/search/?\?([^#]+)"
+    test = ("https://www.furaffinity.net/search/?q=cute", {
+        "pattern": r"https://d.facdn.net/art/[^/]+/\d+/\d+.\w+\.\w+",
+        "range": "45-50",
+        "count": 6,
+    })
+
+    def metadata(self):
+        self.query = text.parse_query(self.user)
+        return {"search": self.query.get("q")}
+
+    def posts(self):
+        return self._pagination_search(self.query)
+
+
 class FuraffinityPostExtractor(FuraffinityExtractor):
     """Extractor for individual posts on furaffinity"""
     subcategory = "post"