improve util.load/save_cookiestxt() and add tests

- take a file object as argument instead of an filename
- accept whitespace before comments ("   # comment")
- map expiration "0" to None and not the number 0

gallery_dl/extractor/common.py

@@ -197,7 +197,8 @@ def _init_cookies(self):
             elif isinstance(cookies, str):
                 cookiefile = util.expand_path(cookies)
                 try:
-                    cookies = util.load_cookiestxt(cookiefile)
+                    with open(cookiefile) as fp:
+                        cookies = util.load_cookiestxt(fp)
                 except Exception as exc:
                     self.log.warning("cookies: %s", exc)
                 else:
@@ -217,7 +218,8 @@ def _store_cookies(self):
         """Store the session's cookiejar in a cookies.txt file"""
         if self._cookiefile and self.config("cookies-update", True):
             try:
-                util.save_cookiestxt(self._cookiefile, self._cookiejar)
+                with open(self._cookiefile, "w") as fp:
+                    util.save_cookiestxt(fp, self._cookiejar)
             except OSError as exc:
                 self.log.warning("cookies: %s", exc)
 

gallery_dl/util.py

@@ -136,66 +136,65 @@ def remove_directory(path):
         pass
 
 
-def load_cookiestxt(path):
+def load_cookiestxt(fp):
     """Parse a Netscape cookies.txt file and return a list of its Cookies"""
     cookies = []
 
-    with open(path) as fp:
-        for line in fp:
-
-            # strip '#HttpOnly_' and trailing '\n'
-            if line.startswith("#HttpOnly_"):
-                line = line[10:]
-            if line[-1] == "\n":
-                line = line[:-1]
-
-            # ignore empty lines and comments
-            if not line or line[0] in ("#", "$"):
-                continue
-
-            domain, domain_specified, path, secure, expires, name, value = \
-                line.split("\t")
-            if not name:
-                name = value
-                value = None
-
-            cookies.append(Cookie(
-                0, name, value,
-                None, False,
-                domain,
-                domain_specified == "TRUE",
-                domain.startswith("."),
-                path, False,
-                secure == "TRUE",
-                expires or None,
-                False, None, None, {},
-            ))
+    for line in fp:
+
+        line = line.lstrip()
+        # strip '#HttpOnly_'
+        if line.startswith("#HttpOnly_"):
+            line = line[10:]
+        # ignore empty lines and comments
+        if not line or line[0] in ("#", "$"):
+            continue
+        # strip trailing '\n'
+        if line[-1] == "\n":
+            line = line[:-1]
+
+        domain, domain_specified, path, secure, expires, name, value = \
+            line.split("\t")
+        if not name:
+            name = value
+            value = None
+
+        cookies.append(Cookie(
+            0, name, value,
+            None, False,
+            domain,
+            domain_specified == "TRUE",
+            domain.startswith("."),
+            path, False,
+            secure == "TRUE",
+            None if expires == "0" or not expires else expires,
+            False, None, None, {},
+        ))
 
     return cookies
 
 
-def save_cookiestxt(path, cookies):
-    """Store 'cookies' in Netscape cookies.txt format"""
-    with open(path, "w") as fp:
-        fp.write("# Netscape HTTP Cookie File\n\n")
+def save_cookiestxt(fp, cookies):
+    """Write 'cookies' in Netscape cookies.txt format to 'fp'"""
+    fp.write("# Netscape HTTP Cookie File\n\n")
 
-        for cookie in cookies:
-            if cookie.value is None:
-                name = ""
-                value = cookie.name
-            else:
-                name = cookie.name
-                value = cookie.value
-
-            fp.write("\t".join((
-                cookie.domain,
-                "TRUE" if cookie.domain.startswith(".") else "FALSE",
-                cookie.path,
-                "TRUE" if cookie.secure else "FALSE",
-                "0" if cookie.expires is None else str(cookie.expires),
-                name,
-                value,
-            )) + "\n")
+    for cookie in cookies:
+        if cookie.value is None:
+            name = ""
+            value = cookie.name
+        else:
+            name = cookie.name
+            value = cookie.value
+
+        fp.write("\t".join((
+            cookie.domain,
+            "TRUE" if cookie.domain.startswith(".") else "FALSE",
+            cookie.path,
+            "TRUE" if cookie.secure else "FALSE",
+            "0" if cookie.expires is None else str(cookie.expires),
+            name,
+            value,
+        )) + "\n")
 
 
 def code_to_language(code, default=None):

test/test_util.py

@@ -1,16 +1,18 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2019 Mike Fährmann
+# Copyright 2015-2020 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
 import unittest
 import sys
+import io
 import random
 import string
+import http.cookiejar
 
 from gallery_dl import util, text, exception
 
@@ -158,6 +160,100 @@ def _run_test(self, func, tests):
             self.assertEqual(func(*args), result)
 
 
+class TestCookiesTxt(unittest.TestCase):
+
+    def test_load_cookiestxt(self):
+
+        def _assert(content, expected):
+            cookies = util.load_cookiestxt(io.StringIO(content, None))
+            for c, e in zip(cookies, expected):
+                self.assertEqual(c.__dict__, e.__dict__)
+
+        _assert("", [])
+        _assert("\n\n\n", [])
+        _assert("$ Comment", [])
+        _assert("# Comment", [])
+        _assert(" # Comment \n\n $ Comment ", [])
+        _assert(
+            ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue",
+            [self._cookie("name", "value", ".example.org")],
+        )
+        _assert(
+            ".example.org\tTRUE\t/\tTRUE\t\tname\t",
+            [self._cookie("name", "", ".example.org")],
+        )
+        _assert(
+            "# Netscape HTTP Cookie File\n"
+            "\n"
+            "# default\n"
+            ".example.org	TRUE	/	FALSE	0	n1	v1\n"
+            ".example.org	TRUE	/	TRUE	2145945600	n2	v2\n"
+            ".example.org	TRUE	/path	FALSE	0		n3\n"
+            "\n"
+            "  # # extra # #  \n"
+            "www.example.org	FALSE	/	FALSE		n4	\n"
+            "www.example.org	FALSE	/path	FALSE	100	n5	v5\n",
+            [
+                self._cookie(
+                    "n1", "v1", ".example.org", True, "/", False),
+                self._cookie(
+                    "n2", "v2", ".example.org", True, "/", True, 2145945600),
+                self._cookie(
+                    "n3", None, ".example.org", True, "/path", False),
+                self._cookie(
+                    "n4", ""  , "www.example.org", False, "/", False),
+                self._cookie(
+                    "n5", "v5", "www.example.org", False, "/path", False, 100),
+            ],
+        )
+
+        with self.assertRaises(ValueError):
+            util.load_cookiestxt("example.org\tTRUE\t/\tTRUE\t0\tname")
+
+    def test_save_cookiestxt(self):
+
+        def _assert(cookies, expected):
+            fp = io.StringIO(newline=None)
+            util.save_cookiestxt(fp, cookies)
+            self.assertMultiLineEqual(fp.getvalue(), expected)
+
+        _assert([], "# Netscape HTTP Cookie File\n\n")
+        _assert(
+            [self._cookie("name", "value", ".example.org")],
+            "# Netscape HTTP Cookie File\n\n"
+            ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue\n",
+        )
+        _assert(
+            [
+                self._cookie(
+                    "n1", "v1", ".example.org", True, "/", False),
+                self._cookie(
+                    "n2", "v2", ".example.org", True, "/", True, 2145945600),
+                self._cookie(
+                    "n3", None, ".example.org", True, "/path", False),
+                self._cookie(
+                    "n4", ""  , "www.example.org", False, "/", False),
+                self._cookie(
+                    "n5", "v5", "www.example.org", False, "/path", False, 100),
+            ],
+            "# Netscape HTTP Cookie File\n"
+            "\n"
+            ".example.org	TRUE	/	FALSE	0	n1	v1\n"
+            ".example.org	TRUE	/	TRUE	2145945600	n2	v2\n"
+            ".example.org	TRUE	/path	FALSE	0		n3\n"
+            "www.example.org	FALSE	/	FALSE	0	n4	\n"
+            "www.example.org	FALSE	/path	FALSE	100	n5	v5\n",
+        )
+
+    def _cookie(self, name, value, domain, domain_specified=True,
+                path="/", secure=True, expires=None):
+        return http.cookiejar.Cookie(
+            0, name, value, None, False,
+            domain, domain_specified, domain.startswith("."),
+            path, False, secure, expires, False, None, None, {},
+        )
+
+
 class TestFormatter(unittest.TestCase):
 
     kwdict = {