Customize JSON output & general script improvements (#20)

- Customize JSON output: it's now possible to choose which keys should be saved in the output JSON file, both from the console and from the extension (#19 (comment))
- Changed how the extension handles older Firefox versions that don't support the "chrome" namespace
- Now a "source.zip" file is built for the extension release, that contains everything necessary for building the extension
- Improved the usage of variables in the script: only the global values are stored with "var" so that they can be re-written in Gecko and WebKit, but there is no need to use them in the functions
- Improved the script documentation, by adding descriptions directly tied to functions and variables

.github/workflows/BrowserExtension.yml

@@ -31,6 +31,6 @@ jobs:
       - name: Get last commit SHA
         run: echo "SHORT_COMMIT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - name: Create Release
-        run: gh release create ${{ env.SHORT_COMMIT_SHA }} -t ${{ env.SHORT_COMMIT_SHA }} -n "Note that this release is automatically generated. No testing has been done." --generate-notes "extension/extension.zip"
+        run: gh release create ${{ env.SHORT_COMMIT_SHA }} -t ${{ env.SHORT_COMMIT_SHA }} -n "Note that this release is automatically generated. No testing has been done." --generate-notes "extension/extension.zip" "extension/source.zip"
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -3,7 +3,10 @@ TempLinks
 node_modules
 extension/output
 extension/output.zip
+extension/source
+extension/source.zip
 .noupload
 node
 README_REVIEW.md
-extension/package-lock.json
+extension/package-lock.json
+.DS_Store

README.md

@@ -41,7 +41,9 @@ If you need to download a really long list of TikTok, you may want to start
 downloading them while the page continues scrolling. To do that, you can
 download an intermediate file, that will contain all of the links up to the
 point the page has scrolled. To do that, write in the console `requestTxtNow()`,
-press enter and a `TikTokLinks.txt` file will be downloaded.
+press enter and a `TikTokLinks.txt` file will be downloaded. If you're using the
+extension, you can find a "Get partial file" button. Click it and the file will
+be automatically downloaded.
 
 By default, the links in the first intermediate files will be deleted from the
 final (or the second/third etc. file if you want to download more intermediate
@@ -72,6 +74,11 @@ useful settings:
   name.
 - `allow_images`: save also TikTok photos (if disabled, only videos will be
   fetched).
+- `export_format`: you can choose if you want to save the result as a TXT file
+  (`txt`) or as a JSON file (`json`)
+- `exclude_from_json`: populate this array with the keys you don't want to save
+  in your JSON file. By default, the JSON file has the `url`, `caption` and
+  `views` keys.
 
 ### Advanced script options
 

extension/build.js

@@ -1,14 +1,23 @@
 const fs = require("fs");
 const zip = require("jszip");
+const sep = require("path").sep;
+// BUILDING OUTPUT
 (fs.existsSync("output")) && fs.rmSync("output", { recursive: true });
 fs.mkdirSync("output");
-require("child_process").execSync("cd code/vite && npm i && npx vite build --base=ui");
-require("fs-extra").moveSync(`./code/vite/dist`, `./output/ui`);
-fs.mkdirSync("output/icons");
-for (let item of ["manifest.json", "icons/16.png", "icons/48.png", "icons/128.png"]) fs.copyFileSync(`./code/${item}`, `./output/${item}`);
-let firstScript = fs.readFileSync(`../script.js`, "utf-8");
+require("child_process").execSync(`cd code${sep}vite && npm i && npx vite build --base=ui`);
+require("fs-extra").moveSync(`.${sep}code${sep}vite${sep}dist`, `.${sep}output${sep}ui`);
+fs.mkdirSync(`output${sep}icons`);
+for (let item of ["manifest.json", `icons${sep}16.png`, `icons${sep}48.png`, `icons${sep}128.png`]) fs.copyFileSync(`code${sep}${item}`, `output${sep}${item}`);
+let firstScript = fs.readFileSync(`..${sep}script.js`, "utf-8");
 firstScript = firstScript.substring(0, firstScript.indexOf("nodeElaborateCustomArgs();"));
-fs.writeFileSync(`./output/extensionHandler.js`, `${firstScript}${fs.readFileSync("./code/extensionHandler.js", "utf-8")}`);
+fs.writeFileSync(`output${sep}extensionHandler.js`, `${firstScript}${fs.readFileSync(`code${sep}extensionHandler.js`, "utf-8")}`);
 const zipFile = new zip();
-for (let file of fs.readdirSync("./output", { recursive: true })) if (fs.statSync(`./output/${file}`).isFile()) zipFile.file(file, fs.readFileSync(`./output/${file}`), { createFolders: true });
+for (let file of fs.readdirSync("output", { recursive: true })) if (fs.statSync(`output${sep}${file}`).isFile()) zipFile.file(file, fs.readFileSync(`output${sep}${file}`), { createFolders: true });
 zipFile.generateAsync({ type: "nodebuffer" }).then((buffer) => fs.writeFileSync("output.zip", buffer));
+// BUILDING SOURCE CODE ZIP
+const sourceCode = new zip();
+sourceCode.file(`README.md`, `# Build instrutions\n\nYou can find instructions to build this extension in the "extension" folder.\n\nRequirements: Node JS 20; NPM. Tested only on macOS and Linux.`);
+sourceCode.file("script.js", fs.readFileSync(`..${sep}script.js`));
+for (const item of ["build.js", "package.json", "package-lock.json", "README.md"]) sourceCode.file(`extension${sep}${item}`, fs.readFileSync(item));
+for (const item of fs.readdirSync("code", { recursive: true }).filter(item => item.indexOf("node_modules") === -1)) if (fs.statSync(`code${sep}${item}`).isFile()) sourceCode.file(`extension${sep}code${sep}${item}`, fs.readFileSync(`code${sep}${item}`));
+sourceCode.generateAsync({ type: "nodebuffer" }).then((buffer) => fs.writeFileSync("source.zip", buffer));

extension/code/extensionHandler.js

@@ -2,9 +2,9 @@
  * Send to the UI the current "conversion" status
  */
 function updateClient() {
-    (chrome ?? browser).runtime.sendMessage((chrome ?? browser).runtime.id, { operation: scriptOptions.node.resolve && !scriptOptions.node.isResolveTime });
+    (typeof chrome === "undefined" ? browser : chrome).runtime.sendMessage((typeof chrome === "undefined" ? browser : chrome).runtime.id, { operation: scriptOptions.node.resolve && !scriptOptions.node.isResolveTime });
 }
-(chrome ?? browser).runtime.onMessage.addListener((message) => {
+(typeof chrome === "undefined" ? browser : chrome).runtime.onMessage.addListener((message) => {
     switch (message.action) {
         case "start": { // Start running the script
             scriptOptions = { ...scriptOptions, ...message.content }; // Add the values from the object provided

extension/code/vite/src/App.svelte

@@ -8,6 +8,7 @@
   import Settings from "./lib/Settings/SettingsContainer";
   import { slide } from "svelte/transition";
   import Icon from "./assets/icon.svg";
+  import JsonOutput from "./lib/Settings/JsonOutput.svelte";
   /**
    * The settings section shown
    */
@@ -22,18 +23,20 @@
    */
   let isFirstEvent = true;
   let isAuthorizedForTikTok = true;
+  const newBrowser: typeof chrome =
+    typeof chrome === "undefined" ? browser : chrome; // For older Firefox versions
   Settings.subscribe((settingUpdate) => {
     if (isFirstEvent) {
       isFirstEvent = false;
       return;
     }
-    (chrome ?? browser).storage.sync.set({ settings: settingUpdate });
+    newBrowser.storage.sync.set({ settings: settingUpdate });
   });
   /**
    * Checks if the extension is authorized to access TikTok website
    */
   async function checkPermission() {
-    isAuthorizedForTikTok = await (chrome ?? browser).permissions.contains({
+    isAuthorizedForTikTok = await newBrowser.permissions.contains({
       origins: ["*://*.tiktok.com/*"],
     });
   }
@@ -53,13 +56,13 @@
       return update;
     }
     checkPermission();
-    (chrome ?? browser).storage.sync
+    newBrowser.storage.sync
       .get("settings")
       .then(
         (obj) => ($Settings = UpdateJsonProperties(obj.settings, $Settings)),
       );
 
-    (chrome ?? browser).runtime.onMessage.addListener(
+    newBrowser.runtime.onMessage.addListener(
       (msg) => (isConverting = msg.operation), // Currently, messages from the content_script are sent only when the value of the operation changes
     );
     sendMessage({ action: "requestOperation", content: "" }); // Asks if there's a script operation ongoing.
@@ -85,8 +88,8 @@
    * @param msg the message to send
    */
   async function sendMessage(msg: MessageProps) {
-    const ids = await (chrome ?? browser).tabs.query({ active: true });
-    (chrome ?? browser).tabs.sendMessage(ids[0].id as number, msg);
+    const ids = await newBrowser.tabs.query({ active: true });
+    newBrowser.tabs.sendMessage(ids[0].id as number, msg);
   }
 </script>
 
@@ -106,7 +109,7 @@
     <br />
     <button
       on:click={async () => {
-        await (chrome ?? browser).permissions.request({
+        await newBrowser.permissions.request({
           origins: ["*://*.tiktok.com/*"],
         });
         checkPermission();
@@ -121,13 +124,16 @@
       providedOptions={[
         { id: "output", text: "Output file settings", selected: true },
         { id: "scrollTime", text: "Scrolling time" },
+        { id: "json", text: "JSON Output" },
         { id: "advanced", text: "Advanced settings" },
       ]}
     ></OptionPicker><br />
     {#if section === "output"}
       <Output></Output>
     {:else if section === "scrollTime"}
       <Scrolling></Scrolling>
+    {:else if section === "json"}
+      <JsonOutput></JsonOutput>
     {:else if section === "advanced"}
       <Advanced></Advanced>
     {/if}<br />

extension/code/vite/src/lib/Settings/JsonOutput.svelte

@@ -0,0 +1,41 @@
+<script lang="ts">
+    import { slide } from "svelte/transition";
+    import Card from "../Card.svelte";
+    import Settings from "./SettingsContainer";
+    function changeJson(e: Event, str: string) {
+        (e.target as HTMLInputElement).checked
+            ? $Settings.exclude_from_json.push(str)
+            : $Settings.exclude_from_json.splice(
+                  $Settings.exclude_from_json.indexOf(str),
+                  1,
+              );
+        $Settings.exclude_from_json = $Settings.exclude_from_json;
+    }
+</script>
+
+<div in:slide={{ duration: 500 }} out:slide={{ duration: 500 }}>
+    <Card>
+        <h2>JSON output:</h2>
+        <p>
+            Choose which elements will be deleted from the JSON output. If you
+            keep only one item, the output will be a string array.
+        </p>
+        <br />
+        {#each ["url", "views", "caption"] as item}
+            <label class="flex hcenter autoGap">
+                <input
+                    type="checkbox"
+                    checked={$Settings.exclude_from_json.indexOf(item) !== -1}
+                    on:change={(e) => changeJson(e, item)}
+                />
+                Exclude {item} from the output JSON
+            </label><br />
+        {/each}
+    </Card>
+</div>
+
+<style>
+    input {
+        background-color: var(--input);
+    }
+</style>

extension/code/vite/src/lib/Settings/SettingsContainer.ts

@@ -9,6 +9,7 @@ let Settings = writable({
     adapt_text_output: true, // Replace characters that are prohibited on Windows
     allow_images: true, // Save also TikTok Image URLs
     export_format: "txt",
+    exclude_from_json: [], // If you plan to export the content in a JSON file, here you can exclude some properties from the JSON output. You can exclude "url", "views", "caption".
     advanced: {
         get_array_after_scroll: false, // Gets the item links after the webpage is fully scrolled, and not after every scroll.
         get_link_by_filter: true, // Get the website link by inspecting all the links in the container div, instead of looking for data references.