Refactor scripts to put shared code in a shared library

analyze/data_analysis.py

@@ -22,7 +22,7 @@
 from wordcloud import STOPWORDS, WordCloud  # noqa: E402
 
 # Set the current working directory
-CWD = os.path.dirname(os.path.abspath(__file__))
+PATH_WORK_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
 def tags_frequency(csv_path, column_names):
@@ -126,7 +126,7 @@ def tags_frequency(csv_path, column_names):
         fontweight="bold",
     )
     plt.savefig(
-        os.path.join(CWD, "wordCloud_plots/license1_wordCloud.png"),
+        os.path.join(PATH_WORK_DIR, "wordCloud_plots/license1_wordCloud.png"),
         dpi=300,
         bbox_inches="tight",
     )
@@ -193,7 +193,7 @@ def time_trend(csv_path):
     plt.xlabel("Day", fontsize=10)
     plt.ylabel("Amount", fontsize=10)
     plt.savefig(
-        os.path.join(CWD, "line_graphs/license5_total_trend.png"),
+        os.path.join(PATH_WORK_DIR, "line_graphs/license5_total_trend.png"),
         dpi=300,
         bbox_inches="tight",
     )
@@ -387,28 +387,28 @@ def view_compare():
     Compare maximum views of pictures under different licenses.
     """
     license1 = pd.read_csv(
-        os.path.join(CWD, "../flickr/dataset/cleaned_license1.csv")
+        os.path.join(PATH_WORK_DIR, "../flickr/dataset/cleaned_license1.csv")
     )
     license2 = pd.read_csv(
-        os.path.join(CWD, "../flickr/dataset/cleaned_license2.csv")
+        os.path.join(PATH_WORK_DIR, "../flickr/dataset/cleaned_license2.csv")
     )
     license3 = pd.read_csv(
-        os.path.join(CWD, "../flickr/dataset/cleaned_license3.csv")
+        os.path.join(PATH_WORK_DIR, "../flickr/dataset/cleaned_license3.csv")
     )
     license4 = pd.read_csv(
-        os.path.join(CWD, "../flickr/dataset/cleaned_license4.csv")
+        os.path.join(PATH_WORK_DIR, "../flickr/dataset/cleaned_license4.csv")
     )
     license5 = pd.read_csv(
-        os.path.join(CWD, "../flickr/dataset/cleaned_license5.csv")
+        os.path.join(PATH_WORK_DIR, "../flickr/dataset/cleaned_license5.csv")
     )
     license6 = pd.read_csv(
-        os.path.join(CWD, "../flickr/dataset/cleaned_license6.csv")
+        os.path.join(PATH_WORK_DIR, "../flickr/dataset/cleaned_license6.csv")
     )
     license9 = pd.read_csv(
-        os.path.join(CWD, "../flickr/dataset/cleaned_license9.csv")
+        os.path.join(PATH_WORK_DIR, "../flickr/dataset/cleaned_license9.csv")
     )
     license10 = pd.read_csv(
-        os.path.join(CWD, "../flickr/dataset/cleaned_license10.csv")
+        os.path.join(PATH_WORK_DIR, "../flickr/dataset/cleaned_license10.csv")
     )
     licenses = [
         license1,
@@ -469,7 +469,7 @@ def view_compare():
     current_values = plt.gca().get_yticks()
     plt.gca().set_yticklabels(["{:,.0f}".format(x) for x in current_values])
     plt.savefig(
-        os.path.join(CWD, "../analyze/compare_graphs/max_views.png"),
+        os.path.join(PATH_WORK_DIR, "../analyze/compare_graphs/max_views.png"),
         dpi=300,
         bbox_inches="tight",
     )
@@ -481,15 +481,19 @@ def total_usage():
     Generate a bar plot showing the total usage of different licenses.
     """
     # Reads the license total file as the input dataset
-    df = pd.read_csv(os.path.join(CWD, "../flickr/dataset/license_total.csv"))
+    df = pd.read_csv(
+        os.path.join(PATH_WORK_DIR, "../flickr/dataset/license_total.csv")
+    )
     df["License"] = [str(x) for x in list(df["License"])]
     fig = px.bar(df, x="License", y="Total amount", color="License")
-    fig.write_html(os.path.join(CWD, "../analyze/total_usage.html"))
+    fig.write_html(os.path.join(PATH_WORK_DIR, "../analyze/total_usage.html"))
     # fig.show()
 
 
 def main():
-    tags_frequency(os.path.join(CWD, "merged_all_cleaned.csv"), ["tags"])
+    tags_frequency(
+        os.path.join(PATH_WORK_DIR, "merged_all_cleaned.csv"), ["tags"]
+    )
     # df = pd.read_csv("../flickr/dataset/cleaned_license10.csv")
     # print(df.shape)
 

deviantart/deviantart_scratcher.py

@@ -3,9 +3,7 @@
 This file is dedicated to obtain a .csv record report for DeviantArt
 data.
 """
-
 # Standard library
-import datetime as dt
 import os
 import sys
 import traceback
@@ -17,20 +15,23 @@
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
-# Set up current working directory
-CWD = os.path.dirname(os.path.abspath(__file__))
-# Load environment variables
-dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
-load_dotenv(dotenv_path)
+sys.path.append("..")
+# First-party/Local
+from quantify import quantify  # noqa: E402
+
+PATH_REPO_ROOT, PATH_WORK_DIR, PATH_DOTENV, DATETIME_TODAY = quantify.setup(
+    __file__
+)
+load_dotenv(PATH_DOTENV)
 
-# Get the current date
-today = dt.datetime.today()
 # Retrieve API keys
 API_KEYS = os.getenv("GOOGLE_API_KEYS").split(",")
 API_KEYS_IND = 0
 # Set up file path for CSV report
 DATA_WRITE_FILE = (
-    f"{CWD}" f"/data_deviantart_{today.year}_{today.month}_{today.day}.csv"
+    f"{PATH_WORK_DIR}"
+    f"/data_deviantart_"
+    f"{DATETIME_TODAY.year}_{DATETIME_TODAY.month}_{DATETIME_TODAY.day}.csv"
 )
 # Retrieve Programmable Search Engine key from environment variables
 PSE_KEY = os.getenv("PSE_KEY")
@@ -45,7 +46,9 @@ def get_license_list():
     searched via Programmable Search Engine.
     """
     # Read license data from file
-    cc_license_data = pd.read_csv(f"{CWD}/legal-tool-paths.txt", header=None)
+    cc_license_data = pd.read_csv(
+        f"{PATH_WORK_DIR}/legal-tool-paths.txt", header=None
+    )
     # Define regex pattern to extract license types
     license_pattern = r"((?:[^/]+/){2}(?:[^/]+)).*"
     license_list = (

flickr/photos.py

@@ -14,11 +14,14 @@
 import flickrapi
 from dotenv import load_dotenv
 
-# Get the current working directory
-CWD = os.path.dirname(os.path.abspath(__file__))
-# Load environment variables
-dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
-load_dotenv(dotenv_path)
+sys.path.append("..")
+# First-party/Local
+from quantify import quantify  # noqa: E402
+
+PATH_REPO_ROOT, PATH_WORK_DIR, PATH_DOTENV, DATETIME_TODAY = quantify.setup(
+    __file__
+)
+load_dotenv(PATH_DOTENV)
 
 
 def main():
@@ -37,7 +40,7 @@ def main():
         photosJson = flickr.photos.search(license=i, per_page=500)
         dic[i] = [json.loads(photosJson.decode("utf-8"))]
     # Save the dictionary containing photo data to a JSON file
-    with open(os.path.join(CWD, "photos.json"), "w") as json_file:
+    with open(os.path.join(PATH_WORK_DIR, "photos.json"), "w") as json_file:
         json.dump(dic, json_file)
 
 

flickr/photos_detail.py

@@ -20,11 +20,14 @@
 import pandas as pd
 from dotenv import load_dotenv
 
-# Set up current working directory
-CWD = os.path.dirname(os.path.abspath(__file__))
-# Load environment variables
-dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
-load_dotenv(dotenv_path)
+sys.path.append("..")
+# First-party/Local
+from quantify import quantify  # noqa: E402
+
+PATH_REPO_ROOT, PATH_WORK_DIR, PATH_DOTENV, DATETIME_TODAY = quantify.setup(
+    __file__
+)
+load_dotenv(PATH_DOTENV)
 
 # Global variable: Number of retries for error handling
 RETRIES = 0
@@ -188,9 +191,9 @@ def page1_reset(final_csv, raw_data):
 
 
 def main():
-    final_csv_path = os.path.join(CWD, "final.csv")
-    record_txt_path = os.path.join(CWD, "rec.txt")
-    hs_csv_path = os.path.join(CWD, "hs.csv")
+    final_csv_path = os.path.join(PATH_WORK_DIR, "final.csv")
+    record_txt_path = os.path.join(PATH_WORK_DIR, "rec.txt")
+    hs_csv_path = os.path.join(PATH_WORK_DIR, "hs.csv")
 
     # Initialize Flickr API instance
     flickr = flickrapi.FlickrAPI(
@@ -290,7 +293,7 @@ def main():
             # If reached max limit of pages, reset j to 1 and
             # update i to the license in the dictionary
             if j == total + 1 or j > total:
-                license_i_path = os.path.join(CWD, f"license{i}.csv")
+                license_i_path = os.path.join(PATH_WORK_DIR, f"license{i}.csv")
                 clean_saveas_csv(final_csv_path, license_i_path)
                 i += 1
                 j = 1

google_custom_search/google_scratcher.py

@@ -5,7 +5,6 @@
 """
 
 # Standard library
-import datetime as dt
 import os
 import sys
 import traceback
@@ -17,26 +16,33 @@
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
-CWD = os.path.dirname(os.path.abspath(__file__))
-dotenv_path = os.path.join(os.path.dirname(CWD), ".env")
-load_dotenv(dotenv_path)
+sys.path.append("..")
+# First-party/Local
+from quantify import quantify  # noqa: E402
 
-today = dt.datetime.today()
+PATH_REPO_ROOT, PATH_WORK_DIR, PATH_DOTENV, DATETIME_TODAY = quantify.setup(
+    __file__
+)
+load_dotenv(PATH_DOTENV)
+
+# Retrieve API keys
 API_KEYS = os.getenv("GOOGLE_API_KEYS").split(",")
 API_KEYS_IND = 0
+# Set up file path for CSV report
 DATA_WRITE_FILE = (
-    f"{CWD}"
-    f"/data_google_custom_search_{today.year}_{today.month}_{today.day}.csv"
+    f"{PATH_WORK_DIR}"
+    f"/data_google_custom_search_"
+    f"{DATETIME_TODAY.year}_{DATETIME_TODAY.month}_{DATETIME_TODAY.day}.csv"
 )
 DATA_WRITE_FILE_TIME = (
-    f"{CWD}"
+    f"{PATH_WORK_DIR}"
     f"/data_google_custom_search_time_"
-    f"{today.year}_{today.month}_{today.day}.csv"
+    f"{DATETIME_TODAY.year}_{DATETIME_TODAY.month}_{DATETIME_TODAY.day}.csv"
 )
 DATA_WRITE_FILE_COUNTRY = (
-    f"{CWD}"
+    f"{PATH_WORK_DIR}"
     f"/data_google_custom_search_country_"
-    f"{today.year}_{today.month}_{today.day}.csv"
+    f"{DATETIME_TODAY.year}_{DATETIME_TODAY.month}_{DATETIME_TODAY.day}.csv"
 )
 SEARCH_HALFYEAR_SPAN = 20
 PSE_KEY = os.getenv("PSE_KEY")
@@ -49,7 +55,9 @@ def get_license_list():
         np.array: An np array containing all license types that should be
         searched via Programmable Search Engine.
     """
-    cc_license_data = pd.read_csv(f"{CWD}/legal-tool-paths.txt", header=None)
+    cc_license_data = pd.read_csv(
+        f"{PATH_WORK_DIR}/legal-tool-paths.txt", header=None
+    )
     license_pattern = r"((?:[^/]+/){2}(?:[^/]+)).*"
     license_list = (
         cc_license_data[0]
@@ -68,7 +76,10 @@ def get_lang_list():
         for the corresponding language code.
     """
     languages = pd.read_csv(
-        f"{CWD}/google_lang.txt", sep=": ", header=None, engine="python"
+        f"{PATH_WORK_DIR}/google_lang.txt",
+        sep=": ",
+        header=None,
+        engine="python",
     )
     languages[0] = languages[0].str.extract(r'"([^"]+)"')
     languages = languages.set_index(1)
@@ -101,7 +112,7 @@ def get_country_list(select_all=False):
         pd.DataFrame: A Dataframe whose index is country name and has a column
         for the corresponding country code.
     """
-    countries = pd.read_csv(CWD + "/google_countries.tsv", sep="\t")
+    countries = pd.read_csv(PATH_WORK_DIR + "/google_countries.tsv", sep="\t")
     countries["Country"] = countries["Country"].str.replace(",", " ")
     countries = countries.set_index("Country").sort_index()
     if select_all:

internetarchive/internetarchive_scratcher.py

@@ -4,8 +4,6 @@
 """
 
 # Standard library
-import datetime as dt
-import os
 import sys
 import traceback
 
@@ -18,11 +16,17 @@
 from internetarchive.search import Search
 from internetarchive.session import ArchiveSession
 
-today = dt.datetime.today()
-CWD = os.path.dirname(os.path.abspath(__file__))
+sys.path.append("..")
+# First-party/Local
+from quantify import quantify  # noqa: E402
+
+PATH_REPO_ROOT, PATH_WORK_DIR, PATH_DOTENV, DATETIME_TODAY = quantify.setup(
+    __file__
+)
 DATA_WRITE_FILE = (
-    f"{CWD}"
-    f"/data_internetarchive_{today.year}_{today.month}_{today.day}.csv"
+    f"{PATH_WORK_DIR}"
+    f"/data_internetarchive_"
+    f"{DATETIME_TODAY.year}_{DATETIME_TODAY.month}_{DATETIME_TODAY.day}.csv"
 )
 
 
@@ -32,7 +36,9 @@ def get_license_list():
         np.array: An np array containing all license types that should be
         searched via Programmable Search Engine.
     """
-    cc_license_data = pd.read_csv(f"{CWD}/legal-tool-paths.txt", header=None)
+    cc_license_data = pd.read_csv(
+        f"{PATH_WORK_DIR}/legal-tool-paths.txt", header=None
+    )
     license_list = cc_license_data[0].unique()
     return license_list
 

metmuseum/metmuseum_scratcher.py

@@ -5,8 +5,6 @@
 """
 
 # Standard library
-import datetime as dt
-import os
 import sys
 import traceback
 
@@ -15,10 +13,18 @@
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
-today = dt.datetime.today()
-CWD = os.path.dirname(os.path.abspath(__file__))
+sys.path.append("..")
+# First-party/Local
+from quantify import quantify  # noqa: E402
+
+PATH_REPO_ROOT, PATH_WORK_DIR, PATH_DOTENV, DATETIME_TODAY = quantify.setup(
+    __file__
+)
+
 DATA_WRITE_FILE = (
-    f"{CWD}" f"/data_metmuseum_{today.year}_{today.month}_{today.day}.csv"
+    f"{PATH_WORK_DIR}"
+    f"/data_metmuseum_"
+    f"{DATETIME_TODAY.year}_{DATETIME_TODAY.month}_{DATETIME_TODAY.day}.csv"
 )
 
 

quantify/quantify.py

@@ -0,0 +1,21 @@
+# Shared module for common functionalities across scripts
+
+# Standard library
+import datetime
+import os.path
+
+
+def setup(path):
+
+    # Datetime
+    datetime_today = datetime.datetime.today()
+
+    # Paths
+    path_work_dir = os.path.dirname(os.path.abspath(os.path.realpath(path)))
+    path_repo_root = os.path.dirname(
+        os.path.abspath(os.path.realpath(path_work_dir))
+    )
+    path_dotenv = os.path.abspath(
+        os.path.realpath(os.path.join(path_work_dir, ".env"))
+    )
+    return path_repo_root, path_work_dir, path_dotenv, datetime_today