Add file storage option

kolibri/core/auth/csv_utils.py

@@ -1,9 +1,9 @@
 import csv
 import logging
-import os
 from collections import OrderedDict
 from functools import partial
 
+from django.core.files.storage import default_storage
 from django.db.models import OuterRef
 from django.db.models import Q
 
@@ -159,9 +159,9 @@ def map_input(obj):
 )
 
 
-def csv_file_generator(facility, filepath, overwrite=True, demographic=False):
-    if not overwrite and os.path.exists(filepath):
-        raise ValueError("{} already exists".format(filepath))
+def csv_file_generator(facility, filename, overwrite=True, demographic=False):
+    if not overwrite and default_storage.exists(filename):
+        raise ValueError("{} already exists".format(filename))
     queryset = FacilityUser.objects.filter(facility=facility)
 
     header_labels = tuple(
@@ -174,8 +174,6 @@ def csv_file_generator(facility, filepath, overwrite=True, demographic=False):
         column for column in db_columns if demographic or column not in DEMO_FIELDS
     )
 
-    csv_file = open_csv_for_writing(filepath)
-
     mappings = {}
 
     for key in output_mappings:
@@ -184,9 +182,9 @@ def csv_file_generator(facility, filepath, overwrite=True, demographic=False):
 
     map_output = partial(output_mapper, labels=labels, output_mappings=mappings)
 
-    with csv_file as f:
+    with open_csv_for_writing(filename) as f:
         writer = csv.DictWriter(f, header_labels)
-        logger.info("Creating csv file {filename}".format(filename=filepath))
+        logger.info("Creating csv file {filename}".format(filename=filename))
         writer.writeheader()
         usernames = set()
         for item in (

kolibri/core/auth/management/commands/bulkexportusers.py

@@ -1,11 +1,10 @@
 import csv
 import logging
-import ntpath
-import os
 from collections import OrderedDict
 from functools import partial
 
 from django.conf import settings
+from django.core.files.storage import default_storage
 from django.core.management.base import CommandError
 from django.db.models import OuterRef
 from django.db.models import Subquery
@@ -28,7 +27,6 @@
 from kolibri.core.tasks.utils import get_current_job
 from kolibri.core.utils.csv import open_csv_for_writing
 from kolibri.core.utils.csv import output_mapper
-from kolibri.utils import conf
 
 try:
     FileNotFoundError
@@ -152,18 +150,16 @@ def translate_labels():
     )
 
 
-def csv_file_generator(facility, filepath, overwrite=True):
-    if not overwrite and os.path.exists(filepath):
-        raise ValueError("{} already exists".format(filepath))
+def csv_file_generator(facility, filename, overwrite=True):
+    if not overwrite and default_storage.exists(filename):
+        raise ValueError("{} already exists".format(filename))
     queryset = FacilityUser.objects.filter(facility=facility)
 
     header_labels = translate_labels().values()
 
-    csv_file = open_csv_for_writing(filepath)
-
-    with csv_file as f:
+    with open_csv_for_writing(filename) as f:
         writer = csv.DictWriter(f, header_labels)
-        logger.info("Creating csv file {filename}".format(filename=filepath))
+        logger.info("Creating csv file {filename}".format(filename=filename))
         writer.writeheader()
         usernames = set()
 
@@ -248,18 +244,14 @@ def get_facility(self, options):
 
         return default_facility
 
-    def get_filepath(self, options, facility):
+    def get_filename(self, options, facility):
         if options["output_file"] is None:
-            export_dir = os.path.join(conf.KOLIBRI_HOME, "log_export")
-            if not os.path.isdir(export_dir):
-                os.mkdir(export_dir)
-            filepath = os.path.join(
-                export_dir,
-                CSV_EXPORT_FILENAMES["user"].format(facility.name, facility.id[:4]),
+            filename = default_storage.get_available_name(
+                CSV_EXPORT_FILENAMES["user"].format(facility.name, facility.id[:4])
             )
         else:
-            filepath = os.path.join(os.getcwd(), options["output_file"])
-        return filepath
+            filename = options["output_file"]
+        return filename
 
     def handle_async(self, *args, **options):
         # set language for the translation of the messages
@@ -268,14 +260,14 @@ def handle_async(self, *args, **options):
 
         self.overall_error = []
         facility = self.get_facility(options)
-        filepath = self.get_filepath(options, facility)
+        filename = self.get_filename(options, facility)
         job = get_current_job()
         total_rows = FacilityUser.objects.filter(facility=facility).count()
 
         with self.start_progress(total=total_rows) as progress_update:
             try:
                 for row in csv_file_generator(
-                    facility, filepath, overwrite=options["overwrite"]
+                    facility, filename, overwrite=options["overwrite"]
                 ):
                     progress_update(1)
             except (ValueError, IOError) as e:
@@ -288,11 +280,11 @@ def handle_async(self, *args, **options):
             if job:
                 job.extra_metadata["overall_error"] = self.overall_error
                 job.extra_metadata["users"] = total_rows
-                job.extra_metadata["filename"] = ntpath.basename(filepath)
+                job.extra_metadata["filename"] = filename
                 job.save_meta()
             else:
                 logger.info(
-                    "Created csv file {} with {} lines".format(filepath, total_rows)
+                    "Created csv file {} with {} lines".format(filename, total_rows)
                 )
 
         translation.deactivate()

kolibri/core/auth/management/commands/bulkimportusers.py

@@ -483,8 +483,7 @@ def append_error(self, msg):
         self.overall_error.append(str(msg))
 
     def csv_headers_validation(self, filepath):
-        csv_file = open_csv_for_reading(filepath)
-        with csv_file as f:
+        with open_csv_for_reading(filepath) as f:
             header = next(csv.reader(f, strict=True))
             has_header = False
             self.header_translation = {
@@ -882,8 +881,7 @@ def handle_async(self, *args, **options):
             self.exit_if_error()
             self.progress_update(1)  # state=csv_headers
             try:
-                csv_file = open_csv_for_reading(filepath)
-                with csv_file as f:
+                with open_csv_for_reading(filepath) as f:
                     reader = csv.DictReader(f, strict=True)
                     per_line_errors, classes, users, roles = self.csv_values_validation(
                         reader, self.header_translation, self.default_facility

kolibri/core/auth/management/commands/exportusers.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import sys
 
 from django.core.management.base import CommandError
@@ -57,15 +56,13 @@ def handle_async(self, *args, **options):
         else:
             filename = options["output_file"]
 
-        filepath = os.path.join(os.getcwd(), filename)
-
         total_rows = FacilityUser.objects.filter(facility=facility).count()
 
         with self.start_progress(total=total_rows) as progress_update:
             try:
                 for row in csv_file_generator(
                     facility,
-                    filepath,
+                    filename,
                     overwrite=options["overwrite"],
                     demographic=options["demographic"],
                 ):

kolibri/core/auth/management/commands/importusers.py

@@ -197,8 +197,7 @@ def handle(self, *args, **options):
 
         fieldnames = input_fields + tuple(val for val in labels.values())
 
-        csv_file = open_csv_for_reading(options["filepath"])
-        with csv_file as f:
+        with open_csv_for_reading(options["filepath"]) as f:
             header = next(csv.reader(f, strict=True))
             has_header = False
             if all(col in fieldnames for col in header):
@@ -213,8 +212,7 @@ def handle(self, *args, **options):
                     "Mix of valid and invalid header labels found in first row"
                 )
 
-        csv_file = open_csv_for_reading(options["filepath"])
-        with csv_file as f:
+        with open_csv_for_reading(options["filepath"]) as f:
             if has_header:
                 reader = csv.DictReader(f, strict=True)
             else:

kolibri/core/auth/test/test_bulk_export.py

@@ -1,5 +1,4 @@
 import csv
-import tempfile
 
 from django.test import override_settings
 from django.test import TestCase
@@ -34,11 +33,10 @@ def setUpTestData(cls):
             classroom_count=CLASSROOMS, learnergroup_count=1
         )
         cls.facility = cls.data["facility"]
-
-        _, cls.filepath = tempfile.mkstemp(suffix=".csv")
+        cls.filename = "temp.csv"
 
         cls.csv_rows = []
-        for row in cls.b.csv_file_generator(cls.facility, cls.filepath, True):
+        for row in cls.b.csv_file_generator(cls.facility, cls.filename, True):
             cls.csv_rows.append(row)
 
     def test_not_specified(self):
@@ -130,8 +128,7 @@ def test_passwords_as_asterisks(self):
             assert row["password"] == "*"
 
     def get_data_from_csv_file(self):
-        csv_file = open_csv_for_reading(self.filepath)
-        with csv_file as f:
+        with open_csv_for_reading(self.filename) as f:
             results = [row for row in csv.DictReader(f)]
         return results