Preliminary python 3.6 port #2564

As part of our move to Python 3 we must first establish our
initial python 3 dependencies via Poetry.

## Includes:
- Update pyproject.toml and poetry.lock for python 3.6.
- Remove int long designator from settings.py: Python 3
no longer uses the "L" suffix.
- Use full path in initial import conversions. Python 3
has more stringent import path requirements.
- Update imports for 3.6 email module. These differ in Python 3.
- Encode unicode line for system.osi.md5sum.
TypeError: Unicode-objects must be encoded before hashing
- Tidy/update init_sftp_config() re Python 3.
There was a subtle file pointer behaviour change
between 2.7 and 3.6 which broke this mechanism.
- Address TODO re custom opener re Python 3.
- Services page & Users page - Python 3 style sorted():
"cmp" sorted parameter and buildin function dropped
in Python 3.
- init fix of groups page by adaptation to str type.
- Replace chardet dependency with charset-normalizer:
also used by requests.
- Removes redundant encoding on already encoded groupname.
- Init dashboard fix via gevent/websocket update.
Thanks to @FroggyFlox for investigations here.
- Init fix of get_users() - non admin users did not show.
Move to more modern - time limited subprocess.run().
- fix samba, disk partition, rockon, re iteritems()
In Python 3 we use iter(d.items()) instead:
https://peps.python.org/pep-0469/
- Address TODO re ascii to utf-8 on SMART custom options.
- Untested bytecast in smart.py:
custom options autodev related byte assertion.

pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
     "Development Status :: 4 - Beta",
     "Natural Language :: English",
     "Operating System :: POSIX :: Linux",
-    "Programming Language :: Python :: 2 :: Only", # at least initially
+    "Programming Language :: Python :: 3 :: Only", # at least initially
     # The "license" property should auto-invoke License classifier/classifiers.
 ]
 authors = [
@@ -55,7 +55,7 @@ generate-setup-file = false
 # https://python-poetry.org/docs/managing-dependencies/#dependency-groups
 #
 # https://python-poetry.org/docs/1.1/dependency-specification/
-python = "~2.7"
+python = "~3.6"
 
 # [tool.poetry.group.django.dependencies]
 django = "==1.11.29"
@@ -70,14 +70,13 @@ dbus-python = "==1.2.18"
 # N.B. officially Django >= 2.2.1 is required for psycopg2 >= 2.8
 psycopg2 = "==2.8.6"  # last Python 2.7 version, PostgreSQL 13 errorcodes map?
 psycogreen = "==1.0"
-gevent = "==1.1.2"
-gevent-websocket = "==0.9.5"
+gevent = "*"  # can be an extra dependency to gunicorn.
+gevent-websocket = "*"
 # Python WSGI HTTP Server - 20.0 (2019/10/30) dropped Python 2.7.
 gunicorn = "==19.10.0"  # buildout previously used 19.7.1
 
 # [tool.poetry.group.requests.dependencies]
 requests = "==2.27.1"  # Last Python 2/3 version, requires chardet
-chardet = "==4.0.0"  # 5.0.0 requires Python3.
 idna = "==2.10"  # Requests (2.27.1) requires idna<3,>=2.5
 certifi = "==2021.10.8"  # Requests (2.27.1) requires certifi>=2017.4.17
 urllib3 = "==1.26.12"  # Requests (2.27.1) requires urllib3<1.27,>=1.21.1

src/rockstor/cli/__init__.py

@@ -1 +1 @@
-from api_wrapper import APIWrapper  # noqa F401
+from cli.api_wrapper import APIWrapper  # noqa F401

src/rockstor/cli/rest_util.py

@@ -23,7 +23,7 @@
 import base64
 from storageadmin.exceptions import RockStorAPIException
 from functools import wraps
-from base_console import BaseConsole
+from cli.base_console import BaseConsole
 from storageadmin.models import OauthApp
 from django.conf import settings
 

src/rockstor/fs/btrfs.py

@@ -33,7 +33,7 @@
 )
 from system.exceptions import CommandException
 from system.constants import MOUNT, UMOUNT, RMDIR, DEFAULT_MNT_DIR
-from pool_scrub import PoolScrub
+from fs.pool_scrub import PoolScrub
 from huey.contrib.djhuey import task
 from django.conf import settings
 import logging

src/rockstor/rest_framework_custom/__init__.py

@@ -16,5 +16,5 @@
 along with this program. If not, see <http://www.gnu.org/licenses/>.
 """
 
-from generic_view import GenericView  # noqa F401
-from renderers import IgnoreClient  # noqa F401
+from rest_framework_custom.generic_view import GenericView  # noqa F401
+from rest_framework_custom.renderers import IgnoreClient  # noqa F401

src/rockstor/rest_framework_custom/generic_view.py

@@ -21,7 +21,7 @@
                                            SessionAuthentication,)
 from storageadmin.auth import DigestAuthentication
 from rest_framework.permissions import IsAuthenticated
-from oauth_wrapper import RockstorOAuth2Authentication
+from rest_framework_custom.oauth_wrapper import RockstorOAuth2Authentication
 from contextlib import contextmanager
 from storageadmin.util import handle_exception
 from storageadmin.exceptions import RockStorAPIException

src/rockstor/settings.py

@@ -19,7 +19,7 @@
 
 # Django settings for Rockstor project.
 import os
-import subprocess, distro
+import distro
 from huey import SqliteHuey
 
 DEBUG = False
@@ -317,7 +317,7 @@
 enough for all practical purposes and also is the max allowed in btrfs.
 """
 MIN_SHARE_SIZE = 100
-MAX_SHARE_SIZE = 18014398509481984L
+MAX_SHARE_SIZE = 18014398509481984
 
 START_UID = 5000
 END_UID = 6000

src/rockstor/smart_manager/models/__init__.py

@@ -16,23 +16,23 @@
 along with this program. If not, see <http://www.gnu.org/licenses/>.
 """
 
-from cpu_metric import CPUMetric  # noqa E501
-from disk_stat import DiskStat  # noqa E501
-from load_avg import LoadAvg  # noqa E501
-from mem_info import MemInfo  # noqa E501
-from vm_stat import VmStat  # noqa E501
-from service import Service, ServiceStatus  # noqa E501
-from sprobe import SProbe  # noqa E501
-from nfsd import NFSDCallDistribution, NFSDClientDistribution  # noqa E501  # noqa E501
-from nfsd_share import NFSDShareDistribution  # noqa E501
-from pool_usage import PoolUsage  # noqa E501
-from net_stat import NetStat  # noqa E501
-from nfsd_share_client import NFSDShareClientDistribution  # noqa E501
-from share_usage import ShareUsage  # noqa E501
-from nfsd_uid_gid import NFSDUidGidDistribution  # noqa E501
-from task_def import TaskDefinition  # noqa E501
-from task import Task  # noqa E501
-from share_replication import (
+from smart_manager.models.cpu_metric import CPUMetric  # noqa E501
+from smart_manager.models.disk_stat import DiskStat  # noqa E501
+from smart_manager.models.load_avg import LoadAvg  # noqa E501
+from smart_manager.models.mem_info import MemInfo  # noqa E501
+from smart_manager.models.vm_stat import VmStat  # noqa E501
+from smart_manager.models.service import Service, ServiceStatus  # noqa E501
+from smart_manager.models.sprobe import SProbe  # noqa E501
+from smart_manager.models.nfsd import NFSDCallDistribution, NFSDClientDistribution  # noqa E501  # noqa E501
+from smart_manager.models.nfsd_share import NFSDShareDistribution  # noqa E501
+from smart_manager.models.pool_usage import PoolUsage  # noqa E501
+from smart_manager.models.net_stat import NetStat  # noqa E501
+from smart_manager.models.nfsd_share_client import NFSDShareClientDistribution  # noqa E501
+from smart_manager.models.share_usage import ShareUsage  # noqa E501
+from smart_manager.models.nfsd_uid_gid import NFSDUidGidDistribution  # noqa E501
+from smart_manager.models.task_def import TaskDefinition  # noqa E501
+from smart_manager.models.task import Task  # noqa E501
+from smart_manager.models.share_replication import (
     Replica,
     ReplicaTrail,
     ReplicaShare,  # noqa E501

src/rockstor/smart_manager/views/__init__.py

@@ -15,43 +15,43 @@
 You should have received a copy of the GNU General Public License
 along with this program. If not, see <http://www.gnu.org/licenses/>.
 """
-from sprobes import SProbeView  # noqa E501
-from mem_info import MemInfoView  # noqa E501
-from net_stat import NetStatView  # noqa E501
-from disk_stat import DiskStatView  # noqa E501
-from nfs_distrib import NFSDistribView  # noqa E501
-from nfs_client_distrib import NFSDClientDistribView  # noqa E501
-from nfs_share_distrib import NFSDShareDistribView  # noqa E501
-from nfs_share_client_distrib import NFSDShareClientDistribView  # noqa E501
-from cpu_util import CPUMetricView  # noqa E501
-from nfs_uid_gid import NFSDUidGidDistributionView  # noqa E501
-from load_avg import LoadAvgView  # noqa E501
-from sprobe_metadata import SProbeMetadataView, SProbeMetadataDetailView  # noqa E501
-from base_service import BaseServiceView, BaseServiceDetailView  # noqa E501
-from nis_service import NISServiceView  # noqa E501
-from samba_service import SambaServiceView  # noqa E501
-from nfs_service import NFSServiceView  # noqa E501
-from replication import ReplicaListView, ReplicaDetailView  # noqa E501
-from replica_trail import ReplicaTrailListView, ReplicaTrailDetailView  # noqa E501
-from replication_service import ReplicationServiceView  # noqa E501
-from ntp_service import NTPServiceView  # noqa E501
-from ldap_service import LdapServiceView  # noqa E501
-from sftp_service import SFTPServiceView  # noqa E501
-from tasktypes import TaskTypeView  # noqa E501
-from task_scheduler import TaskSchedulerListView, TaskSchedulerDetailView  # noqa E501
-from task_log import TaskLogView  # noqa E501
-from replica_share import ReplicaShareListView, ReplicaShareDetailView  # noqa E501
-from receive_trail import ReceiveTrailListView, ReceiveTrailDetailView  # noqa E501
-from ts_service import TaskSchedulerServiceView  # noqa E501
-from dc_service import DataCollectorServiceView  # noqa E501
-from sm_service import ServiceMonitorView  # noqa E501
-from snmp_service import SNMPServiceView  # noqa E501
-from docker_service import DockerServiceView  # noqa E501
-from smartd_service import SMARTDServiceView  # noqa E501
-from nut_service import NUTServiceView  # noqa E501
-from active_directory import ActiveDirectoryServiceView  # noqa E501
-from receiver_pools import ReceiverPoolListView  # noqa E501
-from ztaskd_service import ZTaskdServiceView  # noqa E501
-from bootstrap_service import BootstrapServiceView  # noqa E501
-from shellinaboxd_service import ShellInABoxServiceView  # noqa E501
-from rockstor_service import RockstorServiceView  # noqa E501
+from smart_manager.views.sprobes import SProbeView  # noqa E501
+from smart_manager.views.mem_info import MemInfoView  # noqa E501
+from smart_manager.views.net_stat import NetStatView  # noqa E501
+from smart_manager.views.disk_stat import DiskStatView  # noqa E501
+from smart_manager.views.nfs_distrib import NFSDistribView  # noqa E501
+from smart_manager.views.nfs_client_distrib import NFSDClientDistribView  # noqa E501
+from smart_manager.views.nfs_share_distrib import NFSDShareDistribView  # noqa E501
+from smart_manager.views.nfs_share_client_distrib import NFSDShareClientDistribView  # noqa E501
+from smart_manager.views.cpu_util import CPUMetricView  # noqa E501
+from smart_manager.views.nfs_uid_gid import NFSDUidGidDistributionView  # noqa E501
+from smart_manager.views.load_avg import LoadAvgView  # noqa E501
+from smart_manager.views.sprobe_metadata import SProbeMetadataView, SProbeMetadataDetailView  # noqa E501
+from smart_manager.views.base_service import BaseServiceView, BaseServiceDetailView  # noqa E501
+from smart_manager.views.nis_service import NISServiceView  # noqa E501
+from smart_manager.views.samba_service import SambaServiceView  # noqa E501
+from smart_manager.views.nfs_service import NFSServiceView  # noqa E501
+from smart_manager.views.replication import ReplicaListView, ReplicaDetailView  # noqa E501
+from smart_manager.views.replica_trail import ReplicaTrailListView, ReplicaTrailDetailView  # noqa E501
+from smart_manager.views.replication_service import ReplicationServiceView  # noqa E501
+from smart_manager.views.ntp_service import NTPServiceView  # noqa E501
+from smart_manager.views.ldap_service import LdapServiceView  # noqa E501
+from smart_manager.views.sftp_service import SFTPServiceView  # noqa E501
+from smart_manager.views.tasktypes import TaskTypeView  # noqa E501
+from smart_manager.views.task_scheduler import TaskSchedulerListView, TaskSchedulerDetailView  # noqa E501
+from smart_manager.views.task_log import TaskLogView  # noqa E501
+from smart_manager.views.replica_share import ReplicaShareListView, ReplicaShareDetailView  # noqa E501
+from smart_manager.views.receive_trail import ReceiveTrailListView, ReceiveTrailDetailView  # noqa E501
+from smart_manager.views.ts_service import TaskSchedulerServiceView  # noqa E501
+from smart_manager.views.dc_service import DataCollectorServiceView  # noqa E501
+from smart_manager.views.sm_service import ServiceMonitorView  # noqa E501
+from smart_manager.views.snmp_service import SNMPServiceView  # noqa E501
+from smart_manager.views.docker_service import DockerServiceView  # noqa E501
+from smart_manager.views.smartd_service import SMARTDServiceView  # noqa E501
+from smart_manager.views.nut_service import NUTServiceView  # noqa E501
+from smart_manager.views.active_directory import ActiveDirectoryServiceView  # noqa E501
+from smart_manager.views.receiver_pools import ReceiverPoolListView  # noqa E501
+from smart_manager.views.ztaskd_service import ZTaskdServiceView  # noqa E501
+from smart_manager.views.bootstrap_service import BootstrapServiceView  # noqa E501
+from smart_manager.views.shellinaboxd_service import ShellInABoxServiceView  # noqa E501
+from smart_manager.views.rockstor_service import RockstorServiceView  # noqa E501

src/rockstor/smart_manager/views/active_directory.py

@@ -22,7 +22,7 @@
 from django.db import transaction
 from rest_framework.response import Response
 
-from base_service import BaseServiceDetailView
+from smart_manager.views.base_service import BaseServiceDetailView
 from smart_manager.models import Service
 from storageadmin.util import handle_exception
 from system.directory_services import (

src/rockstor/smart_manager/views/base_service.py

@@ -90,9 +90,8 @@ def get_queryset(self, *args, **kwargs):
                 sos = []
                 for s in Service.objects.all():
                     sos.append(self._get_or_create_sso(s))
-                return sorted(
-                    sos, cmp=lambda x, y: cmp(x.display_name, y.display_name)
-                )  # noqa
+                # https://docs.python.org/3.6/howto/sorting.html#key-functions
+                return sorted(sos, key=lambda each: each.display_name)
 
 
 class BaseServiceDetailView(ServiceMixin, rfc.GenericView):

src/rockstor/smart_manager/views/bootstrap_service.py

@@ -18,7 +18,7 @@
 
 from rest_framework.response import Response
 from system.services import systemctl
-from base_service import BaseServiceDetailView
+from smart_manager.views.base_service import BaseServiceDetailView
 from smart_manager.models import Service
 import logging
 

src/rockstor/smart_manager/views/cpu_util.py

@@ -18,7 +18,7 @@
 
 from smart_manager.models import CPUMetric
 from smart_manager.serializers import CPUMetricSerializer
-from generic_sprobe import GenericSProbeView
+from smart_manager.views.generic_sprobe import GenericSProbeView
 
 
 class CPUMetricView(GenericSProbeView):

src/rockstor/smart_manager/views/dc_service.py

@@ -20,7 +20,7 @@
 from storageadmin.util import handle_exception
 from system.services import superctl
 from django.db import transaction
-from base_service import BaseServiceDetailView
+from smart_manager.views.base_service import BaseServiceDetailView
 from smart_manager.models import Service
 
 

src/rockstor/smart_manager/views/disk_stat.py

@@ -20,7 +20,7 @@
 from smart_manager.models import DiskStat
 from storageadmin.models import Disk
 from smart_manager.serializers import DiskStatSerializer
-from generic_sprobe import GenericSProbeView
+from smart_manager.views.generic_sprobe import GenericSProbeView
 
 
 class DiskStatView(GenericSProbeView):

src/rockstor/smart_manager/views/docker_service.py

@@ -25,7 +25,7 @@
 from django.db import transaction
 from rest_framework.response import Response
 
-from base_service import BaseServiceDetailView
+from smart_manager.views.base_service import BaseServiceDetailView
 from fs.btrfs import mount_share
 from smart_manager.models import Service
 from storageadmin.models import Share

src/rockstor/smart_manager/views/ldap_service.py

@@ -23,7 +23,7 @@
 from django.db import transaction
 from rest_framework.response import Response
 
-from base_service import BaseServiceDetailView
+from smart_manager.views.base_service import BaseServiceDetailView
 from smart_manager.models import Service
 from storageadmin.util import handle_exception
 from system.directory_services import (

src/rockstor/smart_manager/views/mem_info.py

@@ -18,7 +18,7 @@
 
 from smart_manager.models import MemInfo
 from smart_manager.serializers import MemInfoSerializer
-from generic_sprobe import GenericSProbeView
+from smart_manager.views.generic_sprobe import GenericSProbeView
 
 
 class MemInfoView(GenericSProbeView):

src/rockstor/smart_manager/views/net_stat.py

@@ -18,7 +18,7 @@
 
 from smart_manager.models import NetStat
 from smart_manager.serializers import NetStatSerializer
-from generic_sprobe import GenericSProbeView
+from smart_manager.views.generic_sprobe import GenericSProbeView
 
 
 class NetStatView(GenericSProbeView):

src/rockstor/smart_manager/views/nfs_client_distrib.py

@@ -16,7 +16,7 @@
 along with this program. If not, see <http://www.gnu.org/licenses/>.
 """
 
-from advanced_sprobe import AdvancedSProbeView
+from smart_manager.views.advanced_sprobe import AdvancedSProbeView
 from smart_manager.models import NFSDClientDistribution
 from smart_manager.serializers import NFSDClientDistributionSerializer
 

src/rockstor/smart_manager/views/nfs_distrib.py

@@ -16,7 +16,7 @@
 along with this program. If not, see <http://www.gnu.org/licenses/>.
 """
 
-from advanced_sprobe import AdvancedSProbeView
+from smart_manager.views.advanced_sprobe import AdvancedSProbeView
 from smart_manager.models import NFSDCallDistribution
 from smart_manager.serializers import NFSDCallDistributionSerializer
 

src/rockstor/smart_manager/views/nfs_service.py

@@ -20,7 +20,7 @@
 from storageadmin.util import handle_exception
 from system.services import systemctl
 from django.db import transaction
-from base_service import BaseServiceDetailView
+from smart_manager.views.base_service import BaseServiceDetailView
 from smart_manager.models import Service
 
 import logging

src/rockstor/smart_manager/views/nfs_share_client_distrib.py

@@ -16,7 +16,7 @@
 along with this program. If not, see <http://www.gnu.org/licenses/>.
 """
 
-from advanced_sprobe import AdvancedSProbeView
+from smart_manager.views.advanced_sprobe import AdvancedSProbeView
 from smart_manager.models import NFSDShareClientDistribution
 from smart_manager.serializers import NFSDShareClientDistributionSerializer
 

src/rockstor/smart_manager/views/nfs_share_distrib.py

@@ -16,7 +16,7 @@
 along with this program. If not, see <http://www.gnu.org/licenses/>.
 """
 
-from advanced_sprobe import AdvancedSProbeView
+from smart_manager.views.advanced_sprobe import AdvancedSProbeView
 from smart_manager.models import NFSDShareDistribution
 from smart_manager.serializers import NFSDShareDistributionSerializer
 

src/rockstor/smart_manager/views/nfs_uid_gid.py

@@ -16,7 +16,7 @@
 along with this program. If not, see <http://www.gnu.org/licenses/>.
 """
 
-from advanced_sprobe import AdvancedSProbeView
+from smart_manager.views.advanced_sprobe import AdvancedSProbeView
 from smart_manager.models import NFSDUidGidDistribution
 from smart_manager.serializers import NFSDUidGidDistributionSerializer
 

src/rockstor/smart_manager/views/nis_service.py

@@ -22,7 +22,7 @@
 from system.services import systemctl
 from system.nis import configure_nis
 from django.db import transaction
-from base_service import BaseServiceDetailView
+from smart_manager.views.base_service import BaseServiceDetailView
 from smart_manager.models import Service
 
 import logging

src/rockstor/smart_manager/views/ntp_service.py

@@ -21,7 +21,7 @@
 from system.services import systemctl
 from system.osi import run_command
 from django.db import transaction
-from base_service import BaseServiceDetailView
+from smart_manager.views.base_service import BaseServiceDetailView
 from smart_manager.models import Service
 from django.conf import settings
 import re

src/rockstor/smart_manager/views/replication_service.py

@@ -20,7 +20,7 @@
 from storageadmin.util import handle_exception
 from system.services import superctl
 from django.db import transaction
-from base_service import BaseServiceDetailView
+from smart_manager.views.base_service import BaseServiceDetailView
 from smart_manager.models import Service
 from storageadmin.models import NetworkConnection