From b4b6b668ac5f63652b133c4c02e95572533dd5a7 Mon Sep 17 00:00:00 2001 From: Ritwik Puri Date: Thu, 20 Apr 2023 19:07:20 +0530 Subject: [PATCH] feat: email oauth (v13) (#20790) --- frappe/core/doctype/user/user.py | 8 -- .../doctype/email_account/email_account.js | 49 ++++++++++--- .../doctype/email_account/email_account.json | 40 +++++++++- .../doctype/email_account/email_account.py | 53 ++++++++++---- frappe/email/oauth.py | 73 +++++++++++++++++++ frappe/email/queue.py | 10 +-- frappe/email/receive.py | 31 ++++++-- frappe/email/smtp.py | 44 ++++------- .../doctype/connected_app/connected_app.py | 36 +++++++-- .../doctype/token_cache/token_cache.json | 5 +- .../doctype/token_cache/token_cache.py | 17 +++-- 11 files changed, 273 insertions(+), 93 deletions(-) create mode 100644 frappe/email/oauth.py diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index a28934d00ed1..7bce2f365179 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -769,14 +769,6 @@ def get_email_awaiting(user): ) if waiting: return waiting - else: - frappe.db.sql( - """update `tabUser Email` - set awaiting_password =0 - where parent = %(user)s""", - {"user": user}, - ) - return False def ask_pass_update(): diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 362c4dff57f4..8f00d4b8a708 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -76,7 +76,6 @@ frappe.ui.form.on("Email Account", { frm.set_value(key, value); }); } - frm.events.show_gmail_message_for_less_secure_apps(frm); }, use_imap: function(frm) { @@ -109,13 +108,13 @@ frappe.ui.form.on("Email Account", { onload: function(frm) { frm.set_df_property("append_to", "only_select", true); frm.set_query("append_to", "frappe.email.doctype.email_account.email_account.get_append_to"); + frm.events.show_oauth_authorization_message(frm); }, refresh: function(frm) { frm.events.set_domain_fields(frm); frm.events.enable_incoming(frm); frm.events.notify_if_unreplied(frm); - frm.events.show_gmail_message_for_less_secure_apps(frm); if(frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) { delete frappe.route_flags.delete_user_from_locals; @@ -123,13 +122,45 @@ frappe.ui.form.on("Email Account", { } }, - show_gmail_message_for_less_secure_apps: function(frm) { - frm.dashboard.clear_headline(); - if (frm.doc.service==="GMail") { - let msg = __("GMail will only work if you enable 2-step authentication and use app-specific password."); - let cta = __("Read the step by step guide here."); - msg += ` ${cta}`; - frm.dashboard.set_headline_alert(msg); + authorize_api_access: function (frm) { + frm.events.oauth_access(frm); + }, + + oauth_access: function(frm) { + frappe.model.with_doc("Connected App", frm.doc.connected_app, () => { + const connected_app = frappe.get_doc("Connected App", frm.doc.connected_app); + return frappe.call({ + doc: connected_app, + method: "initiate_web_application_flow", + args: { + success_uri: window.location.pathname, + user: frm.doc.connected_user, + }, + callback: function (r) { + window.open(r.message, "_self"); + }, + }); + }); + }, + + show_oauth_authorization_message(frm) { + if (frm.doc.auth_method === "OAuth") { + frappe.call({ + method: "frappe.integrations.doctype.connected_app.connected_app.has_token", + args: { + connected_app: frm.doc.connected_app, + connected_user: frm.doc.connected_user, + }, + callback: (r) => { + if (!r.message) { + let msg = __( + 'OAuth has been enabled but not authorised. Please use "Authorise API Access" button to do the same.' + ); + frm.dashboard.clear_headline(); + frm.dashboard.set_headline_alert(msg, "yellow"); + } + }, + }); } }, diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index 61729cc59cb4..f0892d83fccb 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -14,10 +14,14 @@ "domain", "service", "authentication_column", + "auth_method", "password", "awaiting_password", "ascii_encode_password", + "authorize_api_access", "column_break_10", + "connected_app", + "connected_user", "login_id_is_different", "login_id", "mailbox_settings", @@ -97,6 +101,7 @@ "label": "Email Login ID" }, { + "depends_on": "eval: doc.auth_method === \"Basic\"", "fieldname": "password", "fieldtype": "Password", "hide_days": 1, @@ -105,6 +110,7 @@ }, { "default": "0", + "depends_on": "eval: doc.auth_method === \"Basic\"", "fieldname": "awaiting_password", "fieldtype": "Check", "hide_days": 1, @@ -113,6 +119,7 @@ }, { "default": "0", + "depends_on": "eval: doc.auth_method === \"Basic\"", "fieldname": "ascii_encode_password", "fieldtype": "Check", "hide_days": 1, @@ -571,12 +578,41 @@ "fieldname": "use_starttls", "fieldtype": "Check", "label": "Use STARTTLS" + }, + { + "default": "Basic", + "fieldname": "auth_method", + "fieldtype": "Select", + "label": "Method", + "options": "Basic\nOAuth" + }, + { + "depends_on": "eval: doc.auth_method === \"OAuth\"", + "fieldname": "connected_app", + "fieldtype": "Link", + "label": "Connected App", + "mandatory_depends_on": "eval: doc.auth_method === \"OAuth\"", + "options": "Connected App" + }, + { + "depends_on": "eval: doc.auth_method === \"OAuth\"", + "fieldname": "connected_user", + "fieldtype": "Link", + "label": "Connected User", + "mandatory_depends_on": "eval: doc.auth_method === \"OAuth\"", + "options": "User" + }, + { + "depends_on": "eval: doc.auth_method === \"OAuth\" && !doc.__islocal && !doc.__unsaved", + "fieldname": "authorize_api_access", + "fieldtype": "Button", + "label": "Authorize API Access" } ], "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2022-08-19 10:26:00.762228", + "modified": "2023-04-20 17:47:17.222361", "modified_by": "Administrator", "module": "Email", "name": "Email Account", @@ -598,4 +634,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 37571b9f01d0..d361947bf591 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -81,17 +81,18 @@ def validate(self): if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - # if self.enable_incoming and not self.append_to: - # frappe.throw(_("Append To is mandatory for incoming mails")) - + use_oauth = self.auth_method == "OAuth" self.use_starttls = cint(self.use_imap and self.use_starttls and not self.use_ssl) - if ( - not self.awaiting_password - and not frappe.local.flags.in_install - and not frappe.local.flags.in_patch - ): - if self.password or self.smtp_server in ("127.0.0.1", "localhost"): + validate_oauth = False + if use_oauth: + # no need for awaiting password for oauth + self.awaiting_password = 0 + self.password = None + validate_oauth = not (self.is_new() and not self.get_oauth_token()) + + if not self.awaiting_password and not frappe.local.flags.in_install: + if validate_oauth or self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 @@ -100,7 +101,8 @@ def validate(self): self.check_smtp() else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): - frappe.throw(_("Password is required or select Awaiting Password")) + if not use_oauth: + frappe.throw(_("Password is required or select Awaiting Password")) if self.notify_if_unreplied: if not self.send_notification_to: @@ -188,12 +190,15 @@ def check_smtp(self): if not self.smtp_server: frappe.throw(_("{0} is required").format("SMTP Server")) + oauth_token = self.get_oauth_token() server = SMTPServer( login=getattr(self, "login_id", None) or self.email_id, server=self.smtp_server, port=cint(self.smtp_port), use_tls=cint(self.use_tls), use_ssl=cint(self.use_ssl_for_outgoing), + use_oauth=self.auth_method == "OAuth", + access_token=oauth_token.get_password("access_token") if oauth_token else None, ) if self.password and not self.no_smtp_authentication: server.password = self.get_password() @@ -205,6 +210,7 @@ def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"): if frappe.cache().get_value("workers:no-internet") == True: return None + oauth_token = self.get_oauth_token() args = frappe._dict( { "email_account": self.name, @@ -217,6 +223,8 @@ def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"): "uid_validity": self.uidvalidity, "incoming_port": get_port(self), "initial_sync_count": self.initial_sync_count or 100, + "use_oauth": self.auth_method == "OAuth", + "access_token": oauth_token.get_password("access_token") if oauth_token else None, } ) @@ -848,6 +856,11 @@ def append_email_to_sent_folder(self, message): except Exception: frappe.log_error() + def get_oauth_token(self): + if self.auth_method == "OAuth": + connected_app = frappe.get_doc("Connected App", self.connected_app) + return connected_app.get_active_token(self.connected_user) + @frappe.whitelist() def get_append_to( @@ -939,15 +952,27 @@ def notify_unreplied(): def pull(now=False): """Will be called via scheduler, pull emails from all enabled Email accounts.""" + from frappe.integrations.doctype.connected_app.connected_app import has_token + if frappe.cache().get_value("workers:no-internet") == True: if test_internet(): frappe.cache().set_value("workers:no-internet", False) else: return + queued_jobs = get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site] - for email_account in frappe.get_list( - "Email Account", filters={"enable_incoming": 1, "awaiting_password": 0} + + for email_account in frappe.get_all( + "Email Account", + filters={"enable_incoming": 1, "awaiting_password": 0}, + fields=["name", "connected_user", "connected_app", "auth_method"], ): + if email_account.auth_method == "OAuth" and not has_token( + email_account.connected_app, email_account.connected_user + ): + # don't try to pull from accounts which dont have access token (for Oauth) + continue + if now: pull_from_email_account(email_account.name) @@ -1068,10 +1093,10 @@ def remove_user_email_inbox(email_account): doc.save(ignore_permissions=True) -@frappe.whitelist(allow_guest=False) +@frappe.whitelist() def set_email_password(email_account, user, password): account = frappe.get_doc("Email Account", email_account) - if account.awaiting_password: + if account.awaiting_password and account.auth_method != "OAuth": account.awaiting_password = 0 account.password = password try: diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py new file mode 100644 index 000000000000..00991e0ea831 --- /dev/null +++ b/frappe/email/oauth.py @@ -0,0 +1,73 @@ +import base64 +from imaplib import IMAP4 +from poplib import POP3 +from smtplib import SMTP + +import frappe + + +class Oauth: + def __init__( + self, + conn, + email_account, + email, + access_token, + mechanism="XOAUTH2", + ): + + self.email_account = email_account + self.email = email + self._mechanism = mechanism + self._conn = conn + self._access_token = access_token + + self._validate() + + def _validate(self) -> None: + if not self._access_token: + frappe.throw( + frappe._("Please Authorize OAuth for Email Account {}").format(self.email_account), + title=frappe._("OAuth Error"), + ) + + @property + def _auth_string(self) -> str: + return f"user={self.email}\1auth=Bearer {self._access_token}\1\1" + + def connect(self) -> None: + try: + if isinstance(self._conn, POP3): + self._connect_pop() + + elif isinstance(self._conn, IMAP4): + self._connect_imap() + + else: + # SMTP + self._connect_smtp() + + except Exception: + frappe.log_error( + title="Email Connection Error - Authentication Failed", + ) + # raising a bare exception here as we have a lot of exception handling present + # where the connect method is called from - hence just logging and raising. + raise + + def _connect_pop(self) -> None: + # NOTE: poplib doesn't have AUTH command implementation + res = self._conn._shortcmd( + "AUTH {} {}".format( + self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8") + ) + ) + + if not res.startswith(b"+OK"): + raise + + def _connect_imap(self) -> None: + self._conn.authenticate(self._mechanism, lambda x: self._auth_string) + + def _connect_smtp(self) -> None: + self._conn.auth(self._mechanism, lambda x: self._auth_string, initial_response_ok=False) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index d67a83cfbfb0..330fbf06ae66 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -409,8 +409,6 @@ def flush(from_test=False): msgprint(_("Emails are muted")) from_test = True - smtpserver_dict = frappe._dict() - try: queued_jobs = set(get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site]) except Exception: @@ -424,13 +422,8 @@ def flush(from_test=False): if email.name: job_name = f"email_queue_sendmail_{email.name}" - smtpserver = smtpserver_dict.get(email.sender) - if not smtpserver: - smtpserver = SMTPServer() - smtpserver_dict[email.sender] = smtpserver - if from_test: - send_one(email.name, smtpserver, auto_commit) + send_one(email.name, auto_commit) else: if job_name in queued_jobs: frappe.logger().debug(f"Not queueing job {job_name} because it is in queue already") @@ -438,7 +431,6 @@ def flush(from_test=False): send_one_args = { "email": email.name, - "smtpserver": smtpserver, "auto_commit": auto_commit, } enqueue( diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 3aa930d3f710..c3d88976c7d6 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -18,6 +18,7 @@ import frappe from frappe import _, safe_decode, safe_encode from frappe.core.doctype.file.file import MaxFileSizeReachedError, get_random_filename +from frappe.email.oauth import Oauth from frappe.utils import ( cint, convert_utc_to_user_timezone, @@ -70,10 +71,7 @@ def process_message(self, mail): def connect(self): """Connect to **Email Account**.""" - if cint(self.settings.use_imap): - return self.connect_imap() - else: - return self.connect_pop() + return self.connect_imap() if cint(self.settings.use_imap) else self.connect_pop() def connect_imap(self): """Connect to IMAP""" @@ -89,7 +87,17 @@ def connect_imap(self): if self.settings.use_starttls: self.imap.starttls() - self.imap.login(self.settings.username, self.settings.password) + if self.settings.use_oauth: + Oauth( + self.imap, + self.settings.email_account, + self.settings.username, + self.settings.access_token, + ).connect() + + else: + self.imap.login(self.settings.username, self.settings.password) + # connection established! return True @@ -110,8 +118,17 @@ def connect_pop(self): self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) - self.pop.user(self.settings.username) - self.pop.pass_(self.settings.password) + if self.settings.use_oauth: + Oauth( + self.pop, + self.settings.email_account, + self.settings.username, + self.settings.access_token, + ).connect() + + else: + self.pop.user(self.settings.username) + self.pop.pass_(self.settings.password) # connection established! return True diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index da851b542968..962b00b82493 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -11,36 +11,10 @@ import frappe from frappe import _ +from frappe.email.oauth import Oauth from frappe.utils import cint, cstr, parse_addr -def send(email, append_to=None, retry=1): - """Deprecated: Send the message or add it to Outbox Email""" - - def _send(retry): - try: - smtpserver = SMTPServer(append_to=append_to) - - # validate is called in as_string - email_body = email.as_string() - - smtpserver.sess.sendmail(email.sender, email.recipients + (email.cc or []), email_body) - except smtplib.SMTPSenderRefused: - frappe.throw(_("Invalid login or password"), title="Email Failed") - raise - except smtplib.SMTPRecipientsRefused: - frappe.msgprint(_("Invalid recipient address"), title="Email Failed") - raise - except (smtplib.SMTPServerDisconnected, smtplib.SMTPAuthenticationError): - if not retry: - raise - else: - retry = retry - 1 - _send(retry) - - _send(retry) - - def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sender=None): """Returns outgoing email account based on `append_to` or the default outgoing account. If default outgoing account is not found, it will @@ -75,7 +49,6 @@ def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sen "enable_incoming": 1, "append_to": append_to, }, - cache=True, ) if email_accounts: @@ -113,6 +86,7 @@ def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sen if ( email_account.smtp_server in ["localhost", "127.0.0.1"] or email_account.no_smtp_authentication + or email_account.auth_method == "OAuth" ): raise_exception = False email_account.password = email_account.get_password(raise_exception=raise_exception) @@ -198,6 +172,8 @@ def __init__( use_tls=None, use_ssl=None, append_to=None, + use_oauth=0, + access_token=None, ): # get defaults from mail settings @@ -213,7 +189,8 @@ def __init__( self.use_ssl = cint(use_ssl) self.login = login self.password = password - + self.use_oauth = use_oauth + self.access_token = access_token else: self.setup_email_account(append_to) @@ -243,6 +220,10 @@ def setup_email_account(self, append_to=None, sender=None): self.email_account.get("always_use_account_name_as_sender_name") ) + oauth_token = self.email_account.get_oauth_token() + self.use_oauth = self.email_account.auth_method == "OAuth" + self.access_token = oauth_token.get_password("access_token") if oauth_token else None + @property def sess(self): """get session""" @@ -279,7 +260,10 @@ def sess(self): self._sess.starttls() self._sess.ehlo() - if self.login and self.password: + if self.use_oauth: + Oauth(self._sess, self.email_account, self.login, self.access_token).connect() + + elif self.password: ret = self._sess.login(str(self.login or ""), str(self.password or "")) # check if logged correctly diff --git a/frappe/integrations/doctype/connected_app/connected_app.py b/frappe/integrations/doctype/connected_app/connected_app.py index 8a8b239b25d2..7c0363c20437 100644 --- a/frappe/integrations/doctype/connected_app/connected_app.py +++ b/frappe/integrations/doctype/connected_app/connected_app.py @@ -15,6 +15,8 @@ # Disable mandatory TLS in developer mode and tests os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" +os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" + class ConnectedApp(Document): """Connect to a remote oAuth Server. Retrieve and store user's access token @@ -58,7 +60,7 @@ def get_oauth2_session(self, user=None, init=False): def initiate_web_application_flow(self, user=None, success_uri=None): """Return an authorization URL for the user. Save state in Token Cache.""" user = user or frappe.session.user - oauth = self.get_oauth2_session(init=True) + oauth = self.get_oauth2_session(user, init=True) query_params = self.get_query_params() authorization_url, state = oauth.authorization_url(self.authorization_uri, **query_params) token_cache = self.get_token_cache(user) @@ -97,6 +99,25 @@ def get_token_cache(self, user): return token_cache + def get_active_token(self, user=None): + user = user or frappe.session.user + token_cache = self.get_token_cache(user) + if token_cache and token_cache.is_expired(): + oauth_session = self.get_oauth2_session(user) + + try: + token = oauth_session.refresh_token( + body=f"redirect_uri={self.redirect_uri}", + token_url=self.token_uri, + ) + except Exception: + frappe.log_error(title="Token Refresh Error") + return None + + token_cache.update_data(token) + + return token_cache + def get_scopes(self): return [row.scope for row in self.scopes] @@ -104,7 +125,7 @@ def get_query_params(self): return {param.key: param.value for param in self.query_parameters} -@frappe.whitelist(allow_guest=True) +@frappe.whitelist(methods=["GET"], allow_guest=True) def callback(code=None, state=None): """Handle client's code. @@ -112,8 +133,6 @@ def callback(code=None, state=None): transmit a code that can be used by the local server to obtain an access token. """ - if frappe.request.method != "GET": - frappe.throw(_("Invalid request method: {}").format(frappe.request.method)) if frappe.session.user == "Guest": frappe.local.response["type"] = "redirect" @@ -137,9 +156,16 @@ def callback(code=None, state=None): code=code, client_secret=connected_app.get_password("client_secret"), include_client_id=True, - **query_params + **query_params, ) token_cache.update_data(token) frappe.local.response["type"] = "redirect" frappe.local.response["location"] = token_cache.get("success_uri") or connected_app.get_url() + + +@frappe.whitelist() +def has_token(connected_app, connected_user=None): + app = frappe.get_doc("Connected App", connected_app) + token_cache = app.get_token_cache(connected_user or frappe.session.user) + return bool(token_cache and token_cache.get_password("access_token", False)) diff --git a/frappe/integrations/doctype/token_cache/token_cache.json b/frappe/integrations/doctype/token_cache/token_cache.json index c0164050314d..50e17e271915 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.json +++ b/frappe/integrations/doctype/token_cache/token_cache.json @@ -86,7 +86,7 @@ } ], "links": [], - "modified": "2020-11-13 13:35:53.714352", + "modified": "2023-04-20 17:56:27.891296", "modified_by": "Administrator", "module": "Integrations", "name": "Token Cache", @@ -105,6 +105,5 @@ } ], "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 + "sort_order": "DESC" } \ No newline at end of file diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py index a6aeb3eb2695..68ff7136cd92 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.py +++ b/frappe/integrations/doctype/token_cache/token_cache.py @@ -4,12 +4,14 @@ from __future__ import unicode_literals -from datetime import timedelta +from datetime import datetime, timedelta + +import pytz import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import cint, cstr +from frappe.utils import cint, cstr, get_time_zone class TokenCache(Document): @@ -53,16 +55,19 @@ def update_data(self, data): return self def get_expires_in(self): - expiry_time = frappe.utils.get_datetime(self.modified) + timedelta(seconds=self.expires_in) - return (expiry_time - frappe.utils.now_datetime()).total_seconds() + system_timezone = pytz.timezone(get_time_zone()) + modified = system_timezone.localize(frappe.utils.get_datetime(self.modified)) + expiry_utc = modified.astimezone(pytz.utc) + timedelta(seconds=self.expires_in) + now_utc = datetime.utcnow().replace(tzinfo=pytz.utc) + return cint((expiry_utc - now_utc).total_seconds()) def is_expired(self): return self.get_expires_in() < 0 def get_json(self): return { - "access_token": self.get_password("access_token", ""), - "refresh_token": self.get_password("refresh_token", ""), + "access_token": self.get_password("access_token", False), + "refresh_token": self.get_password("refresh_token", False), "expires_in": self.get_expires_in(), "token_type": self.token_type, }