Merge pull request #8579 from IQSS/8227-verify-email

improve email verification, no popup, auto-verify Shib users

doc/release-notes/5663-shib-confirm-email.md

@@ -0,0 +1,7 @@
+For Shib users we now set the emailconfirmed timestamp on login. (The guides say we do this already but are wrong. It was only being set on account creation.)
+
+For Shib users, I also prevent "check for your welcome email to verify your address" from being shown in the in-app welcome/new account notification.
+
+I put in a check to make sure Shib users never get a "verify your email address" email notification.
+
+Finally, I removed the hasNoStaleVerificationTokens check from the hasVerifiedEmail method. We've never worried about if there are stale verification tokens in the database or not and this check was preventing "Verified" from being shown, even when the user has a timestamp (the timestamp being the way we know if an email is verified or not).

doc/release-notes/8227-verify-email.md

@@ -0,0 +1 @@
+The "Verify Email" button has been changed to "Send Verification Email" and rather than sometimes showing a popup now always sends a fresh verification email (and invalidates previous verification emails).

doc/sphinx-guides/source/admin/user-administration.rst

@@ -63,7 +63,7 @@ The app will send a standard welcome email with a URL the user can click, which,
 
 Should users' URL token expire, they will see a "Verify Email" button on the account information page to send another URL.
 
-Sysadmins can determine which users have verified their email addresses by looking for the presence of the value ``emailLastConfirmed`` in the JSON output from listing users (see :ref:`admin` section of Native API in the API Guide). As mentioned in the :doc:`/user/account` section of the User Guide, the email addresses for Shibboleth users are re-confirmed on every login.
+Sysadmins can determine which users have verified their email addresses by looking for the presence of the value ``emailLastConfirmed`` in the JSON output from listing users (see :ref:`admin` section of Native API in the API Guide). As mentioned in the :doc:`/user/account` section of the User Guide, the email addresses for Shibboleth users are re-confirmed on every login (so their welcome email does not contain a URL to click for this purpose).
 
 Deleting an API Token
 ---------------------

src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java

@@ -636,9 +636,8 @@ public AuthenticatedUser createAuthenticatedUser(UserRecordIdentifier userRecord
         authenticatedUser.setAuthenticatedUserLookup(auusLookup);
 
         if (ShibAuthenticationProvider.PROVIDER_ID.equals(auusLookup.getAuthenticationProviderId())) {
-            Timestamp emailConfirmedNow = new Timestamp(new Date().getTime());
             // Email addresses for Shib users are confirmed by the Identity Provider.
-            authenticatedUser.setEmailConfirmed(emailConfirmedNow);
+            authenticatedUser.updateEmailConfirmedToNow();
             authenticatedUser = save(authenticatedUser);
         } else {
             /* @todo Rather than creating a token directly here it might be
@@ -665,6 +664,7 @@ public boolean identifierExists( String idtf ) {
 
     public AuthenticatedUser updateAuthenticatedUser(AuthenticatedUser user, AuthenticatedUserDisplayInfo userDisplayInfo) {
         user.applyDisplayInfo(userDisplayInfo);
+        user.updateEmailConfirmedToNow();
         actionLogSvc.log( new ActionLogRecord(ActionLogRecord.ActionType.Auth, "updateUser")
             .setInfo(user.getIdentifier()));
         return update(user);

src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java

@@ -546,11 +546,6 @@ public boolean showVerifyEmailButton() {
         return !confirmEmailService.hasVerifiedEmail(currentUser);
     }
 
-    public boolean getHasActiveVerificationToken(){
-        //for user page to determine how to handle Confirm Email click
-        return confirmEmailService.hasActiveVerificationToken(currentUser);
-    }
-
     public boolean isEmailIsVerified() {
         return confirmEmailService.hasVerifiedEmail(currentUser);
     }

src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibAuthenticationProvider.java

@@ -33,4 +33,8 @@ public boolean isDisplayIdentifier() {
         return false;
     }
 
+    // We don't override "isEmailVerified" because we're using timestamps
+    // ("emailconfirmed" on the "authenticateduser" table) to know if
+    // Shib users have confirmed/verified their email or not.
+
 }

src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java

@@ -9,11 +9,13 @@
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2TokenData;
 import edu.harvard.iq.dataverse.userdata.UserUtil;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP;
+import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProvider;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty;
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
 import java.io.Serializable;
 import java.sql.Timestamp;
+import java.util.Date;
 import java.util.List;
 import java.util.Objects;
 import javax.json.Json;
@@ -192,6 +194,13 @@ public void applyDisplayInfo( AuthenticatedUserDisplayInfo inf ) {
         }
     }
 
+    // For Shib users, set "email confirmed" timestamp on login.
+    public void updateEmailConfirmedToNow() {
+        if (ShibAuthenticationProvider.PROVIDER_ID.equals(this.getAuthenticatedUserLookup().getAuthenticationProviderId())) {
+            Timestamp emailConfirmedNow = new Timestamp(new Date().getTime());
+            this.setEmailConfirmed(emailConfirmedNow);
+        }
+    }
 
     //For User List Admin dashboard
     @Transient

src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailPage.java

@@ -5,6 +5,7 @@
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
+import java.util.Arrays;
 import java.util.logging.Logger;
 import javax.ejb.EJB;
 import javax.faces.view.ViewScoped;
@@ -49,17 +50,27 @@ public class ConfirmEmailPage implements java.io.Serializable {
     ConfirmEmailData confirmEmailData;
 
     public String init() {
+        String failureDetails = "";
         if (token != null) {
-            ConfirmEmailExecResponse confirmEmailExecResponse = confirmEmailService.processToken(token);
-            confirmEmailData = confirmEmailExecResponse.getConfirmEmailData();
-            if (confirmEmailData != null) {
-                user = confirmEmailData.getAuthenticatedUser();
-                session.setUser(user);
-                JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("confirmEmail.details.success"));
-                return "/dataverse.xhtml?faces-redirect=true";
+            ConfirmEmailExecResponse confirmEmailExecResponse = null;
+            try {
+                confirmEmailExecResponse = confirmEmailService.processToken(token);
+                confirmEmailData = confirmEmailExecResponse.getConfirmEmailData();
+                if (confirmEmailData != null) {
+                    user = confirmEmailData.getAuthenticatedUser();
+                    session.setUser(user);
+                    JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("confirmEmail.details.success"));
+                    return "/dataverse.xhtml?faces-redirect=true";
+                } else {
+                    failureDetails = BundleUtil.getStringFromBundle("confirmEmail.details.failure.noConfirmEmailDataFromToken");
+                }
+            } catch (Exception ex) {
+                failureDetails = ex.getLocalizedMessage();
             }
+        } else {
+            failureDetails = BundleUtil.getStringFromBundle("confirmEmail.details.failure.noToken");
         }
-        JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("confirmEmail.details.failure"));
+        JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("confirmEmail.details.failure", Arrays.asList(failureDetails)));
         /**
          * @todo It would be nice to send a 404 response but if we enable this
          * then the user sees the contents of 404.xhtml rather than the contents

src/main/java/edu/harvard/iq/dataverse/confirmemail/ConfirmEmailServiceBean.java

@@ -53,11 +53,19 @@ public class ConfirmEmailServiceBean {
      * @return true if verified, false otherwise
      */
     public boolean hasVerifiedEmail(AuthenticatedUser user) {
+        // Look up the user again in case the "verify email" link was clicked in another browser.
+        user = authenticationService.findByID(user.getId());
         boolean hasTimestamp = user.getEmailConfirmed() != null;
-        boolean hasNoStaleVerificationTokens = this.findSingleConfirmEmailDataByUser(user) == null;
         boolean isVerifiedByAuthProvider = authenticationService.lookupProvider(user).isEmailVerified();
-
-        return (hasTimestamp && hasNoStaleVerificationTokens) || isVerifiedByAuthProvider;
+        // Note: In practice, we are relying on hasTimestamp to know if an email
+        // has been confirmed/verified or not. We have switched the Shib code to automatically
+        // overwrite the "confirm email" timestamp on login. So hasTimeStamp will be enough.
+        // If we ever want to get away from using "confirmed email" timestamps for Shib users
+        // we can make use of the isVerifiedByAuthProvider boolean. Currently,
+        // isVerifiedByAuthProvider is set to false in the super class and nothing
+        // is overridden in the shib auth provider (or any auth provider) but we could override
+        // isVerifiedByAuthProvider in the Shib auth provider and have it return true.
+        return hasTimestamp || isVerifiedByAuthProvider;
     }
 
     /**
@@ -128,6 +136,11 @@ private void sendLinkOnEmailChange(AuthenticatedUser aUser, String confirmationU
             userNotification.setType(UserNotification.Type.CONFIRMEMAIL);
             String subject = MailUtil.getSubjectTextBasedOnNotification(userNotification, null);
             logger.fine("sending email to " + toAddress + " with this subject: " + subject);
+            if (ShibAuthenticationProvider.PROVIDER_ID.equals(aUser.getAuthenticatedUserLookup().getAuthenticationProviderId())) {
+                // Shib users have "emailconfirmed" timestamp set on login.
+                logger.info("Returning early to prevent an email confirmation link from being sent to Shib user " + aUser.getUserIdentifier() + ".");
+                return;
+            }
             mailService.sendSystemEmail(toAddress, subject, messageBody);
         } catch (Exception ex) {
             /**
@@ -141,55 +154,59 @@ private void sendLinkOnEmailChange(AuthenticatedUser aUser, String confirmationU
     }
 
     /**
-     * Process the email confirmation token, allowing the user to confirm the
-     * email address or report on a invalid token.
+     * Process the email confirmation token. If all looks good, set the
+     * timestamp and delete the token/confirmEmailData.
      *
      * @param tokenQueried
+     * @return ConfirmEmailExecResponse
+     * @throws Exception with details of the problem we can show the user.
      */
-    public ConfirmEmailExecResponse processToken(String tokenQueried) {
+    public ConfirmEmailExecResponse processToken(String tokenQueried) throws Exception {
         deleteAllExpiredTokens();
-        ConfirmEmailExecResponse tokenUnusable = new ConfirmEmailExecResponse(tokenQueried, null);
-        ConfirmEmailData confirmEmailData = findSingleConfirmEmailDataByToken(tokenQueried);
-        if (confirmEmailData != null) {
-            if (confirmEmailData.isExpired()) {
-                // shouldn't reach here since tokens are being expired above
-                return tokenUnusable;
-            } else {
-                ConfirmEmailExecResponse goodTokenCanProceed = new ConfirmEmailExecResponse(tokenQueried, confirmEmailData);
-                if (confirmEmailData == null) {
-                    logger.fine("Invalid token.");
-                    return null;
-                }
-                long nowInMilliseconds = new Date().getTime();
-                Timestamp emailConfirmed = new Timestamp(nowInMilliseconds);
-                AuthenticatedUser authenticatedUser = confirmEmailData.getAuthenticatedUser();
-                if (authenticatedUser.isDeactivated()) {
-                    logger.fine("User is deactivated.");
-                    return null;
-                }
-                authenticatedUser.setEmailConfirmed(emailConfirmed);
-                em.remove(confirmEmailData);
-                return goodTokenCanProceed;
-            }
-        } else {
-            return tokenUnusable;
+        ConfirmEmailData confirmEmailData;
+        try {
+            confirmEmailData = findSingleConfirmEmailDataByToken(tokenQueried);
+        } catch (ConfirmEmailException ex) {
+            logger.info("processToken: could not find single ConfirmEmailData row using token " + tokenQueried);
+            throw new Exception(BundleUtil.getStringFromBundle("confirmEmail.details.failure.invalidToken"));
+        }
+        if (confirmEmailData == null) {
+            // shouldn't reach here because "invalid token" exception should have already been thrown.
+            logger.info("processToken: ConfirmEmailData is null using token " + tokenQueried);
+            throw new Exception(BundleUtil.getStringFromBundle("confirmEmail.details.failure.lookupFailed"));
+        }
+        if (confirmEmailData.isExpired()) {
+            // shouldn't reach here since tokens are being expired above
+            logger.info("processToken: Token is expired: " + tokenQueried);
+            throw new Exception(BundleUtil.getStringFromBundle("confirmEmail.details.failure.tokenExpired"));
+        }
+        // No need for null check because confirmEmailData always has a user (a foreign key).
+        AuthenticatedUser authenticatedUser = confirmEmailData.getAuthenticatedUser();
+        if (authenticatedUser.isDeactivated()) {
+            logger.info("processToken: User is deactivated. Token was " + tokenQueried);
+            throw new Exception(BundleUtil.getStringFromBundle("confirmEmail.details.failure.userDeactivated"));
         }
+        ConfirmEmailExecResponse response = new ConfirmEmailExecResponse(tokenQueried, confirmEmailData);
+        long nowInMilliseconds = new Date().getTime();
+        Timestamp emailConfirmed = new Timestamp(nowInMilliseconds);
+        authenticatedUser.setEmailConfirmed(emailConfirmed);
+        em.remove(confirmEmailData);
+        return response;
     }
 
     /**
      * @param token
      * @return Null or a single row of email confirmation data.
      */
-    private ConfirmEmailData findSingleConfirmEmailDataByToken(String token) {
-        ConfirmEmailData confirmEmailData = null;
+    private ConfirmEmailData findSingleConfirmEmailDataByToken(String token) throws ConfirmEmailException {
         TypedQuery<ConfirmEmailData> typedQuery = em.createNamedQuery("ConfirmEmailData.findByToken", ConfirmEmailData.class);
         typedQuery.setParameter("token", token);
         try {
-            confirmEmailData = typedQuery.getSingleResult();
+            return typedQuery.getSingleResult();
         } catch (NoResultException | NonUniqueResultException ex) {
-            logger.fine("When looking up " + token + " caught " + ex);
+            logger.info("findSingleConfirmEmailDataByToken: When looking up " + token + " caught an exception:" + ex);
+            throw new ConfirmEmailException("");
         }
-        return confirmEmailData;
     }
 
     public ConfirmEmailData findSingleConfirmEmailDataByUser(AuthenticatedUser user) {

src/main/java/propertyFiles/Bundle.properties

@@ -183,8 +183,6 @@ contact.context.support.ending=\n\n---\n\nMessage sent from Support contact form
 account.info=Account Information
 account.edit=Edit Account
 account.apiToken=API Token
-account.emailvalidation.header=Email Validation
-account.emailvalidation.token.exists=A verification email has been sent to {0}. Please check your inbox.
 user.isShibUser=Account information cannot be edited when logged in through an institutional account.
 user.helpShibUserMigrateOffShibBeforeLink=Leaving your institution? Please contact
 user.helpShibUserMigrateOffShibAfterLink=for assistance.
@@ -200,7 +198,8 @@ wasReturnedByReviewer=, was returned by the curator of
 # TODO: Confirm that "toReview" can be deleted.
 toReview=Don't forget to publish it or send it back to the contributor!
 # Bundle file editors, please note that "notification.welcome" is used in a unit test.
-notification.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the {1}. Want to test out Dataverse features? Use our {2}. Also, check for your welcome email to verify your address.
+notification.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the {1}. Want to test out Dataverse features? Use our {2}.
+notification.welcomeConfirmEmail=Also, check for your welcome email to verify your address.
 notification.demoSite=Demo Site
 notification.requestFileAccess=File access requested for dataset: {0} was made by {1} ({2}).
 notification.grantFileAccess=Access granted for files in dataset: {0}.
@@ -364,10 +363,16 @@ authenticationProvider.name.shib=Shibboleth
 
 #confirmemail.xhtml
 confirmEmail.pageTitle=Email Verification
-confirmEmail.submitRequest=Verify Email
+confirmEmail.submitRequest=Send Verification Email
 confirmEmail.submitRequest.success=A verification email has been sent to {0}. Note, the verify link will expire after {1}.
 confirmEmail.details.success=Email address verified!
-confirmEmail.details.failure=We were unable to verify your email address. Please navigate to your Account Information page and click the "Verify Email" button.
+confirmEmail.details.failure=We were unable to verify your email address: {0}. Please navigate to your Account Information page and click the "Send Verification Email" button.
+confirmEmail.details.failure.invalidToken=Invalid token
+confirmEmail.details.failure.lookupFailed=Lookup by token failed
+confirmEmail.details.failure.tokenExpired=Token expired
+confirmEmail.details.failure.userDeactivated=User deactivated
+confirmEmail.details.failure.noToken=No token provided
+confirmEmail.details.failure.noConfirmEmailDataFromToken=Problem processing token
 confirmEmail.details.goToAccountPageButton=Go to Account Information
 confirmEmail.notVerified=Not Verified
 confirmEmail.verified=Verified