Misc fixes and cleanup for project quota

1) The Coverity Scan reports some issues for the project
   quota patch, including:

1.1) zfs_prop_get_userquota() directly uses the const quota
   type value as the condition check by wrong.

1.2) dmu_objset_userquota_get_ids() may cause dnode::dn_newgid
   to be overwritten by dnode::dn->dn_oldprojid.

2) This patch fixes related issues. It also enhances the logic
   for zfs_project_item_alloc() to avoid buffer overflow.

3) Skip project quota ability check if does not change project
   quota related things (id or flag). Otherwise, it will cause
   chattr (for other non project quota flags) operation failed
   if project quota disabled.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Fan Yong <fan.yong@intel.com>
Closes #7251 
Closes #7265

cmd/zfs/zfs_project.c

@@ -43,15 +43,15 @@
 
 typedef struct zfs_project_item {
 	list_node_t	zpi_list;
-	char		zpi_name[PATH_MAX];
+	char		zpi_name[0];
 } zfs_project_item_t;
 
 static void
 zfs_project_item_alloc(list_t *head, const char *name)
 {
 	zfs_project_item_t *zpi;
 
-	zpi = safe_malloc(sizeof (zfs_project_item_t));
+	zpi = safe_malloc(sizeof (zfs_project_item_t) + strlen(name) + 1);
 	strcpy(zpi->zpi_name, name);
 	list_insert_tail(head, zpi);
 }

lib/libzfs/libzfs_dataset.c

@@ -3179,7 +3179,8 @@ zfs_prop_get_userquota(zfs_handle_t *zhp, const char *propname,
 	} else if (propvalue == 0 &&
 	    (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_GROUPQUOTA ||
 	    type == ZFS_PROP_USEROBJQUOTA || type == ZFS_PROP_GROUPOBJQUOTA ||
-	    type == ZFS_PROP_PROJECTQUOTA || ZFS_PROP_PROJECTOBJQUOTA)) {
+	    type == ZFS_PROP_PROJECTQUOTA ||
+	    type == ZFS_PROP_PROJECTOBJQUOTA)) {
 		(void) strlcpy(propbuf, "none", proplen);
 	} else if (type == ZFS_PROP_USERQUOTA || type == ZFS_PROP_GROUPQUOTA ||
 	    type == ZFS_PROP_USERUSED || type == ZFS_PROP_GROUPUSED ||

module/zfs/dmu_objset.c

@@ -2077,7 +2077,7 @@ dmu_objset_userquota_get_ids(dnode_t *dn, boolean_t before, dmu_tx_t *tx)
 		if (flags & DN_ID_OLD_EXIST) {
 			dn->dn_newuid = dn->dn_olduid;
 			dn->dn_newgid = dn->dn_oldgid;
-			dn->dn_newgid = dn->dn_oldprojid;
+			dn->dn_newprojid = dn->dn_oldprojid;
 		} else {
 			dn->dn_newuid = 0;
 			dn->dn_newgid = 0;

module/zfs/zfs_vnops.c

@@ -2891,10 +2891,12 @@ zfs_setattr(struct inode *ip, vattr_t *vap, int flags, cred_t *cr)
 		}
 
 		if (XVA_ISSET_REQ(xvap, XAT_PROJINHERIT) &&
+		    (xoap->xoa_projinherit !=
+		    ((zp->z_pflags & ZFS_PROJINHERIT) != 0)) &&
 		    (!dmu_objset_projectquota_enabled(os) ||
 		    (!S_ISREG(ip->i_mode) && !S_ISDIR(ip->i_mode)))) {
-				ZFS_EXIT(zfsvfs);
-				return (SET_ERROR(ENOTSUP));
+			ZFS_EXIT(zfsvfs);
+			return (SET_ERROR(ENOTSUP));
 		}
 	}