diff --git a/doc/release-notes/9356-rate-limiting.md b/doc/release-notes/9356-rate-limiting.md new file mode 100644 index 00000000000..1d68669af26 --- /dev/null +++ b/doc/release-notes/9356-rate-limiting.md @@ -0,0 +1,20 @@ +## Rate Limiting using JCache (with Hazelcast as provided by Payara) +The option to rate limit has been added to prevent users from over taxing the system either deliberately or by runaway automated processes. +Rate limiting can be configured on a tier level with tier 0 being reserved for guest users and tiers 1-any for authenticated users. +Superuser accounts are exempt from rate limiting. +Rate limits can be imposed on command APIs by configuring the tier, the command, and the hourly limit in the database. +Two database settings configure the rate limiting. +Note: If either of these settings exist in the database rate limiting will be enabled. +If neither setting exists rate limiting is disabled. + +`:RateLimitingDefaultCapacityTiers` is a comma separated list of default values for each tier. +In the following example, the default for tier `0` (guest users) is set to 10,000 calls per command per hour and tier `1` (authenticated users) is set to 20,000 calls per command per hour. +Tiers not specified in this setting will default to `-1` (No Limit). I.e., -d "10000" is equivalent to -d "10000,-1,-1,..." +`curl http://localhost:8080/api/admin/settings/:RateLimitingDefaultCapacityTiers -X PUT -d '10000,20000'` + +`:RateLimitingCapacityByTierAndAction` is a JSON object specifying the rate by tier and a list of actions (commands). +This allows for more control over the rate limit of individual API command calls. +In the following example, calls made by a guest user (tier 0) for API `GetLatestPublishedDatasetVersionCommand` is further limited to only 10 calls per hour, while an authenticated user (tier 1) will be able to make 30 calls per hour to the same API. +`curl http://localhost:8080/api/admin/settings/:RateLimitingCapacityByTierAndAction -X PUT -d '[{"tier": 0, "limitPerHour": 10, "actions": ["GetLatestPublishedDatasetVersionCommand", "GetPrivateUrlCommand", "GetDatasetCommand", "GetLatestAccessibleDatasetVersionCommand"]}, {"tier": 0, "limitPerHour": 1, "actions": ["CreateGuestbookResponseCommand", "UpdateDatasetVersionCommand", "DestroyDatasetCommand", "DeleteDataFileCommand", "FinalizeDatasetPublicationCommand", "PublishDatasetCommand"]}, {"tier": 1, "limitPerHour": 30, "actions": ["CreateGuestbookResponseCommand", "GetLatestPublishedDatasetVersionCommand", "GetPrivateUrlCommand", "GetDatasetCommand", "GetLatestAccessibleDatasetVersionCommand", "UpdateDatasetVersionCommand", "DestroyDatasetCommand", "DeleteDataFileCommand", "FinalizeDatasetPublicationCommand", "PublishDatasetCommand"]}]'` + +Hazelcast is configured in Payara and should not need any changes for this feature \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/installation/files/examples/rate-limit-actions-setting.json b/doc/sphinx-guides/source/_static/installation/files/examples/rate-limit-actions-setting.json new file mode 100644 index 00000000000..3dfc7648dc3 --- /dev/null +++ b/doc/sphinx-guides/source/_static/installation/files/examples/rate-limit-actions-setting.json @@ -0,0 +1,40 @@ +[ + { + "tier": 0, + "limitPerHour": 10, + "actions": [ + "GetLatestPublishedDatasetVersionCommand", + "GetPrivateUrlCommand", + "GetDatasetCommand", + "GetLatestAccessibleDatasetVersionCommand" + ] + }, + { + "tier": 0, + "limitPerHour": 1, + "actions": [ + "CreateGuestbookResponseCommand", + "UpdateDatasetVersionCommand", + "DestroyDatasetCommand", + "DeleteDataFileCommand", + "FinalizeDatasetPublicationCommand", + "PublishDatasetCommand" + ] + }, + { + "tier": 1, + "limitPerHour": 30, + "actions": [ + "CreateGuestbookResponseCommand", + "GetLatestPublishedDatasetVersionCommand", + "GetPrivateUrlCommand", + "GetDatasetCommand", + "GetLatestAccessibleDatasetVersionCommand", + "UpdateDatasetVersionCommand", + "DestroyDatasetCommand", + "DeleteDataFileCommand", + "FinalizeDatasetPublicationCommand", + "PublishDatasetCommand" + ] + } +] \ No newline at end of file diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 2baa2827250..1a3ef88a5aa 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1373,6 +1373,33 @@ Before being moved there, on your machine, large file uploads via API will cause RAM and/or swap usage bursts. You might want to point this to a different location, restrict maximum size of it, and monitor for stale uploads. +.. _cache-rate-limiting: + +Configure Your Dataverse Installation to Use JCache (with Hazelcast as Provided by Payara) for Rate Limiting +------------------------------------------------------------------------------------------------------------ + +Rate limiting has been added to prevent users from over taxing the system either deliberately or by runaway automated processes. +Rate limiting can be configured on a tier level with tier 0 being reserved for guest users and tiers 1-any for authenticated users. +Superuser accounts are exempt from rate limiting. +Rate limits can be imposed on command APIs by configuring the tier, the command, and the hourly limit in the database. +Two database settings configure the rate limiting. +Note: If either of these settings exist in the database rate limiting will be enabled (note that a Payara restart is required for the setting to take effect). If neither setting exists rate limiting is disabled. + +- :RateLimitingDefaultCapacityTiers is the number of calls allowed per hour if the specific command is not configured. The values represent the number of calls per hour per user for tiers 0,1,... + A value of -1 can be used to signify no rate limit. Tiers not specified in this setting will default to `-1` (No Limit). I.e., -d "10000" is equivalent to -d "10000,-1,-1,..." + +.. code-block:: bash + + curl http://localhost:8080/api/admin/settings/:RateLimitingDefaultCapacityTiers -X PUT -d '10000,20000' + +- :RateLimitingCapacityByTierAndAction is a JSON object specifying the rate by tier and a list of actions (commands). This allows for more control over the rate limit of individual API command calls. + In the following example, calls made by a guest user (tier 0) for API GetLatestPublishedDatasetVersionCommand is further limited to only 10 calls per hour, while an authenticated user (tier 1) will be able to make 30 calls per hour to the same API. + +:download:`rate-limit-actions.json ` Example json for RateLimitingCapacityByTierAndAction + +.. code-block:: bash + + curl http://localhost:8080/api/admin/settings/:RateLimitingCapacityByTierAndAction -X PUT -d '[{"tier": 0, "limitPerHour": 10, "actions": ["GetLatestPublishedDatasetVersionCommand", "GetPrivateUrlCommand", "GetDatasetCommand", "GetLatestAccessibleDatasetVersionCommand"]}, {"tier": 0, "limitPerHour": 1, "actions": ["CreateGuestbookResponseCommand", "UpdateDatasetVersionCommand", "DestroyDatasetCommand", "DeleteDataFileCommand", "FinalizeDatasetPublicationCommand", "PublishDatasetCommand"]}, {"tier": 1, "limitPerHour": 30, "actions": ["CreateGuestbookResponseCommand", "GetLatestPublishedDatasetVersionCommand", "GetPrivateUrlCommand", "GetDatasetCommand", "GetLatestAccessibleDatasetVersionCommand", "UpdateDatasetVersionCommand", "DestroyDatasetCommand", "DeleteDataFileCommand", "FinalizeDatasetPublicationCommand", "PublishDatasetCommand"]}]' .. _Branding Your Installation: @@ -4496,3 +4523,17 @@ tab. files saved with these headers on S3 - since they no longer have to be generated and added to the streamed file on the fly. The setting is ``false`` by default, preserving the legacy behavior. + +:RateLimitingDefaultCapacityTiers ++++++++++++++++++++++++++++++++++ +Number of calls allowed per hour if the specific command is not configured. The values represent the number of calls per hour per user for tiers 0,1,... +A value of -1 can be used to signify no rate limit. Also, by default, a tier not defined would receive a default of no limit. + +:RateLimitingCapacityByTierAndAction +++++++++++++++++++++++++++++++++++++ +JSON object specifying the rate by tier and a list of actions (commands). This allows for more control over the rate limit of individual API command calls. +In the following example, calls made by a guest user (tier 0) for API GetLatestPublishedDatasetVersionCommand is further limited to only 10 calls per hour, while an authenticated user (tier 1) will be able to make 30 calls per hour to the same API. +{"rateLimits":[ +{"tier": 0, "limitPerHour": 10, "actions": ["GetLatestPublishedDatasetVersionCommand", "GetPrivateUrlCommand", "GetDatasetCommand", "GetLatestAccessibleDatasetVersionCommand"]}, +{"tier": 0, "limitPerHour": 1, "actions": ["CreateGuestbookResponseCommand", "UpdateDatasetVersionCommand", "DestroyDatasetCommand", "DeleteDataFileCommand", "FinalizeDatasetPublicationCommand", "PublishDatasetCommand"]}, +{"tier": 1, "limitPerHour": 30, "actions": ["CreateGuestbookResponseCommand", "GetLatestPublishedDatasetVersionCommand", "GetPrivateUrlCommand", "GetDatasetCommand", "GetLatestAccessibleDatasetVersionCommand", "UpdateDatasetVersionCommand", "DestroyDatasetCommand", "DeleteDataFileCommand", "FinalizeDatasetPublicationCommand", "PublishDatasetCommand"]}]} diff --git a/pom.xml b/pom.xml index 8b2850e1df9..f736f04cf32 100644 --- a/pom.xml +++ b/pom.xml @@ -210,6 +210,18 @@ provided + + + jakarta.json.bind + jakarta.json.bind-api + + + + org.eclipse + yasson + test + + org.glassfish @@ -542,6 +554,10 @@ dataverse-spi 2.0.0 + + javax.cache + cache-api + org.junit.jupiter @@ -653,6 +669,11 @@ 3.9.0 test + + com.hazelcast + hazelcast + test + diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 05325a26f3a..4daa1fbadaf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -24,6 +24,7 @@ import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.impl.CheckRateLimitForDatasetPage; import edu.harvard.iq.dataverse.engine.command.impl.CreatePrivateUrlCommand; import edu.harvard.iq.dataverse.engine.command.impl.CuratePublishedDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.DeaccessionDatasetVersionCommand; @@ -36,6 +37,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.PublishDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.export.ExportService; +import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; import io.gdcc.spi.export.ExportException; import io.gdcc.spi.export.Exporter; import edu.harvard.iq.dataverse.ingest.IngestRequest; @@ -242,6 +244,8 @@ public enum DisplayMode { SolrClientService solrClientService; @EJB DvObjectServiceBean dvObjectService; + @EJB + CacheFactoryBean cacheFactory; @Inject DataverseRequestServiceBean dvRequestService; @Inject @@ -1930,7 +1934,10 @@ private void setIdByPersistentId() { } private String init(boolean initFull) { - + // Check for rate limit exceeded. Must be done before anything else to prevent unnecessary processing. + if (!cacheFactory.checkRate(session.getUser(), new CheckRateLimitForDatasetPage(null,null))) { + return BundleUtil.getStringFromBundle("command.exception.user.ratelimited", Arrays.asList(CheckRateLimitForDatasetPage.class.getSimpleName())); + } //System.out.println("_YE_OLDE_QUERY_COUNTER_"); // for debug purposes setDataverseSiteUrl(systemConfig.getDataverseSiteUrl()); diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 10dfa4a0e4f..4f0a3f14b99 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.dataverse.DataverseUtil; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.impl.CheckRateLimitForCollectionPage; import edu.harvard.iq.dataverse.engine.command.impl.CreateDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.CreateSavedSearchCommand; import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseCommand; @@ -31,6 +32,8 @@ import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.List; + +import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; import jakarta.ejb.EJB; import jakarta.faces.application.FacesMessage; import jakarta.faces.context.FacesContext; @@ -118,6 +121,8 @@ public enum LinkMode { @Inject DataverseHeaderFragment dataverseHeaderFragment; @EJB PidProviderFactoryBean pidProviderFactoryBean; + @EJB + CacheFactoryBean cacheFactory; private Dataverse dataverse = new Dataverse(); @@ -318,7 +323,10 @@ public void updateOwnerDataverse() { public String init() { //System.out.println("_YE_OLDE_QUERY_COUNTER_"); // for debug purposes - + // Check for rate limit exceeded. Must be done before anything else to prevent unnecessary processing. + if (!cacheFactory.checkRate(session.getUser(), new CheckRateLimitForCollectionPage(null,null))) { + return BundleUtil.getStringFromBundle("command.exception.user.ratelimited", Arrays.asList(CheckRateLimitForCollectionPage.class.getSimpleName())); + } if (this.getAlias() != null || this.getId() != null || this.getOwnerId() == null) {// view mode for a dataverse if (this.getAlias() != null) { dataverse = dataverseService.findByAlias(this.getAlias()); diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java index 3793b6eeeb4..bb3fa475847 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java +++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; +import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; import edu.harvard.iq.dataverse.engine.DataverseEngine; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; @@ -16,6 +17,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.engine.command.exception.RateLimitCommandException; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.pidproviders.PidProviderFactoryBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; @@ -176,7 +178,9 @@ public class EjbDataverseEngine { @EJB EjbDataverseEngineInner innerEngine; - + + @EJB + CacheFactoryBean cacheFactory; @Resource EJBContext ejbCtxt; @@ -202,7 +206,11 @@ public R submit(Command aCommand) throws CommandException { try { logRec.setUserIdentifier( aCommand.getRequest().getUser().getIdentifier() ); - + // Check for rate limit exceeded. Must be done before anything else to prevent unnecessary processing. + if (!cacheFactory.checkRate(aCommand.getRequest().getUser(), aCommand)) { + throw new RateLimitCommandException(BundleUtil.getStringFromBundle("command.exception.user.ratelimited", Arrays.asList(aCommand.getClass().getSimpleName())), aCommand); + } + // Check permissions - or throw an exception Map> requiredMap = aCommand.getRequiredPermissions(); if (requiredMap == null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java index 93892376edc..d63fcfa3e34 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java @@ -147,6 +147,8 @@ private AuthenticatedUser createAuthenticatedUserForView (Object[] dbRowValues, user.setMutedEmails(Type.tokenizeToSet((String) dbRowValues[15])); user.setMutedNotifications(Type.tokenizeToSet((String) dbRowValues[15])); + user.setRateLimitTier((int)dbRowValues[17]); + user.setRoles(roles); return user; } @@ -419,7 +421,7 @@ private List getUserListCore(String searchTerm, qstr += " u.createdtime, u.lastlogintime, u.lastapiusetime, "; qstr += " prov.id, prov.factoryalias, "; qstr += " u.deactivated, u.deactivatedtime, "; - qstr += " u.mutedEmails, u.mutedNotifications "; + qstr += " u.mutedEmails, u.mutedNotifications, u.rateLimitTier "; qstr += " FROM authenticateduser u,"; qstr += " authenticateduserlookup prov_lookup,"; qstr += " authenticationproviderrow prov"; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 60e0b79662b..b7305a24f69 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -20,6 +20,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.GetLatestAccessibleDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetLatestPublishedDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetSpecificPublishedDatasetVersionCommand; +import edu.harvard.iq.dataverse.engine.command.exception.RateLimitCommandException; import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; import edu.harvard.iq.dataverse.license.LicenseServiceBean; import edu.harvard.iq.dataverse.locality.StorageSiteServiceBean; @@ -421,7 +422,7 @@ public Command handleLatestPublished() { })); return dsv; } - + protected DataFile findDataFileOrDie(String id) throws WrappedResponse { DataFile datafile; if (id.equals(PERSISTENT_ID_KEY)) { @@ -575,6 +576,8 @@ protected T execCommand( Command cmd ) throws WrappedResponse { try { return engineSvc.submit(cmd); + } catch (RateLimitCommandException ex) { + throw new WrappedResponse(rateLimited(ex.getMessage())); } catch (IllegalCommandException ex) { //for 8859 for api calls that try to update datasets with TOA out of compliance if (ex.getMessage().toLowerCase().contains("terms of use")){ @@ -776,11 +779,15 @@ protected Response notFound( String msg ) { protected Response badRequest( String msg ) { return error( Status.BAD_REQUEST, msg ); } - + protected Response forbidden( String msg ) { return error( Status.FORBIDDEN, msg ); } - + + protected Response rateLimited( String msg ) { + return error( Status.TOO_MANY_REQUESTS, msg ); + } + protected Response conflict( String msg ) { return error( Status.CONFLICT, msg ); } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index b307c655798..d6d3e0317ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -16,6 +16,7 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; + import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import java.io.Serializable; import java.sql.Timestamp; @@ -42,6 +43,7 @@ import jakarta.persistence.PostLoad; import jakarta.persistence.PrePersist; import jakarta.persistence.Transient; +import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; @@ -146,6 +148,10 @@ public class AuthenticatedUser implements User, Serializable { @Transient private Set mutedNotificationsSet = new HashSet<>(); + @Column(nullable=false) + @Min(value = 1, message = "Rate Limit Tier must be greater than 0.") + private int rateLimitTier = 1; + @PrePersist void prePersist() { mutedNotifications = Type.toStringValue(mutedNotificationsSet); @@ -397,6 +403,13 @@ public void setDeactivatedTime(Timestamp deactivatedTime) { this.deactivatedTime = deactivatedTime; } + public int getRateLimitTier() { + return rateLimitTier; + } + public void setRateLimitTier(int rateLimitTier) { + this.rateLimitTier = rateLimitTier; + } + @OneToOne(mappedBy = "authenticatedUser") private AuthenticatedUserLookup authenticatedUserLookup; @@ -435,7 +448,6 @@ public void setShibIdentityProvider(String shibIdentityProvider) { public JsonObjectBuilder toJson() { //JsonObjectBuilder authenicatedUserJson = Json.createObjectBuilder(); - NullSafeJsonBuilder authenicatedUserJson = NullSafeJsonBuilder.jsonObjectBuilder(); authenicatedUserJson.add("id", this.id); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/RateLimitCommandException.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/RateLimitCommandException.java new file mode 100644 index 00000000000..99a665b31ac --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/RateLimitCommandException.java @@ -0,0 +1,16 @@ +package edu.harvard.iq.dataverse.engine.command.exception; + +import edu.harvard.iq.dataverse.engine.command.Command; + +/** + * An exception raised when a command cannot be executed, due to the + * issuing user being rate limited. + * + * @author + */ +public class RateLimitCommandException extends CommandException { + + public RateLimitCommandException(String message, Command aCommand) { + super(message, aCommand); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForCollectionPage.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForCollectionPage.java new file mode 100644 index 00000000000..9dcf0428fff --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForCollectionPage.java @@ -0,0 +1,16 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +public class CheckRateLimitForCollectionPage extends AbstractVoidCommand { + public CheckRateLimitForCollectionPage(DataverseRequest aRequest, DvObject dvObject) { + super(aRequest, dvObject); + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForDatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForDatasetPage.java new file mode 100644 index 00000000000..04a27d082f4 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForDatasetPage.java @@ -0,0 +1,17 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +public class CheckRateLimitForDatasetPage extends AbstractVoidCommand { + + public CheckRateLimitForDatasetPage(DataverseRequest aRequest, DvObject dvObject) { + super(aRequest, dvObject); + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index b05c88c0be2..2d1667f0cc5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -239,6 +239,10 @@ public enum Key { CVocConf, + // Default calls per hour for each tier. csv format (30,60,...) + RateLimitingDefaultCapacityTiers, + // json defined list of capacities by tier and action list. See RateLimitSetting.java + RateLimitingCapacityByTierAndAction, /** * A link to an installation of https://github.com/IQSS/miniverse or * some other metrics app. diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 3f1ec3dd7eb..3f2f36ea36a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1146,11 +1146,21 @@ public Long getTestStorageQuotaLimit() { return settingsService.getValueForKeyAsLong(SettingsServiceBean.Key.StorageQuotaSizeInBytes); } /** - * Should we store tab-delimited files produced during ingest *with* the - * variable name header line included? + * Should we store tab-delimited files produced during ingest *with* the + * variable name header line included? * @return boolean - defaults to false. */ public boolean isStoringIngestedFilesWithHeaders() { return settingsService.isTrueForKey(SettingsServiceBean.Key.StoreIngestedTabularFilesWithVarHeaders, false); } + + /** + * RateLimitUtil will parse the json to create a List + */ + public String getRateLimitsJson() { + return settingsService.getValueForKey(SettingsServiceBean.Key.RateLimitingCapacityByTierAndAction, ""); + } + public String getRateLimitingDefaultCapacityTiers() { + return settingsService.getValueForKey(SettingsServiceBean.Key.RateLimitingDefaultCapacityTiers, ""); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java b/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java new file mode 100644 index 00000000000..36b2b35b48f --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java @@ -0,0 +1,60 @@ +package edu.harvard.iq.dataverse.util.cache; + +import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.engine.command.Command; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.annotation.PostConstruct; +import jakarta.ejb.EJB; +import jakarta.ejb.Singleton; +import jakarta.ejb.Startup; +import jakarta.inject.Inject; + +import javax.cache.Cache; +import javax.cache.CacheManager; +import javax.cache.configuration.CompleteConfiguration; +import javax.cache.configuration.MutableConfiguration; +import javax.cache.spi.CachingProvider; +import java.util.logging.Logger; + +@Singleton +@Startup +public class CacheFactoryBean implements java.io.Serializable { + private static final Logger logger = Logger.getLogger(CacheFactoryBean.class.getCanonicalName()); + // Retrieved from Hazelcast, implements ConcurrentMap and is threadsafe + Cache rateLimitCache; + @EJB + SystemConfig systemConfig; + @Inject + CacheManager manager; + @Inject + CachingProvider provider; + public final static String RATE_LIMIT_CACHE = "rateLimitCache"; + + @PostConstruct + public void init() { + rateLimitCache = manager.getCache(RATE_LIMIT_CACHE); + if (rateLimitCache == null) { + CompleteConfiguration config = + new MutableConfiguration() + .setTypes( String.class, String.class ); + rateLimitCache = manager.createCache(RATE_LIMIT_CACHE, config); + } + } + + /** + * Check if user can make this call or if they are rate limited + * @param user + * @param command + * @return true if user is superuser or rate not limited + */ + public boolean checkRate(User user, Command command) { + final String action = command.getClass().getSimpleName(); + int capacity = RateLimitUtil.getCapacity(systemConfig, user, action); + if (capacity == RateLimitUtil.NO_LIMIT) { + return true; + } else { + String cacheKey = RateLimitUtil.generateCacheKey(user, action); + return (!RateLimitUtil.rateLimited(rateLimitCache, cacheKey, capacity)); + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitSetting.java b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitSetting.java new file mode 100644 index 00000000000..54da5a46670 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitSetting.java @@ -0,0 +1,40 @@ +package edu.harvard.iq.dataverse.util.cache; + +import java.util.ArrayList; +import java.util.List; + +public class RateLimitSetting { + + private int tier; + private int limitPerHour = RateLimitUtil.NO_LIMIT; + private List actions = new ArrayList<>(); + + private int defaultLimitPerHour; + + public RateLimitSetting() {} + + public void setTier(int tier) { + this.tier = tier; + } + public int getTier() { + return this.tier; + } + public void setLimitPerHour(int limitPerHour) { + this.limitPerHour = limitPerHour; + } + public int getLimitPerHour() { + return this.limitPerHour; + } + public void setActions(List actions) { + this.actions = actions; + } + public List getActions() { + return this.actions; + } + public void setDefaultLimit(int defaultLimitPerHour) { + this.defaultLimitPerHour = defaultLimitPerHour; + } + public int getDefaultLimitPerHour() { + return this.defaultLimitPerHour; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java new file mode 100644 index 00000000000..b566cd42fe1 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java @@ -0,0 +1,134 @@ +package edu.harvard.iq.dataverse.util.cache; + +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.GuestUser; +import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.json.bind.Jsonb; +import jakarta.json.bind.JsonbBuilder; +import jakarta.json.bind.JsonbException; + +import javax.cache.Cache; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.logging.Logger; + +import static java.lang.Math.max; +import static java.lang.Math.min; + +public class RateLimitUtil { + private static final Logger logger = Logger.getLogger(RateLimitUtil.class.getCanonicalName()); + static final List rateLimits = new CopyOnWriteArrayList<>(); + static final Map rateLimitMap = new ConcurrentHashMap<>(); + public static final int NO_LIMIT = -1; + + static String generateCacheKey(final User user, final String action) { + return (user != null ? user.getIdentifier() : GuestUser.get().getIdentifier()) + + (action != null ? ":" + action : ""); + } + static int getCapacity(SystemConfig systemConfig, User user, String action) { + if (user != null && user.isSuperuser()) { + return NO_LIMIT; + } + // get the capacity, i.e. calls per hour, from config + return (user instanceof AuthenticatedUser authUser) ? + getCapacityByTierAndAction(systemConfig, authUser.getRateLimitTier(), action) : + getCapacityByTierAndAction(systemConfig, 0, action); + } + static boolean rateLimited(final Cache rateLimitCache, final String key, int capacityPerHour) { + if (capacityPerHour == NO_LIMIT) { + return false; + } + long currentTime = System.currentTimeMillis() / 60000L; // convert to minutes + double tokensPerMinute = (capacityPerHour / 60.0); + // Get the last time this bucket was added to + final String keyLastUpdate = String.format("%s:last_update",key); + long lastUpdate = longFromKey(rateLimitCache, keyLastUpdate); + long deltaTime = currentTime - lastUpdate; + // Get the current number of tokens in the bucket + long tokens = longFromKey(rateLimitCache, key); + long tokensToAdd = (long) (deltaTime * tokensPerMinute); + if (tokensToAdd > 0) { // Don't update timestamp if we aren't adding any tokens to the bucket + tokens = min(capacityPerHour, tokens + tokensToAdd); + rateLimitCache.put(keyLastUpdate, String.valueOf(currentTime)); + } + // Update with any added tokens and decrement 1 token for this call if not rate limited (0 tokens) + rateLimitCache.put(key, String.valueOf(max(0, tokens-1))); + return tokens < 1; + } + + static int getCapacityByTierAndAction(SystemConfig systemConfig, Integer tier, String action) { + if (rateLimits.isEmpty()) { + init(systemConfig); + } + + if (rateLimitMap.containsKey(getMapKey(tier, action))) { + return rateLimitMap.get(getMapKey(tier,action)); + } else if (rateLimitMap.containsKey(getMapKey(tier))) { + return rateLimitMap.get(getMapKey(tier)); + } else { + return getCapacityByTier(systemConfig, tier); + } + } + static int getCapacityByTier(SystemConfig systemConfig, int tier) { + int value = NO_LIMIT; + String csvString = systemConfig.getRateLimitingDefaultCapacityTiers(); + try { + if (!csvString.isEmpty()) { + int[] values = Arrays.stream(csvString.split(",")).mapToInt(Integer::parseInt).toArray(); + if (tier < values.length) { + value = values[tier]; + } + } + } catch (NumberFormatException nfe) { + logger.warning(nfe.getMessage()); + } + return value; + } + static void init(SystemConfig systemConfig) { + getRateLimitsFromJson(systemConfig); + /* Convert the List of Rate Limit Settings containing a list of Actions to a fast lookup Map where the key is: + for default if no action defined: "{tier}:" and the value is the default limit for the tier + for each action: "{tier}:{action}" and the value is the limit defined in the setting + */ + rateLimitMap.clear(); + rateLimits.forEach(r -> { + r.setDefaultLimit(getCapacityByTier(systemConfig, r.getTier())); + rateLimitMap.put(getMapKey(r.getTier()), r.getDefaultLimitPerHour()); + r.getActions().forEach(a -> rateLimitMap.put(getMapKey(r.getTier(), a), r.getLimitPerHour())); + }); + } + + @SuppressWarnings("java:S2133") // <- To enable casting to generic in JSON-B we need a class instance, false positive + static void getRateLimitsFromJson(SystemConfig systemConfig) { + String setting = systemConfig.getRateLimitsJson(); + rateLimits.clear(); + if (!setting.isEmpty()) { + try (Jsonb jsonb = JsonbBuilder.create()) { + rateLimits.addAll(jsonb.fromJson(setting, + new ArrayList() {}.getClass().getGenericSuperclass())); + } catch (JsonbException e) { + logger.warning("Unable to parse Rate Limit Json: " + e.getLocalizedMessage() + " Json:(" + setting + ")"); + rateLimits.add(new RateLimitSetting()); // add a default entry to prevent re-initialization + // Note: Usually using Exception in a catch block is an antipattern and should be avoided. + // As the JSON-B interface does not specify a non-generic type, we have to use this. + } catch (Exception e) { + logger.warning("Could not close JSON-B reader"); + } + } + } + static String getMapKey(int tier) { + return getMapKey(tier, null); + } + static String getMapKey(int tier, String action) { + return tier + ":" + (action != null ? action : ""); + } + static long longFromKey(Cache cache, String key) { + Object l = cache.get(key); + return l != null ? Long.parseLong(String.valueOf(l)) : 0L; + } +} diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 17dd0933f55..1b9ffd53e55 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2629,6 +2629,7 @@ pid.allowedCharacters=^[A-Za-z0-9._/:\\-]* command.exception.only.superusers={1} can only be called by superusers. command.exception.user.deactivated={0} failed: User account has been deactivated. command.exception.user.deleted={0} failed: User account has been deleted. +command.exception.user.ratelimited={0} failed: Rate limited due to too many requests. #Admin-API admin.api.auth.mustBeSuperUser=Forbidden. You must be a superuser. diff --git a/src/main/resources/db/migration/V6.1.0.7.sql b/src/main/resources/db/migration/V6.1.0.7.sql new file mode 100644 index 00000000000..470483e2bf4 --- /dev/null +++ b/src/main/resources/db/migration/V6.1.0.7.sql @@ -0,0 +1 @@ +ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS ratelimittier int DEFAULT 1; diff --git a/src/test/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBeanTest.java new file mode 100644 index 00000000000..92fd6731e93 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBeanTest.java @@ -0,0 +1,332 @@ +package edu.harvard.iq.dataverse.util.cache; + +import com.hazelcast.cluster.Address; +import com.hazelcast.config.Config; +import com.hazelcast.core.Hazelcast; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.map.IMap; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.GuestUser; +import edu.harvard.iq.dataverse.engine.command.Command; +import edu.harvard.iq.dataverse.engine.command.impl.ListDataverseContentCommand; +import edu.harvard.iq.dataverse.engine.command.impl.ListExplicitGroupsCommand; +import edu.harvard.iq.dataverse.engine.command.impl.ListFacetsCommand; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.testing.Tags; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +import javax.cache.Cache; +import javax.cache.CacheManager; +import javax.cache.configuration.CacheEntryListenerConfiguration; +import javax.cache.configuration.Configuration; +import javax.cache.integration.CompletionListener; +import javax.cache.processor.EntryProcessor; +import javax.cache.processor.EntryProcessorException; +import javax.cache.processor.EntryProcessorResult; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +public class CacheFactoryBeanTest { + private SystemConfig mockedSystemConfig; + static CacheFactoryBean cache = null; + + AuthenticatedUser authUser = new AuthenticatedUser(); + GuestUser guestUser = GuestUser.get(); + static final String settingDefaultCapacity = "30,60,120"; + public String getJsonSetting() { + return """ + [ + { + "tier": 0, + "limitPerHour": 10, + "actions": [ + "GetLatestPublishedDatasetVersionCommand", + "GetPrivateUrlCommand", + "GetDatasetCommand", + "GetLatestAccessibleDatasetVersionCommand" + ] + }, + { + "tier": 0, + "limitPerHour": 1, + "actions": [ + "CreateGuestbookResponseCommand", + "UpdateDatasetVersionCommand", + "DestroyDatasetCommand", + "DeleteDataFileCommand", + "FinalizeDatasetPublicationCommand", + "PublishDatasetCommand" + ] + }, + { + "tier": 1, + "limitPerHour": 30, + "actions": [ + "CreateGuestbookResponseCommand", + "GetLatestPublishedDatasetVersionCommand", + "GetPrivateUrlCommand", + "GetDatasetCommand", + "GetLatestAccessibleDatasetVersionCommand", + "UpdateDatasetVersionCommand", + "DestroyDatasetCommand", + "DeleteDataFileCommand", + "FinalizeDatasetPublicationCommand", + "PublishDatasetCommand" + ] + } + ]"""; + } + @BeforeEach + public void init() throws IOException { + // Reuse cache and config for all tests + if (cache == null) { + mockedSystemConfig = mock(SystemConfig.class); + doReturn(settingDefaultCapacity).when(mockedSystemConfig).getRateLimitingDefaultCapacityTiers(); + doReturn(getJsonSetting()).when(mockedSystemConfig).getRateLimitsJson(); + cache = new CacheFactoryBean(); + cache.systemConfig = mockedSystemConfig; + if (cache.rateLimitCache == null) { + cache.rateLimitCache = new TestCache(getConfig()); + } + + // Clear the static data, so it can be reloaded with the new mocked data + RateLimitUtil.rateLimitMap.clear(); + RateLimitUtil.rateLimits.clear(); + } + + // Reset to default auth user + authUser.setRateLimitTier(1); + authUser.setSuperuser(false); + authUser.setUserIdentifier("authUser"); + } + + @AfterAll + public static void cleanup() { + Hazelcast.shutdownAll(); + } + @Test + public void testGuestUserGettingRateLimited() { + Command action = new ListDataverseContentCommand(null,null); + boolean rateLimited = false; + int cnt = 0; + for (; cnt <100; cnt++) { + rateLimited = !cache.checkRate(guestUser, action); + if (rateLimited) { + break; + } + } + String key = RateLimitUtil.generateCacheKey(guestUser, action.getClass().getSimpleName()); + assertTrue(cache.rateLimitCache.containsKey(key)); + assertTrue(rateLimited && cnt > 1 && cnt <= 30, "rateLimited:"+rateLimited + " cnt:"+cnt); + } + + @Test + public void testAdminUserExemptFromGettingRateLimited() { + Command action = new ListExplicitGroupsCommand(null,null); + authUser.setSuperuser(true); + authUser.setUserIdentifier("admin"); + boolean rateLimited = false; + int cnt = 0; + for (; cnt <100; cnt++) { + rateLimited = !cache.checkRate(authUser, action); + if (rateLimited) { + break; + } + } + assertFalse(rateLimited); + assertTrue(cnt >= 99, "cnt:"+cnt); + } + + @Test + @Tag(Tags.NOT_ESSENTIAL_UNITTESTS) + public void testAuthenticatedUserGettingRateLimited() throws InterruptedException { + Command action = new ListFacetsCommand(null,null); + authUser.setRateLimitTier(2); // 120 cals per hour - 1 added token every 30 seconds + boolean rateLimited = false; + int cnt; + for (cnt = 0; cnt <200; cnt++) { + rateLimited = !cache.checkRate(authUser, action); + if (rateLimited) { + break; + } + } + assertTrue(rateLimited); + assertEquals(120, cnt); + + for (cnt = 0; cnt <60; cnt++) { + Thread.sleep(1000);// Wait for bucket to be replenished (check each second for 1 minute max) + rateLimited = !cache.checkRate(authUser, action); + if (!rateLimited) { + break; + } + } + assertFalse(rateLimited, "rateLimited:"+rateLimited + " cnt:"+cnt); + + // Now change the user's tier, so it is no longer limited + authUser.setRateLimitTier(3); // tier 3 = no limit + for (cnt = 0; cnt <200; cnt++) { + rateLimited = !cache.checkRate(authUser, action); + if (rateLimited) { + break; + } + } + assertFalse(rateLimited); + assertEquals(200, cnt); + } + + private Config getConfig() { + return getConfig(null); + } + private Config getConfig(String members) { + Config config = new Config(); + config.getNetworkConfig().getJoin().getAutoDetectionConfig().setEnabled(false); + config.getNetworkConfig().getJoin().getMulticastConfig().setEnabled(false); + config.getNetworkConfig().getJoin().getAwsConfig().setEnabled(false); + config.getNetworkConfig().getJoin().getAzureConfig().setEnabled(false); + config.getNetworkConfig().getJoin().getTcpIpConfig().setEnabled(true); + if (members != null) { + config.getNetworkConfig().getJoin().getAutoDetectionConfig().setEnabled(true); + config.getNetworkConfig().getJoin().getTcpIpConfig().addMember(members); + } + return config; + } + + // convert Hazelcast IMap to JCache Cache + private class TestCache implements Cache{ + HazelcastInstance hzInstance; + IMap cache; + TestCache(Config config) { + hzInstance = Hazelcast.newHazelcastInstance(config); + cache = hzInstance.getMap("test"); + Address address = hzInstance.getCluster().getLocalMember().getAddress(); + cache.put("memberAddress", String.format("%s:%d", address.getHost(), address.getPort())); + } + @Override + public String get(String s) { + return cache.get(s); + } + @Override + public Map getAll(Set set) { + return null; + } + @Override + public boolean containsKey(String s) { + return get(s) != null; + } + @Override + public void loadAll(Set set, boolean b, CompletionListener completionListener) { + + } + @Override + public void put(String s, String s2) { + cache.put(s,s2); + } + @Override + public String getAndPut(String s, String s2) { + return null; + } + @Override + public void putAll(Map map) { + + } + @Override + public boolean putIfAbsent(String s, String s2) { + return false; + } + @Override + public boolean remove(String s) { + return false; + } + @Override + public boolean remove(String s, String s2) { + return false; + } + @Override + public String getAndRemove(String s) { + return null; + } + @Override + public boolean replace(String s, String s2, String v1) { + return false; + } + @Override + public boolean replace(String s, String s2) { + return false; + } + @Override + public String getAndReplace(String s, String s2) { + return null; + } + @Override + public void removeAll(Set set) { + + } + @Override + public void removeAll() { + + } + @Override + public void clear() { + cache.clear(); + } + @Override + public > C getConfiguration(Class aClass) { + return null; + } + @Override + public T invoke(String s, EntryProcessor entryProcessor, Object... objects) throws EntryProcessorException { + return null; + } + @Override + public Map> invokeAll(Set set, EntryProcessor entryProcessor, Object... objects) { + return null; + } + @Override + public String getName() { + return null; + } + @Override + public CacheManager getCacheManager() { + return null; + } + @Override + public void close() { + hzInstance.shutdown(); + } + @Override + public boolean isClosed() { + return false; + } + @Override + public T unwrap(Class aClass) { + return null; + } + @Override + public void registerCacheEntryListener(CacheEntryListenerConfiguration cacheEntryListenerConfiguration) { + + } + @Override + public void deregisterCacheEntryListener(CacheEntryListenerConfiguration cacheEntryListenerConfiguration) { + + } + @Override + public Iterator> iterator() { + return null; + } + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtilTest.java new file mode 100644 index 00000000000..5ddcc190993 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtilTest.java @@ -0,0 +1,152 @@ +package edu.harvard.iq.dataverse.util.cache; + +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.GuestUser; +import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.util.SystemConfig; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +public class RateLimitUtilTest { + + static SystemConfig mockedSystemConfig = mock(SystemConfig.class); + static SystemConfig mockedSystemConfigBad = mock(SystemConfig.class); + + static String getJsonSetting() { + return """ + [ + { + "tier": 0, + "limitPerHour": 10, + "actions": [ + "GetLatestPublishedDatasetVersionCommand", + "GetPrivateUrlCommand", + "GetDatasetCommand", + "GetLatestAccessibleDatasetVersionCommand" + ] + }, + { + "tier": 0, + "limitPerHour": 1, + "actions": [ + "CreateGuestbookResponseCommand", + "UpdateDatasetVersionCommand", + "DestroyDatasetCommand", + "DeleteDataFileCommand", + "FinalizeDatasetPublicationCommand", + "PublishDatasetCommand" + ] + }, + { + "tier": 1, + "limitPerHour": 30, + "actions": [ + "CreateGuestbookResponseCommand", + "GetLatestPublishedDatasetVersionCommand", + "GetPrivateUrlCommand", + "GetDatasetCommand", + "GetLatestAccessibleDatasetVersionCommand", + "UpdateDatasetVersionCommand", + "DestroyDatasetCommand", + "DeleteDataFileCommand", + "FinalizeDatasetPublicationCommand", + "PublishDatasetCommand" + ] + } + ]"""; + } + static final String settingJsonBad = "{\n"; + + @BeforeAll + public static void setUp() { + doReturn(getJsonSetting()).when(mockedSystemConfig).getRateLimitsJson(); + doReturn("100,200").when(mockedSystemConfig).getRateLimitingDefaultCapacityTiers(); + doReturn(settingJsonBad).when(mockedSystemConfigBad).getRateLimitsJson(); + doReturn("100,200").when(mockedSystemConfigBad).getRateLimitingDefaultCapacityTiers(); + } + @BeforeEach + public void resetRateLimitUtilSettings() { + RateLimitUtil.rateLimitMap.clear(); + RateLimitUtil.rateLimits.clear(); + } + @ParameterizedTest + @CsvSource(value = { + "100,0,", + "200,1,", + "1,0,DestroyDatasetCommand", + "100,0,Default Limit", + "30,1,DestroyDatasetCommand", + "200,1,Default Limit", + "-1,2,Default No Limit" + }) + void testConfig(int exp, int tier, String action) { + if (action == null) { + assertEquals(exp, RateLimitUtil.getCapacityByTier(mockedSystemConfig, tier)); + } else { + assertEquals(exp, RateLimitUtil.getCapacityByTierAndAction(mockedSystemConfig, tier, action)); + } + } + @ParameterizedTest + @CsvSource(value = { + "100,0,", + "200,1,", + "100,0,GetLatestAccessibleDatasetVersionCommand", + "200,1,GetLatestAccessibleDatasetVersionCommand", + "-1,2,GetLatestAccessibleDatasetVersionCommand" + }) + void testBadJson(int exp, int tier, String action) { + if (action == null) { + assertEquals(exp, RateLimitUtil.getCapacityByTier(mockedSystemConfigBad, tier)); + } else { + assertEquals(exp, RateLimitUtil.getCapacityByTierAndAction(mockedSystemConfigBad, tier, action)); + } + } + + @Test + public void testGenerateCacheKey() { + User user = GuestUser.get(); + assertEquals(RateLimitUtil.generateCacheKey(user,"action1"), ":guest:action1"); + } + @Test + public void testGetCapacity() { + SystemConfig config = mock(SystemConfig.class); + resetRateLimitUtil(config, true); + + GuestUser guestUser = GuestUser.get(); + assertEquals(10, RateLimitUtil.getCapacity(config, guestUser, "GetPrivateUrlCommand")); + + AuthenticatedUser authUser = new AuthenticatedUser(); + authUser.setRateLimitTier(1); + assertEquals(30, RateLimitUtil.getCapacity(config, authUser, "GetPrivateUrlCommand")); + authUser.setSuperuser(true); + assertEquals(RateLimitUtil.NO_LIMIT, RateLimitUtil.getCapacity(config, authUser, "GetPrivateUrlCommand")); + + // no setting means rate limiting is not on + resetRateLimitUtil(config, false); + + assertEquals(RateLimitUtil.NO_LIMIT, RateLimitUtil.getCapacity(config, guestUser, "GetPrivateUrlCommand")); + assertEquals(RateLimitUtil.NO_LIMIT, RateLimitUtil.getCapacity(config, guestUser, "xyz")); + assertEquals(RateLimitUtil.NO_LIMIT, RateLimitUtil.getCapacity(config, authUser, "GetPrivateUrlCommand")); + assertEquals(RateLimitUtil.NO_LIMIT, RateLimitUtil.getCapacity(config, authUser, "abc")); + authUser.setRateLimitTier(99); + assertEquals(RateLimitUtil.NO_LIMIT, RateLimitUtil.getCapacity(config, authUser, "def")); + } + private void resetRateLimitUtil(SystemConfig config, boolean enable) { + doReturn(enable ? getJsonSetting() : "").when(config).getRateLimitsJson(); + doReturn(enable ? "100,200" : "").when(config).getRateLimitingDefaultCapacityTiers(); + RateLimitUtil.rateLimitMap.clear(); + RateLimitUtil.rateLimits.clear(); + } +}