Skip to content

Commit

Permalink
[CLOUDTRUST-1635] Fix concurrency issue
Browse files Browse the repository at this point in the history 
* Fix concurrency issue,
* Exit loop if any failure
* Improve logging
  • Loading branch information
@harture
harture authored Sep 9, 2019
1 parent 0493c60 commit 0fd1017
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 102 deletions.
45 changes: 0 additions & 45 deletions src/main/java/io/cloudtrust/keycloak/eventemitter/ConcurrentEvictingQueue.java
View file

This file was deleted.

58 changes: 52 additions & 6 deletions src/main/java/io/cloudtrust/keycloak/eventemitter/EventEmitterProvider.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.cloudtrust.keycloak.eventemitter;

import com.fasterxml.jackson.core.JsonProcessingException;
import io.cloudtrust.keycloak.snowflake.IdGenerator;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
Expand All @@ -14,6 +15,7 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Base64;
import java.util.concurrent.LinkedBlockingQueue;


/**
Expand All @@ -36,8 +38,8 @@ public class EventEmitterProvider implements EventListenerProvider{
private CloseableHttpClient httpClient;
private HttpClientContext httpContext;
private IdGenerator idGenerator;
private ConcurrentEvictingQueue<IdentifiedEvent> pendingEvents;
private ConcurrentEvictingQueue<IdentifiedAdminEvent> pendingAdminEvents;
private LinkedBlockingQueue<IdentifiedEvent> pendingEvents;
private LinkedBlockingQueue<IdentifiedAdminEvent> pendingAdminEvents;
private String targetUri;
private String username;
private String secretToken;
Expand All @@ -55,8 +57,8 @@ public class EventEmitterProvider implements EventListenerProvider{
*/
EventEmitterProvider(CloseableHttpClient httpClient, IdGenerator idGenerator,
String targetUri, SerialisationFormat format,
ConcurrentEvictingQueue<IdentifiedEvent> pendingEvents,
ConcurrentEvictingQueue<IdentifiedAdminEvent> pendingAdminEvents){
LinkedBlockingQueue<IdentifiedEvent> pendingEvents,
LinkedBlockingQueue<IdentifiedAdminEvent> pendingAdminEvents){
logger.debug("EventEmitterProvider contructor call");
this.httpClient = httpClient;
this.httpContext = HttpClientContext.create();
Expand Down Expand Up @@ -86,7 +88,19 @@ public void onEvent(Event event) {
logger.debugf("Pending Events to send stored in buffer: %d", pendingEvents.size());
long uid = idGenerator.nextValidId();
IdentifiedEvent identifiedEvent = new IdentifiedEvent(uid, event);
pendingEvents.offer(identifiedEvent);

while(!pendingEvents.offer(identifiedEvent)){
Event skippedEvent = pendingEvents.poll();
if(skippedEvent != null) {
String strEvent = null;
try {
strEvent = SerialisationUtils.toJson(skippedEvent);
} catch (JsonProcessingException e) {
strEvent = "SerializationFailure";
}
logger.errorf("Event dropped(%s) due to full queue", strEvent);
}
}

sendEvents();
}
Expand All @@ -97,7 +111,19 @@ public void onEvent(AdminEvent adminEvent, boolean b) {
logger.debugf("Pending AdminEvents to send stored in buffer: %d", pendingAdminEvents.size());
long uid = idGenerator.nextValidId();
IdentifiedAdminEvent identifiedAdminEvent = new IdentifiedAdminEvent(uid, adminEvent);
pendingAdminEvents.offer(identifiedAdminEvent);

while(!pendingAdminEvents.offer(identifiedAdminEvent)){
AdminEvent skippedAdminEvent = pendingAdminEvents.poll();
if(skippedAdminEvent != null) {
String strAdminEvent = null;
try {
strAdminEvent = SerialisationUtils.toJson(skippedAdminEvent);
} catch (JsonProcessingException e) {
strAdminEvent = "SerializationFailure";
}
logger.errorf("AdminEvent dropped(%s) due to full queue", strAdminEvent);
}
}

sendEvents();
}
Expand Down Expand Up @@ -125,27 +151,37 @@ private void sendEventsWithJsonFormat() {
for (int i=0; i < pendingEventsSize; i++){
IdentifiedEvent event = pendingEvents.poll();

if(event == null){
break;
}

try {
String json = SerialisationUtils.toJson(event);
sendJson(json);
} catch (EventEmitterException | IOException e) {
pendingEvents.offer(event);
logger.infof("Failed to send event(ID=%s), try again later.", event.getUid());
logger.debug("Failed to serialize or send event", e);
break;
}
}

int pendingAdminEventsSize = pendingAdminEvents.size();
for (int i=0; i < pendingAdminEventsSize; i++){
IdentifiedAdminEvent event = pendingAdminEvents.poll();

if(event == null){
break;
}

try {
String json = SerialisationUtils.toJson(event);
sendJson(json);
} catch (EventEmitterException | IOException e) {
pendingAdminEvents.offer(event);
logger.infof("Failed to send adminEvent(ID=%s), try again later.", event.getUid());
logger.debug("Failed to serialize or send adminEvent", e);
break;
}
}
}
Expand All @@ -156,27 +192,37 @@ private void sendEventsWithFlatbufferFormat() {
for (int i=0; i < pendingEventsSize; i++){
IdentifiedEvent event = pendingEvents.poll();

if(event == null){
break;
}

try {
ByteBuffer buffer = SerialisationUtils.toFlat(event);
sendBytes(buffer, Event.class.getSimpleName());
} catch (EventEmitterException | IOException e) {
pendingEvents.offer(event);
logger.infof("Failed to send event(ID=%s), try again later.", event.getUid());
logger.debug("Failed to serialize or send event", e);
break;
}
}

int pendingAdminEventsSize = pendingAdminEvents.size();
for (int i=0; i < pendingAdminEventsSize; i++){
IdentifiedAdminEvent event = pendingAdminEvents.poll();

if(event == null){
break;
}

try {
ByteBuffer buffer = SerialisationUtils.toFlat(event);
sendBytes(buffer, AdminEvent.class.getSimpleName());
} catch (EventEmitterException | IOException e) {
pendingAdminEvents.offer(event);
logger.infof("Failed to send adminEvent(ID=%s), try again later.", event.getUid());
logger.debug("Failed to serialize or send adminEvent", e);
break;
}
}
}
Expand Down
9 changes: 5 additions & 4 deletions src/main/java/io/cloudtrust/keycloak/eventemitter/EventEmitterProviderFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;

/**
* Factory for EventEmitterProvider.
Expand Down Expand Up @@ -50,8 +51,8 @@ public class EventEmitterProviderFactory implements EventListenerProviderFactory

private CloseableHttpClient httpClient;
private IdGenerator idGenerator;
private ConcurrentEvictingQueue<IdentifiedEvent> pendingEventsToSend;
private ConcurrentEvictingQueue<IdentifiedAdminEvent> pendingAdminEventsToSend;
private LinkedBlockingQueue<IdentifiedEvent> pendingEventsToSend;
private LinkedBlockingQueue<IdentifiedAdminEvent> pendingAdminEventsToSend;



Expand Down Expand Up @@ -132,8 +133,8 @@ public void init(Config.Scope config) {
// Initialisation
httpClient = HttpClients.createDefault();
idGenerator = new IdGenerator(keycloakId, datacenterId);
pendingEventsToSend = new ConcurrentEvictingQueue<>(bufferCapacity);
pendingAdminEventsToSend = new ConcurrentEvictingQueue<>(bufferCapacity);
pendingEventsToSend = new LinkedBlockingQueue<>(bufferCapacity);
pendingAdminEventsToSend = new LinkedBlockingQueue<>(bufferCapacity);

}

Expand Down
37 changes: 0 additions & 37 deletions src/test/java/io/cloudtrust/keycloak/eventemitter/ConcurrentEvictingQueueTest.java
View file

This file was deleted.

21 changes: 11 additions & 10 deletions src/test/java/io/cloudtrust/keycloak/eventemitter/EventEmitterProviderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.concurrent.LinkedBlockingQueue;

import org.xnio.streams.ChannelInputStream;

Expand Down Expand Up @@ -58,8 +59,8 @@ public void testFlatbufferFormatOutput() throws IOException, InterruptedExceptio
Undertow server = startHttpServer(handler);
CloseableHttpClient httpClient = HttpClients.createDefault();
IdGenerator idGenerator = new IdGenerator(1,1);
ConcurrentEvictingQueue<IdentifiedEvent> pendingEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
ConcurrentEvictingQueue<IdentifiedAdminEvent> pendingAdminEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedEvent> pendingEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedAdminEvent> pendingAdminEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
EventEmitterProvider eventEmitterProvider = new EventEmitterProvider(httpClient,
idGenerator, TARGET, SerialisationFormat.FLATBUFFER, pendingEvents,pendingAdminEvents);

Expand Down Expand Up @@ -88,8 +89,8 @@ public void testJsonFormatOutput() throws IOException, InterruptedException {
Undertow server = startHttpServer(handler);
CloseableHttpClient httpClient = HttpClients.createDefault();
IdGenerator idGenerator = new IdGenerator(1,1);
ConcurrentEvictingQueue<IdentifiedEvent> pendingEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
ConcurrentEvictingQueue<IdentifiedAdminEvent> pendingAdminEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedEvent> pendingEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedAdminEvent> pendingAdminEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
EventEmitterProvider eventEmitterProvider = new EventEmitterProvider(httpClient,
idGenerator, TARGET, SerialisationFormat.JSON, pendingEvents,pendingAdminEvents);

Expand All @@ -111,8 +112,8 @@ public void testJsonFormatOutput() throws IOException, InterruptedException {
public void testNoConnection() throws IOException {
CloseableHttpClient httpClient = HttpClients.createDefault();
IdGenerator idGenerator = new IdGenerator(1,1);
ConcurrentEvictingQueue<IdentifiedEvent> pendingEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
ConcurrentEvictingQueue<IdentifiedAdminEvent> pendingAdminEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedEvent> pendingEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedAdminEvent> pendingAdminEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
EventEmitterProvider eventEmitterProvider = new EventEmitterProvider(httpClient,
idGenerator, TARGET, SerialisationFormat.JSON, pendingEvents,pendingAdminEvents);

Expand All @@ -138,8 +139,8 @@ public void testServerError() throws IOException, InterruptedException {
Undertow server = startHttpServer(handler);
CloseableHttpClient httpClient = HttpClients.createDefault();
IdGenerator idGenerator = new IdGenerator(1,1);
ConcurrentEvictingQueue<IdentifiedEvent> pendingEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
ConcurrentEvictingQueue<IdentifiedAdminEvent> pendingAdminEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedEvent> pendingEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedAdminEvent> pendingAdminEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
EventEmitterProvider eventEmitterProvider = new EventEmitterProvider(httpClient,
idGenerator, TARGET, SerialisationFormat.JSON, pendingEvents,pendingAdminEvents);

Expand All @@ -166,8 +167,8 @@ public void testBufferAndSend() throws IOException, InterruptedException {

CloseableHttpClient httpClient = HttpClients.createDefault();
IdGenerator idGenerator = new IdGenerator(1,1);
ConcurrentEvictingQueue<IdentifiedEvent> pendingEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
ConcurrentEvictingQueue<IdentifiedAdminEvent> pendingAdminEvents = new ConcurrentEvictingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedEvent> pendingEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
LinkedBlockingQueue<IdentifiedAdminEvent> pendingAdminEvents = new LinkedBlockingQueue<>(BUFFER_CAPACITY);
EventEmitterProvider eventEmitterProvider = new EventEmitterProvider(httpClient,
idGenerator, TARGET, SerialisationFormat.JSON, pendingEvents,pendingAdminEvents);

Expand Down

0 comments on commit 0fd1017

Please sign in to comment.