Skip to content

Commit

Permalink
Merge pull request #67 from treasure-data/update-event-to-new-version
Browse files Browse the repository at this point in the history
Update user event to new version
  • Loading branch information
hieudion authored Apr 20, 2020
2 parents 72df95c + 6a3c39c commit 84af0dc
Show file tree
Hide file tree
Showing 12 changed files with 47 additions and 45 deletions.
25 changes: 14 additions & 11 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,31 +1,34 @@
## 0.3.9 - 2020-03-24
* [enhancement] Update UserEvent to the new API [#67](https://github.com/treasure-data/embulk-input-zendesk/pull/67)

## 0.3.8 - 2020-03-24
* [enhancement] Support `Chat` target #65 [#65](https://github.com/treasure-data/embulk-input-zendesk/pull/65)
* [enhancement] Apply Embulk Gradle #64 [#64](https://github.com/treasure-data/embulk-input-zendesk/pull/64)
* [enhancement] Support `Chat` target [#65](https://github.com/treasure-data/embulk-input-zendesk/pull/65)
* [enhancement] Apply Embulk Gradle [#64](https://github.com/treasure-data/embulk-input-zendesk/pull/64)

## 0.3.7 - 2019-08-29
* [enhancement] Replace ConfigSource#getObjectNodes method in embulk core #63 [#63](https://github.com/treasure-data/embulk-input-zendesk/pull/63)
* [enhancement] Replace ConfigSource#getObjectNodes method in embulk core [#63](https://github.com/treasure-data/embulk-input-zendesk/pull/63)

## 0.3.6 - 2019-07-02
* [enhancement] Improve error message #61 [#61](https://github.com/treasure-data/embulk-input-zendesk/pull/61)
* [enhancement] Support `end_time` field for incremental #60 [#60](https://github.com/treasure-data/embulk-input-zendesk/pull/60)
* [enhancement] Improve error message [#61](https://github.com/treasure-data/embulk-input-zendesk/pull/61)
* [enhancement] Support `end_time` field for incremental [#60](https://github.com/treasure-data/embulk-input-zendesk/pull/60)

## 0.3.5 - 2019-06-03
* [enhancement] Add new targets #58 [#58](https://github.com/treasure-data/embulk-input-zendesk/pull/58)
* [enhancement] Add new targets [#58](https://github.com/treasure-data/embulk-input-zendesk/pull/58)

## 0.3.4 - 2019-04-11
* [enhancement] Add new time format #56 [#56](https://github.com/treasure-data/embulk-input-zendesk/pull/56)
* [enhancement] Add new time format [#56](https://github.com/treasure-data/embulk-input-zendesk/pull/56)

## 0.3.3 - 2019-04-11
* [fixed] Fix trailing slash #55 [#55](https://github.com/treasure-data/embulk-input-zendesk/pull/55)
* [fixed] Fix trailing slash [#55](https://github.com/treasure-data/embulk-input-zendesk/pull/55)

## 0.3.2 - 2019-04-10
* [fixed] Fix generate config diff based on incremental config #54 [#54](https://github.com/treasure-data/embulk-input-zendesk/pull/54)
* [fixed] Fix generate config diff based on incremental config [#54](https://github.com/treasure-data/embulk-input-zendesk/pull/54)

## 0.3.1 - 2019-04-09
* [fixed] Fix checking 404 by status code when fetching related objects #52 [#52](https://github.com/treasure-data/embulk-input-zendesk/pull/52)
* [fixed] Fix checking 404 by status code when fetching related objects [#52](https://github.com/treasure-data/embulk-input-zendesk/pull/52)

## 0.3.0 - 2019-04-08
* [enhancement] Convert embulk-input-zendesk to Java #50 [#50](https://github.com/treasure-data/embulk-input-zendesk/pull/50)
* [enhancement] Convert embulk-input-zendesk to Java [#50](https://github.com/treasure-data/embulk-input-zendesk/pull/50)

## 0.2.14 - 2019-01-25
* [fixed] Disable pagination and add `dedup` option for non-incremental targets [#49](https://github.com/treasure-data/embulk-input-zendesk/pull/49)
Expand Down
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ repositories {
}

group = "org.embulk"
version = "0.3.8"
version = "0.3.9"
description = "Import records from Zendesk."

sourceCompatibility = 1.8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,10 +138,6 @@ public interface PluginTask extends Task
@ConfigDefault("[]")
List<String> getRelationshipTypes();

@Config("profile_source")
@ConfigDefault("null")
Optional<String> getProfileSource();

@Config("end_time")
@ConfigDefault("null")
Optional<String> getEndTime();
Expand Down Expand Up @@ -468,8 +464,8 @@ private void validateCustomObject(PluginTask task)
private void validateUserEvent(PluginTask task)
{
if (task.getTarget().equals(Target.USER_EVENTS)) {
if (!task.getProfileSource().isPresent()) {
throw new ConfigException("Profile Source is required for User Event Target");
if (task.getUserEventType().isPresent() && !task.getUserEventSource().isPresent()) {
throw new ConfigException("User Profile Source is required when filtering by User Event Type");
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/embulk/input/zendesk/models/Target.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public enum Target
*/
TICKETS("tickets"), USERS("users"), ORGANIZATIONS("organizations"), TICKET_EVENTS("ticket_events"),
TICKET_METRICS("metric_sets"), TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms"),
RECIPIENTS("recipients"), SCORES("responses"), OBJECT_RECORDS("data"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data"),
RECIPIENTS("recipients"), SCORES("responses"), OBJECT_RECORDS("data"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("events"),
CHAT("docs");

String jsonName;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter
stream.forEach(s ->
{
Stream<JsonNode> userEventStream = StreamSupport.stream(new UserEventSpliterator(s.get("id").asText(), buildUserEventURI(s.get("id").asText()),
getZendeskRestClient(), task, Exec.isPreview()), true);
getZendeskRestClient(), task, Exec.isPreview()), false);
userEventStream.forEach(recordImporter::addRecord);
});
}
Expand Down Expand Up @@ -110,21 +110,20 @@ private String buildOrganizationWithUserURI(final String path)
private String buildUserEventURI(final String userID)
{
final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl())
.setPath(ZendeskConstants.Url.API_USER_EVENT)
.setParameter("identifier", task.getProfileSource().get() + ":user_id:" + userID);
.setPath(String.format(ZendeskConstants.Url.API_USER_EVENT, userID));

task.getUserEventSource().ifPresent(eventSource -> uriBuilder.setParameter("source", eventSource));
task.getUserEventType().ifPresent(eventType -> uriBuilder.setParameter("type", eventType));
task.getUserEventSource().ifPresent(eventSource -> uriBuilder.setParameter("filter[source]", eventSource));
task.getUserEventType().ifPresent(eventType -> uriBuilder.setParameter("filter[type]", eventType));
task.getStartTime().ifPresent(startTime -> {
try {
uriBuilder.setParameter("start_time", ZendeskDateUtils.convertToDateTimeFormat(startTime, ZendeskConstants.Misc.ISO_INSTANT));
uriBuilder.setParameter("filter[start_time]", ZendeskDateUtils.convertToDateTimeFormat(startTime, ZendeskConstants.Misc.ISO_INSTANT));
}
catch (DataException e) {
uriBuilder.setParameter("start_time", ZendeskDateUtils.convertToDateTimeFormat(Instant.EPOCH.toString(), ZendeskConstants.Misc.ISO_INSTANT));
uriBuilder.setParameter("filter[start_time]", ZendeskDateUtils.convertToDateTimeFormat(Instant.EPOCH.toString(), ZendeskConstants.Misc.ISO_INSTANT));
}
});

task.getEndTime().ifPresent(endTime -> uriBuilder.setParameter("end_time", ZendeskDateUtils.convertToDateTimeFormat(endTime, ZendeskConstants.Misc.ISO_INSTANT)));
task.getEndTime().ifPresent(endTime -> uriBuilder.setParameter("filter[end_time]", ZendeskDateUtils.convertToDateTimeFormat(endTime, ZendeskConstants.Misc.ISO_INSTANT)));

return uriBuilder.toString();
}
Expand All @@ -138,9 +137,8 @@ private JsonNode mockJsonNode()
" \"type\": \"remove_from_cart\",\n" +
" \"source\": \"shopify\",\n" +
" \"description\": \"\",\n" +
" \"authenticated\": true,\n" +
" \"created_at\": \"2019-03-06T02:34:22Z\",\n" +
" \"received_at\": \"2019-03-06T02:34:22Z\",\n" +
" \"created_at\": \"2019-03-06T02:34:12.381847424Z\",\n" +
" \"received_at\": \"2019-03-06T02:34:12.381847424Z\",\n" +
" \"properties\": {\n" +
" \"model\": 221,\n" +
" \"size\": 6\n" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public UserEventSpliterator(final String userID, final String path, final Zendes
protected void handleRunIterator(final Iterator<JsonNode> iterator, final Consumer<? super JsonNode> action)
{
iterator.forEachRemaining(
item -> {
item -> {
if (!ZendeskUtils.isNull(item)) {
JsonNode temp = iterator.next();
// Because in the returned json doesn't have user_id, so we try to add to it
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public static class Url
public static final String API_NPS_INCREMENTAL = API + "/nps/incremental";
public static final String API_OBJECT_RECORD = "api/sunshine/objects/records";
public static final String API_RELATIONSHIP_RECORD = "api/sunshine/relationships/records";
public static final String API_USER_EVENT = "api/sunshine/events";
public static final String API_USER_EVENT = "api/v2/users/%s/events";
public static final String API_CHAT = API + "/chats";
public static final String API_CHAT_SEARCH = API_CHAT + "/search";
}
Expand All @@ -49,6 +49,7 @@ public static class Misc
public static final String JAVA_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ss.SSS'Z'";
public static final String ISO_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ssXXX";
public static final String ISO_INSTANT = "uuuu-MM-dd'T'HH:mm:ss'Z'";
public static final String JAVA_TIMESTAMP_NANO_OF_SECOND = "uuuu-MM-dd'T'HH:mm:ss.n'Z'";
public static final String RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE = "uuuu-MM-dd HH:mm:ssZ";
public static final String TOO_RECENT_START_TIME = "Too recent start_time.";
public static final int RECORDS_SIZE_PER_PAGE = 100;
Expand All @@ -58,7 +59,7 @@ public static class Misc
public static final int GUESS_BUFFER_SIZE = 1024 * 1024;
public static final List<String> SUPPORT_DATE_TIME_FORMAT = ImmutableList.copyOf(Arrays.asList(ZendeskConstants.Misc.ISO_INSTANT, ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT,
ZendeskConstants.Misc.JAVA_TIMESTAMP_FORMAT, ZendeskConstants.Misc.ISO_TIMESTAMP_FORMAT,
ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE));
ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE, ZendeskConstants.Misc.JAVA_TIMESTAMP_NANO_OF_SECOND));
}

public static class Regex
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ public void testGuessGenerateColumnsForUserEvents()
{
final ConfigSource src = ZendeskTestHelper.getConfigSource("base.yml");
src.set("target", "user_events");
src.set("profile_source", "dummy");
setupTestGuessGenerateColumn(src, "data/user_event.json", "data/expected/user_events_column.json");
}

Expand Down Expand Up @@ -309,7 +308,9 @@ public void validateUserEventShouldThrowException()
{
ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
configSource.set("target", Target.USER_EVENTS.name().toLowerCase());
assertValidation(configSource, "Profile Source is required for User Event Target");
configSource.set("user_event_type", "user_event_type");
configSource.remove("user_event_source");
assertValidation(configSource, "User Profile Source is required when filtering by User Event Type");
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import org.embulk.input.zendesk.RecordImporter;
import org.embulk.input.zendesk.ZendeskInputPlugin.PluginTask;
import org.embulk.input.zendesk.clients.ZendeskRestClient;
import org.embulk.input.zendesk.models.Target;
import org.embulk.input.zendesk.utils.ZendeskTestHelper;
import org.junit.Before;
import org.junit.Rule;
Expand Down Expand Up @@ -47,10 +48,10 @@ public void testGetData()
{
setup();

JsonNode jsonNode = zendeskUserEventService.getDataFromPath("https://abc.zendesk.com/api/sunshine/objects/records?type=user&per_page=1000", 0, true, 0);
JsonNode jsonNode = zendeskUserEventService.getDataFromPath("https://abc.zendesk.com/api/v2/users/1194092277/events?filter%5Bstart_time%5D=2019-01-20T07%3A14%3A50Z&filter%5Bend_time%5D=2019-06-20T07%3A14%3A53Z", 0, true, 0);
assertFalse(jsonNode.isNull());
assertTrue(jsonNode.has("data"));
assertTrue(jsonNode.get("data").isArray());
assertTrue(jsonNode.has(Target.USER_EVENTS.getJsonName()));
assertTrue(jsonNode.get(Target.USER_EVENTS.getJsonName()).isArray());
}

@Test
Expand Down Expand Up @@ -78,7 +79,7 @@ public void testUrlForUserEvent()

String expectedURIForOrganization = "https://abc.zendesk.com/api/v2/organizations?per_page=100&page=1";
String expectedURIForUser = "https://abc.zendesk.com/api/v2/organizations/360857467053/users.json?per_page=100&page=1";
String expectedURIForUserEvent = "https://abc.zendesk.com/api/sunshine/events?identifier=support%3Auser_id%3A1194092277&start_time=2019-01-20T07%3A14%3A50Z&end_time=2019-06-20T07%3A14%3A53Z";
String expectedURIForUserEvent = "https://abc.zendesk.com/api/v2/users/1194092277/events?filter%5Bstart_time%5D=2019-01-20T07%3A14%3A50Z&filter%5Bend_time%5D=2019-06-20T07%3A14%3A53Z";
List<String> expectedURI = Arrays.asList(expectedURIForOrganization, expectedURIForUser, expectedURIForUserEvent);

zendeskUserEventService.addRecordToImporter(0, recordImporter);
Expand All @@ -104,7 +105,7 @@ public void testAddRecordToImporterInNonPreviewMode()
.thenReturn(dataJsonUser.toString());
when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations/360857467055/users.json?per_page=100&page=1"), eq(task), eq(false)))
.thenReturn(dataJsonUser.toString());
when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/sunshine/events?identifier=support%3Auser_id%3A1194092277&start_time=2019-01-20T07%3A14%3A50Z&end_time=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/users/1194092277/events?filter%5Bstart_time%5D=2019-01-20T07%3A14%3A50Z&filter%5Bend_time%5D=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
.thenReturn(dataJsonUserEvent.toString());

zendeskUserEventService.addRecordToImporter(0, recordImporter);
Expand All @@ -131,7 +132,7 @@ public void testAddRecordToImporterWithDuplicateUser()
.thenReturn(dataJsonUser.toString());
when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations/360857467055/users.json?per_page=100&page=1"), eq(task), eq(false)))
.thenReturn(dataJsonUser.toString());
when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/sunshine/events?identifier=support%3Auser_id%3A1194092277&start_time=2019-01-20T07%3A14%3A50Z&end_time=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/users/1194092277/events?filter%5Bstart_time%5D=2019-01-20T07%3A14%3A50Z&filter%5Bend_time%5D=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
.thenReturn(dataJsonUserEvent.toString());

zendeskUserEventService.addRecordToImporter(0, recordImporter);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ public void testIsoToEpochSecondShouldReturnCorrectValue()

value = ZendeskDateUtils.isoToEpochSecond("2019-02-20 06:50:45+0000");
Assert.assertEquals(expectedValue, value);

value = ZendeskDateUtils.isoToEpochSecond("2019-02-20T06:50:45.215149154Z");
Assert.assertEquals(expectedValue, value);
}

@Test
Expand Down
1 change: 0 additions & 1 deletion src/test/resources/config/user_events.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ retry_initial_wait_sec: 1
dedup: false
max_retry_wait_sec: 30
retry_limit: 2
profile_source: support
start_time: '2019-01-20T07:14:50Z'
end_time: '2019-06-20T07:14:53Z'
columns:
Expand Down
6 changes: 3 additions & 3 deletions src/test/resources/data/user_event.json
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
{
"data": [
"events": [
{
"id": "5c7f31aef8df240001e60bbf",
"type": "remove_from_cart",
"source": "shopify",
"description": "",
"authenticated": true,
"created_at": "2019-03-06T02:34:22Z",
"received_at": "2019-03-06T02:34:22Z",
"created_at": "2019-03-06T02:34:22.381847424Z",
"received_at": "2019-03-06T02:34:22.381847424Z",
"properties": {
"model": 221,
"size": 6
Expand Down

0 comments on commit 84af0dc

Please sign in to comment.