Update user event to new version

CHANGELOG.md

@@ -1,31 +1,34 @@
+## 0.3.9 - 2020-03-24
+* [enhancement] Update UserEvent to the new API [#66](https://github.com/treasure-data/embulk-input-zendesk/pull/66)
+
 ## 0.3.8 - 2020-03-24
-* [enhancement] Support `Chat` target #65 [#65](https://github.com/treasure-data/embulk-input-zendesk/pull/65)
-* [enhancement] Apply Embulk Gradle  #64 [#64](https://github.com/treasure-data/embulk-input-zendesk/pull/64)
+* [enhancement] Support `Chat` target [#65](https://github.com/treasure-data/embulk-input-zendesk/pull/65)
+* [enhancement] Apply Embulk Gradle  [#64](https://github.com/treasure-data/embulk-input-zendesk/pull/64)
 
 ## 0.3.7 - 2019-08-29
-* [enhancement] Replace ConfigSource#getObjectNodes method in embulk core #63 [#63](https://github.com/treasure-data/embulk-input-zendesk/pull/63)
+* [enhancement] Replace ConfigSource#getObjectNodes method in embulk core [#63](https://github.com/treasure-data/embulk-input-zendesk/pull/63)
 
 ## 0.3.6 - 2019-07-02
-* [enhancement] Improve error message #61 [#61](https://github.com/treasure-data/embulk-input-zendesk/pull/61)
-* [enhancement] Support `end_time` field for incremental #60 [#60](https://github.com/treasure-data/embulk-input-zendesk/pull/60)
+* [enhancement] Improve error message [#61](https://github.com/treasure-data/embulk-input-zendesk/pull/61)
+* [enhancement] Support `end_time` field for incremental [#60](https://github.com/treasure-data/embulk-input-zendesk/pull/60)
 
 ## 0.3.5 - 2019-06-03
-* [enhancement] Add new targets #58 [#58](https://github.com/treasure-data/embulk-input-zendesk/pull/58)
+* [enhancement] Add new targets [#58](https://github.com/treasure-data/embulk-input-zendesk/pull/58)
 
 ## 0.3.4 - 2019-04-11
-* [enhancement] Add new time format #56 [#56](https://github.com/treasure-data/embulk-input-zendesk/pull/56)
+* [enhancement] Add new time format [#56](https://github.com/treasure-data/embulk-input-zendesk/pull/56)
 
 ## 0.3.3 - 2019-04-11
-* [fixed] Fix trailing slash #55 [#55](https://github.com/treasure-data/embulk-input-zendesk/pull/55)
+* [fixed] Fix trailing slash [#55](https://github.com/treasure-data/embulk-input-zendesk/pull/55)
 
 ## 0.3.2 - 2019-04-10
-* [fixed] Fix generate config diff based on incremental config #54 [#54](https://github.com/treasure-data/embulk-input-zendesk/pull/54)
+* [fixed] Fix generate config diff based on incremental config [#54](https://github.com/treasure-data/embulk-input-zendesk/pull/54)
 
 ## 0.3.1 - 2019-04-09
-* [fixed] Fix checking 404 by status code when fetching related objects #52 [#52](https://github.com/treasure-data/embulk-input-zendesk/pull/52)
+* [fixed] Fix checking 404 by status code when fetching related objects [#52](https://github.com/treasure-data/embulk-input-zendesk/pull/52)
 
 ## 0.3.0 - 2019-04-08
-* [enhancement] Convert embulk-input-zendesk to Java #50 [#50](https://github.com/treasure-data/embulk-input-zendesk/pull/50)
+* [enhancement] Convert embulk-input-zendesk to Java [#50](https://github.com/treasure-data/embulk-input-zendesk/pull/50)
 
 ## 0.2.14 - 2019-01-25
 * [fixed] Disable pagination and add `dedup` option for non-incremental targets [#49](https://github.com/treasure-data/embulk-input-zendesk/pull/49)

build.gradle

@@ -13,7 +13,7 @@ repositories {
 }
 
 group = "org.embulk"
-version = "0.3.8"
+version = "0.3.9"
 description = "Import records from Zendesk."
 
 sourceCompatibility = 1.8

src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java

@@ -138,10 +138,6 @@ public interface PluginTask extends Task
         @ConfigDefault("[]")
         List<String> getRelationshipTypes();
 
-        @Config("profile_source")
-        @ConfigDefault("null")
-        Optional<String> getProfileSource();
-
         @Config("end_time")
         @ConfigDefault("null")
         Optional<String> getEndTime();
@@ -468,8 +464,8 @@ private void validateCustomObject(PluginTask task)
     private void validateUserEvent(PluginTask task)
     {
         if (task.getTarget().equals(Target.USER_EVENTS)) {
-            if (!task.getProfileSource().isPresent()) {
-                throw new ConfigException("Profile Source is required for User Event Target");
+            if (task.getUserEventType().isPresent() && !task.getUserEventSource().isPresent()) {
+                throw new ConfigException("User Profile Source is required when filtering by User Event Type");
             }
         }
     }

src/main/java/org/embulk/input/zendesk/models/Target.java

@@ -12,7 +12,7 @@ public enum Target
     */
     TICKETS("tickets"), USERS("users"), ORGANIZATIONS("organizations"), TICKET_EVENTS("ticket_events"),
     TICKET_METRICS("metric_sets"), TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms"),
-    RECIPIENTS("recipients"), SCORES("responses"), OBJECT_RECORDS("data"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data"),
+    RECIPIENTS("recipients"), SCORES("responses"), OBJECT_RECORDS("data"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("events"),
     CHAT("docs");
 
     String jsonName;

src/main/java/org/embulk/input/zendesk/services/ZendeskUserEventService.java

@@ -69,7 +69,7 @@ public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter
                     stream.forEach(s ->
                     {
                         Stream<JsonNode> userEventStream = StreamSupport.stream(new UserEventSpliterator(s.get("id").asText(), buildUserEventURI(s.get("id").asText()),
-                                getZendeskRestClient(), task, Exec.isPreview()), true);
+                                getZendeskRestClient(), task, Exec.isPreview()), false);
                         userEventStream.forEach(recordImporter::addRecord);
                     });
                 }
@@ -110,21 +110,20 @@ private String buildOrganizationWithUserURI(final String path)
     private String buildUserEventURI(final String userID)
     {
         final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl())
-                .setPath(ZendeskConstants.Url.API_USER_EVENT)
-                .setParameter("identifier", task.getProfileSource().get() + ":user_id:" + userID);
+                .setPath(String.format(ZendeskConstants.Url.API_USER_EVENT, userID));
 
-        task.getUserEventSource().ifPresent(eventSource -> uriBuilder.setParameter("source", eventSource));
-        task.getUserEventType().ifPresent(eventType -> uriBuilder.setParameter("type", eventType));
+        task.getUserEventSource().ifPresent(eventSource -> uriBuilder.setParameter("filter[source]", eventSource));
+        task.getUserEventType().ifPresent(eventType -> uriBuilder.setParameter("filter[type]", eventType));
         task.getStartTime().ifPresent(startTime -> {
             try {
-                uriBuilder.setParameter("start_time", ZendeskDateUtils.convertToDateTimeFormat(startTime, ZendeskConstants.Misc.ISO_INSTANT));
+                uriBuilder.setParameter("filter[start_time]", ZendeskDateUtils.convertToDateTimeFormat(startTime, ZendeskConstants.Misc.ISO_INSTANT));
             }
             catch (DataException e) {
-                uriBuilder.setParameter("start_time", ZendeskDateUtils.convertToDateTimeFormat(Instant.EPOCH.toString(), ZendeskConstants.Misc.ISO_INSTANT));
+                uriBuilder.setParameter("filter[start_time]", ZendeskDateUtils.convertToDateTimeFormat(Instant.EPOCH.toString(), ZendeskConstants.Misc.ISO_INSTANT));
             }
         });
 
-        task.getEndTime().ifPresent(endTime -> uriBuilder.setParameter("end_time", ZendeskDateUtils.convertToDateTimeFormat(endTime, ZendeskConstants.Misc.ISO_INSTANT)));
+        task.getEndTime().ifPresent(endTime -> uriBuilder.setParameter("filter[end_time]", ZendeskDateUtils.convertToDateTimeFormat(endTime, ZendeskConstants.Misc.ISO_INSTANT)));
 
         return uriBuilder.toString();
     }
@@ -138,9 +137,8 @@ private JsonNode mockJsonNode()
                     "      \"type\": \"remove_from_cart\",\n" +
                     "      \"source\": \"shopify\",\n" +
                     "      \"description\": \"\",\n" +
-                    "      \"authenticated\": true,\n" +
-                    "      \"created_at\": \"2019-03-06T02:34:22Z\",\n" +
-                    "      \"received_at\": \"2019-03-06T02:34:22Z\",\n" +
+                    "      \"created_at\": \"2019-03-06T02:34:12.381847424Z\",\n" +
+                    "      \"received_at\": \"2019-03-06T02:34:12.381847424Z\",\n" +
                     "      \"properties\": {\n" +
                     "        \"model\": 221,\n" +
                     "        \"size\": 6\n" +

src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/UserEventSpliterator.java

@@ -5,6 +5,8 @@
 import org.embulk.input.zendesk.ZendeskInputPlugin;
 import org.embulk.input.zendesk.clients.ZendeskRestClient;
 import org.embulk.input.zendesk.utils.ZendeskUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.Iterator;
 import java.util.function.Consumer;
@@ -13,6 +15,8 @@ public class UserEventSpliterator extends SunshineSpliterator
 {
     private String userID;
 
+    private static final Logger logger = LoggerFactory.getLogger(UserEventSpliterator.class);
+
     public UserEventSpliterator(final String userID, final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
     {
         super(path, zendeskRestClient, task, isPreview);
@@ -23,7 +27,7 @@ public UserEventSpliterator(final String userID, final String path, final Zendes
     protected void handleRunIterator(final Iterator<JsonNode> iterator, final Consumer<? super JsonNode> action)
     {
         iterator.forEachRemaining(
-                item -> {
+            item -> {
                     if (!ZendeskUtils.isNull(item)) {
                         JsonNode temp = iterator.next();
                         // Because in the returned json doesn't have user_id, so we try to add to it

src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java

@@ -37,7 +37,7 @@ public static class Url
         public static final String API_NPS_INCREMENTAL = API + "/nps/incremental";
         public static final String API_OBJECT_RECORD = "api/sunshine/objects/records";
         public static final String API_RELATIONSHIP_RECORD = "api/sunshine/relationships/records";
-        public static final String API_USER_EVENT = "api/sunshine/events";
+        public static final String API_USER_EVENT = "api/v2/users/%s/events";
         public static final String API_CHAT = API + "/chats";
         public static final String API_CHAT_SEARCH = API_CHAT + "/search";
     }
@@ -49,6 +49,7 @@ public static class Misc
         public static final String JAVA_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ss.SSS'Z'";
         public static final String ISO_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ssXXX";
         public static final String ISO_INSTANT = "uuuu-MM-dd'T'HH:mm:ss'Z'";
+        public static final String JAVA_TIMESTAMP_NANO_OF_SECOND = "uuuu-MM-dd'T'HH:mm:ss.n'Z'";
         public static final String RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE = "uuuu-MM-dd HH:mm:ssZ";
         public static final String TOO_RECENT_START_TIME = "Too recent start_time.";
         public static final int RECORDS_SIZE_PER_PAGE = 100;
@@ -58,7 +59,7 @@ public static class Misc
         public static final int GUESS_BUFFER_SIZE = 1024 * 1024;
         public static final List<String> SUPPORT_DATE_TIME_FORMAT = ImmutableList.copyOf(Arrays.asList(ZendeskConstants.Misc.ISO_INSTANT, ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT,
                 ZendeskConstants.Misc.JAVA_TIMESTAMP_FORMAT, ZendeskConstants.Misc.ISO_TIMESTAMP_FORMAT,
-                ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE));
+                ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE, ZendeskConstants.Misc.JAVA_TIMESTAMP_NANO_OF_SECOND));
     }
 
     public static class Regex

src/test/java/org/embulk/input/zendesk/TestZendeskInputPlugin.java

@@ -110,7 +110,6 @@ public void testGuessGenerateColumnsForUserEvents()
     {
         final ConfigSource src = ZendeskTestHelper.getConfigSource("base.yml");
         src.set("target", "user_events");
-        src.set("profile_source", "dummy");
         setupTestGuessGenerateColumn(src, "data/user_event.json", "data/expected/user_events_column.json");
     }
 
@@ -309,7 +308,9 @@ public void validateUserEventShouldThrowException()
     {
         ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
         configSource.set("target", Target.USER_EVENTS.name().toLowerCase());
-        assertValidation(configSource, "Profile Source is required for User Event Target");
+        configSource.set("user_event_type", "user_event_type");
+        configSource.remove("user_event_source");
+        assertValidation(configSource, "User Profile Source is required when filtering by User Event Type");
     }
 
     @Test

src/test/java/org/embulk/input/zendesk/services/TestZendeskUserEventService.java

@@ -5,6 +5,7 @@
 import org.embulk.input.zendesk.RecordImporter;
 import org.embulk.input.zendesk.ZendeskInputPlugin.PluginTask;
 import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
 import org.embulk.input.zendesk.utils.ZendeskTestHelper;
 import org.junit.Before;
 import org.junit.Rule;
@@ -47,10 +48,10 @@ public void testGetData()
     {
         setup();
 
-        JsonNode jsonNode = zendeskUserEventService.getDataFromPath("https://abc.zendesk.com/api/sunshine/objects/records?type=user&per_page=1000", 0, true, 0);
+        JsonNode jsonNode = zendeskUserEventService.getDataFromPath("https://abc.zendesk.com/api/v2/users/1194092277/events?filter%5Bstart_time%5D=2019-01-20T07%3A14%3A50Z&filter%5Bend_time%5D=2019-06-20T07%3A14%3A53Z", 0, true, 0);
         assertFalse(jsonNode.isNull());
-        assertTrue(jsonNode.has("data"));
-        assertTrue(jsonNode.get("data").isArray());
+        assertTrue(jsonNode.has(Target.USER_EVENTS.getJsonName()));
+        assertTrue(jsonNode.get(Target.USER_EVENTS.getJsonName()).isArray());
     }
 
     @Test
@@ -78,7 +79,7 @@ public void testUrlForUserEvent()
 
         String expectedURIForOrganization = "https://abc.zendesk.com/api/v2/organizations?per_page=100&page=1";
         String expectedURIForUser = "https://abc.zendesk.com/api/v2/organizations/360857467053/users.json?per_page=100&page=1";
-        String expectedURIForUserEvent = "https://abc.zendesk.com/api/sunshine/events?identifier=support%3Auser_id%3A1194092277&start_time=2019-01-20T07%3A14%3A50Z&end_time=2019-06-20T07%3A14%3A53Z";
+        String expectedURIForUserEvent = "https://abc.zendesk.com/api/v2/users/1194092277/events?filter%5Bstart_time%5D=2019-01-20T07%3A14%3A50Z&filter%5Bend_time%5D=2019-06-20T07%3A14%3A53Z";
         List<String> expectedURI = Arrays.asList(expectedURIForOrganization, expectedURIForUser, expectedURIForUserEvent);
 
         zendeskUserEventService.addRecordToImporter(0, recordImporter);
@@ -104,7 +105,7 @@ public void testAddRecordToImporterInNonPreviewMode()
                 .thenReturn(dataJsonUser.toString());
         when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations/360857467055/users.json?per_page=100&page=1"), eq(task), eq(false)))
                 .thenReturn(dataJsonUser.toString());
-        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/sunshine/events?identifier=support%3Auser_id%3A1194092277&start_time=2019-01-20T07%3A14%3A50Z&end_time=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/users/1194092277/events?filter%5Bstart_time%5D=2019-01-20T07%3A14%3A50Z&filter%5Bend_time%5D=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
                 .thenReturn(dataJsonUserEvent.toString());
 
         zendeskUserEventService.addRecordToImporter(0, recordImporter);
@@ -131,7 +132,7 @@ public void testAddRecordToImporterWithDuplicateUser()
                 .thenReturn(dataJsonUser.toString());
         when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations/360857467055/users.json?per_page=100&page=1"), eq(task), eq(false)))
                 .thenReturn(dataJsonUser.toString());
-        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/sunshine/events?identifier=support%3Auser_id%3A1194092277&start_time=2019-01-20T07%3A14%3A50Z&end_time=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/users/1194092277/events?filter%5Bstart_time%5D=2019-01-20T07%3A14%3A50Z&filter%5Bend_time%5D=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
                 .thenReturn(dataJsonUserEvent.toString());
 
         zendeskUserEventService.addRecordToImporter(0, recordImporter);

src/test/java/org/embulk/input/zendesk/utils/TestZendeskDateUtils.java

@@ -32,6 +32,9 @@ public void testIsoToEpochSecondShouldReturnCorrectValue()
 
         value = ZendeskDateUtils.isoToEpochSecond("2019-02-20 06:50:45+0000");
         Assert.assertEquals(expectedValue, value);
+
+        value = ZendeskDateUtils.isoToEpochSecond("2019-02-20T06:50:45.215149154Z");
+        Assert.assertEquals(expectedValue, value);
     }
 
     @Test

src/test/resources/config/user_events.yml

@@ -14,7 +14,6 @@ retry_initial_wait_sec: 1
 dedup: false
 max_retry_wait_sec: 30
 retry_limit: 2
-profile_source: support
 start_time: '2019-01-20T07:14:50Z'
 end_time: '2019-06-20T07:14:53Z'
 columns:

src/test/resources/data/user_event.json

@@ -1,13 +1,13 @@
 {
-  "data": [
+  "events": [
     {
       "id": "5c7f31aef8df240001e60bbf",
       "type": "remove_from_cart",
       "source": "shopify",
       "description": "",
       "authenticated": true,
-      "created_at": "2019-03-06T02:34:22Z",
-      "received_at": "2019-03-06T02:34:22Z",
+      "created_at": "2019-03-06T02:34:22.381847424Z",
+      "received_at": "2019-03-06T02:34:22.381847424Z",
       "properties": {
         "model": 221,
         "size": 6