Skip to content

Commit

Permalink
Merge branch 'master' into feat/203
Browse files Browse the repository at this point in the history
  • Loading branch information
cka-y authored Feb 17, 2025
2 parents 61993dc + 2b3e0e5 commit 5600edf
Show file tree
Hide file tree
Showing 9 changed files with 279 additions and 37 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -49,26 +49,20 @@ public abstract class GtfsInput implements Closeable {
*/
public static GtfsInput createFromPath(Path path, NoticeContainer noticeContainer)
throws IOException {
ZipFile zipFile;
if (!Files.exists(path)) {
throw new FileNotFoundException(path.toString());
}
if (Files.isDirectory(path)) {
return new GtfsUnarchivedInput(path);
}
String fileName = path.getFileName().toString().replace(".zip", "");
if (path.getFileSystem().equals(FileSystems.getDefault())) {
// Read from a local ZIP file.
zipFile = new ZipFile(path.toFile());
if (hasSubfolderWithGtfsFile(path)) {
noticeContainer.addValidationNotice(
new InvalidInputFilesInSubfolderNotice(invalidInputMessage));
}
ZipFile zipFile =
path.getFileSystem().equals(FileSystems.getDefault())
// Read from a local ZIP file.
? new ZipFile(path.toFile())
// Load a remote ZIP file to memory.
: new ZipFile(new SeekableInMemoryByteChannel(Files.readAllBytes(path)));

return new GtfsZipFileInput(zipFile, fileName);
}
// Load a remote ZIP file to memory.
zipFile = new ZipFile(new SeekableInMemoryByteChannel(Files.readAllBytes(path)));
if (hasSubfolderWithGtfsFile(path)) {
noticeContainer.addValidationNotice(
new InvalidInputFilesInSubfolderNotice(invalidInputMessage));
Expand Down Expand Up @@ -98,23 +92,13 @@ public static boolean hasSubfolderWithGtfsFile(Path path) throws IOException {
*/
private static boolean containsGtfsFileInSubfolder(ZipInputStream zipInputStream)
throws IOException {
boolean containsSubfolder = false;
String subfolder = null;
ZipEntry entry;
while ((entry = zipInputStream.getNextEntry()) != null) {
String entryName = entry.getName();

if (entry.isDirectory()) {
subfolder = entryName;
containsSubfolder = true;
}
if (containsSubfolder && entryName.contains(subfolder) && entryName.endsWith(".txt")) {
String[] files = entryName.split("/");
String lastElement = files[files.length - 1];

if (GtfsFiles.containsGtfsFile(lastElement)) {
return true;
}
String[] nameParts = entry.getName().split("/");
boolean isInSubfolder = nameParts.length > 1;
boolean isGtfsFile = GtfsFiles.containsGtfsFile(nameParts[nameParts.length - 1]);
if (isInSubfolder && isGtfsFile) {
return true;
}
}
return false;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package org.mobilitydata.gtfsvalidator.notice;

import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR;

import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice;

/** Duplicated elements in locations.geojson file. */
@GtfsValidationNotice(severity = ERROR)
public class GeoJsonDuplicatedElementNotice extends ValidationNotice {
/** The name of the file where the duplicated element was found. */
private final String filename;

/** The duplicated element in the GeoJSON file. */
private final String duplicatedElement;

public GeoJsonDuplicatedElementNotice(String filename, String duplicatedElement) {
this.filename = filename;
this.duplicatedElement = duplicatedElement;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package org.mobilitydata.gtfsvalidator.notice;

import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.INFO;

import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice;

/** Unknown elements in locations.geojson file. */
@GtfsValidationNotice(severity = INFO)
public class GeoJsonUnknownElementNotice extends ValidationNotice {
/** The name of the file where the unknown element was found. */
private final String filename;

/** The unknown element in the GeoJSON file. */
private final String unknownElement;

public GeoJsonUnknownElementNotice(String filename, String unknownElement) {
this.filename = filename;
this.unknownElement = unknownElement;
}
}
Original file line number Diff line number Diff line change
@@ -1,21 +1,18 @@
package org.mobilitydata.gtfsvalidator.table;

import com.google.common.flogger.FluentLogger;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.locationtech.jts.geom.*;
import org.mobilitydata.gtfsvalidator.notice.*;
import org.mobilitydata.gtfsvalidator.util.geojson.GeoJsonGeometryValidator;
import org.mobilitydata.gtfsvalidator.util.geojson.GeometryType;
import org.mobilitydata.gtfsvalidator.util.geojson.UnparsableGeoJsonFeatureException;
import org.mobilitydata.gtfsvalidator.notice.GeoJsonDuplicatedElementNotice;
import org.mobilitydata.gtfsvalidator.util.geojson.*;
import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider;

/**
Expand Down Expand Up @@ -70,8 +67,27 @@ public List<GtfsGeoJsonFeature> extractFeaturesFromStream(
throws IOException, UnparsableGeoJsonFeatureException {
List<GtfsGeoJsonFeature> features = new ArrayList<>();
boolean hasUnparsableFeature = false;
GsonBuilder gsonBuilder = new GsonBuilder();
// Using the MapJsonTypeAdapter to be able to parse JSON objects with duplicate keys and
// unsupported Gson library features
gsonBuilder.registerTypeAdapter(
new TypeToken<Map<String, Object>>() {}.getType(), new MapJsonTypeAdapter());
Gson gson = gsonBuilder.create();

try (InputStreamReader reader = new InputStreamReader(inputStream)) {
JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject();
JsonElement root =
gson.toJsonTree(gson.fromJson(reader, new TypeToken<Map<String, Object>>() {}.getType()));
if (!root.isJsonObject()) {
throw new JsonParseException("Expected a JSON object at the root");
}
JsonObject jsonObject = root.getAsJsonObject();
for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) {
String key = entry.getKey();
if (!"type".equals(key) && !"features".equals(key)) {
noticeContainer.addValidationNotice(
new GeoJsonUnknownElementNotice(GtfsGeoJsonFeature.FILENAME, key));
}
}
if (!jsonObject.has("type")) {
noticeContainer.addValidationNotice(new MissingRequiredElementNotice(null, "type", null));
throw new UnparsableGeoJsonFeatureException("Missing required field 'type'");
Expand All @@ -93,6 +109,9 @@ public List<GtfsGeoJsonFeature> extractFeaturesFromStream(
features.add(gtfsGeoJsonFeature);
}
}
} catch (DuplicateJsonKeyException exception) {
noticeContainer.addValidationNotice(
new GeoJsonDuplicatedElementNotice(GtfsGeoJsonFeature.FILENAME, exception.getKey()));
}
if (hasUnparsableFeature) {
throw new UnparsableGeoJsonFeatureException("Unparsable GeoJSON feature");
Expand All @@ -107,6 +126,19 @@ public GtfsGeoJsonFeature extractFeature(
String featureId = null;
if (feature.isJsonObject()) {
JsonObject featureObject = feature.getAsJsonObject();

// Check for unknown elements in the featureObject
for (Map.Entry<String, JsonElement> entry : featureObject.entrySet()) {
String key = entry.getKey();
if (!GtfsGeoJsonFeature.FEATURE_ID_FIELD_NAME.equals(key)
&& !GtfsGeoJsonFeature.FEATURE_TYPE_FIELD_NAME.equals(key)
&& !GtfsGeoJsonFeature.FEATURE_PROPERTIES_FIELD_NAME.equals(key)
&& !GtfsGeoJsonFeature.GEOMETRY_FIELD_NAME.equals(key)) {
noticeContainer.addValidationNotice(
new GeoJsonUnknownElementNotice(GtfsGeoJsonFeature.FILENAME, key));
}
}

// Handle feature id
if (!featureObject.has(GtfsGeoJsonFeature.FEATURE_ID_FIELD_NAME)) {
missingRequiredFields.add(
Expand Down Expand Up @@ -156,6 +188,15 @@ public GtfsGeoJsonFeature extractFeature(
+ GtfsGeoJsonFeature.GEOMETRY_FIELD_NAME);
} else {
JsonObject geometry = featureObject.getAsJsonObject(GtfsGeoJsonFeature.GEOMETRY_FIELD_NAME);
// Check for unknown elements in the geometry object
for (Map.Entry<String, JsonElement> entry : geometry.entrySet()) {
String key = entry.getKey();
if (!GtfsGeoJsonFeature.GEOMETRY_TYPE_FIELD_NAME.equals(key)
&& !GtfsGeoJsonFeature.GEOMETRY_COORDINATES_FIELD_NAME.equals(key)) {
noticeContainer.addValidationNotice(
new GeoJsonUnknownElementNotice(GtfsGeoJsonFeature.FILENAME, key));
}
}
// Handle geometry type and coordinates
if (!geometry.has(GtfsGeoJsonFeature.GEOMETRY_TYPE_FIELD_NAME)) {
missingRequiredFields.add(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package org.mobilitydata.gtfsvalidator.util.geojson;

public class DuplicateJsonKeyException extends RuntimeException {
private String key;
private String message;

public DuplicateJsonKeyException(String key, String message) {
this.key = key;
this.message = message;
}

public String getKey() {
return key;
}

public String getMessage() {
return message;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package org.mobilitydata.gtfsvalidator.util.geojson;

import com.google.gson.*;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;

/**
* A custom JSON type adapter for parsing JSON objects with duplicate keys. The target class is
* {@link Map}{@code <String, Object>}. as JSonElement is captured by the default Gson TypeAdapter.
*
* <p>When a JSON object has two keys with the same name at the same level, this type adapter throws
* a {@link DuplicateJsonKeyException}.
*/
public class MapJsonTypeAdapter extends TypeAdapter<Map<String, Object>> {

@Override
public void write(JsonWriter out, Map<String, Object> value) throws IOException {
new Gson().toJson(value, Map.class, out);
}

@Override
public Map<String, Object> read(JsonReader in) throws IOException {
return parseJsonObject(in);
}

private Map<String, Object> parseJsonObject(JsonReader in) throws IOException {
Map<String, Object> map = new LinkedHashMap<>();

in.beginObject();
while (in.hasNext()) {
String key = in.nextName();

if (map.containsKey(key)) {
throw new DuplicateJsonKeyException(key, "Duplicated Key: " + key);
}

Object value = parseJsonValue(in);
map.put(key, value);
}
in.endObject();

return map;
}

private Object parseJsonValue(JsonReader in) throws IOException {
switch (in.peek()) {
case BEGIN_OBJECT:
return parseJsonObject(in);
case BEGIN_ARRAY:
return parseJsonArray(in);
case STRING:
return in.nextString();
case NUMBER:
return in.nextDouble();
case BOOLEAN:
return in.nextBoolean();
case NULL:
in.nextNull();
return null;
default:
throw new JsonParseException("Unexpected JSON token: " + in.peek());
}
}

private Object parseJsonArray(JsonReader in) throws IOException {
JsonArray jsonArray = new JsonArray();
in.beginArray();
while (in.hasNext()) {
jsonArray.add(new Gson().toJsonTree(parseJsonValue(in)));
}
in.endArray();
return jsonArray;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package org.mobilitydata.gtfsvalidator.util.geojson;

public class UnknownJsonKeyException extends RuntimeException {
private final String message;
private String key;

public UnknownJsonKeyException(String key, String message) {
this.key = key;
this.message = message;
}

public String getKey() {
return key;
}

public String getMessage() {
return message;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package org.mobilitydata.gtfsvalidator.util.geojson;

import static org.junit.Assert.assertThrows;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import java.io.StringReader;
import java.util.Map;
import org.junit.Before;
import org.junit.Test;

public class GeoJsonTypeAdapterTest {

Gson gson;

@Before
public void before() {
gson =
(new GsonBuilder())
.registerTypeAdapter(
new TypeToken<Map<String, Object>>() {}.getType(), new MapJsonTypeAdapter())
.create();
}

/**
* Test that the custom JSON type adapter can handle a throws DuplicateJsonKeyException when: - A
* JSON object has two keys with the same name at the same level.
*/
@Test
public void testDuplicateKeyExceptionSameLevel() {
final var json = "{ \"type\": 1, \"type\": 2 }";
JsonReader reader = new JsonReader(new StringReader(json));
MapJsonTypeAdapter adapter = new MapJsonTypeAdapter();
assertThrows(
DuplicateJsonKeyException.class,
() -> {
adapter.read(reader);
});
}

/**
* Test that the custom JSON type adapter can handle a simple JSON object that don't contain
* duplicate keys.
*/
@Test
public void testDuplicateKeyExceptionNestedLevel() {
String json =
"{\"type\": \"Alice\", \"features\": { \"properties\": \"Bob\", \"properties\": \"abc\" }}";
JsonReader reader = new JsonReader(new StringReader(json));
MapJsonTypeAdapter adapter = new MapJsonTypeAdapter();

assertThrows(
DuplicateJsonKeyException.class,
() -> {
adapter.read(reader);
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,9 @@ public void testNoticeClassFieldNames() {
"tripIdB",
"tripIdFieldName",
"validator",
"value");
"value",
"duplicatedElement",
"unknownElement");
}

private static List<String> discoverValidationNoticeFieldNames() {
Expand Down

0 comments on commit 5600edf

Please sign in to comment.