Mapsforge: multilingual vector map files v4, closes #34

vtm-android-example/src/org/oscim/android/test/MapsforgeMapActivity.java

@@ -91,6 +91,7 @@ protected void onActivityResult(int requestCode, int resultCode, Intent intent)
 				return;
 
 			MapFileTileSource tileSource = new MapFileTileSource();
+			tileSource.setPreferredLanguage("en");
 			String file = intent.getStringExtra(FilePicker.SELECTED_FILE);
 			if (tileSource.setMapFile(file)) {
 
@@ -104,6 +105,7 @@ protected void onActivityResult(int requestCode, int resultCode, Intent intent)
 				MapPosition pos = new MapPosition();
 				pos.setByBoundingBox(info.boundingBox, Tile.SIZE * 4, Tile.SIZE * 4);
 				mMap.setMapPosition(pos);
+
 				mPrefs.clear();
 			}
 		}

vtm-playground/src/org/oscim/test/MapsforgeTest.java

@@ -33,6 +33,7 @@ public class MapsforgeTest extends GdxMap {
 	public void createLayers() {
 		MapFileTileSource tileSource = new MapFileTileSource();
 		tileSource.setMapFile(System.getProperty("user.home") + "/Downloads/berlin.map");
+		tileSource.setPreferredLanguage("en");
 
 		VectorTileLayer l = mMap.setBaseMap(tileSource);
 		mMap.setTheme(VtmThemes.DEFAULT);

vtm/src/org/oscim/tiling/source/mapfile/MapDatabase.java

@@ -540,7 +540,7 @@ private boolean processPOIs(ITileDataSink mapDataSink, int numberOfPois) {
 			/* bit 1-3 enable optional features
 			 * check if the POI has a name */
 			if ((featureByte & POI_FEATURE_NAME) != 0) {
-				String str = mReadBuffer.readUTF8EncodedString();
+				String str = mTileSource.extractLocalized(mReadBuffer.readUTF8EncodedString());
 				e.tags.add(new Tag(Tag.KEY_NAME, str, false));
 			}
 
@@ -769,7 +769,7 @@ private boolean processWays(QueryParameters queryParameters,
 			if (mTileSource.experimental) {
 				if (hasName) {
 					int textPos = mReadBuffer.readUnsignedInt();
-					String str = mReadBuffer.readUTF8EncodedStringAt(stringOffset + textPos);
+					String str = mTileSource.extractLocalized(mReadBuffer.readUTF8EncodedStringAt(stringOffset + textPos));
 					e.tags.add(new Tag(Tag.KEY_NAME, str, false));
 				}
 				if (hasHouseNr) {
@@ -784,7 +784,7 @@ private boolean processWays(QueryParameters queryParameters,
 				}
 			} else {
 				if (hasName) {
-					String str = mReadBuffer.readUTF8EncodedString();
+					String str = mTileSource.extractLocalized(mReadBuffer.readUTF8EncodedString());
 					e.tags.add(new Tag(Tag.KEY_NAME, str, false));
 				}
 				if (hasHouseNr) {

vtm/src/org/oscim/tiling/source/mapfile/MapFileTileSource.java

@@ -1,6 +1,7 @@
 /*
- * Copyright 2013 Hannes Janetzek
  * Copyright 2013 mapsforge.org
+ * Copyright 2013 Hannes Janetzek
+ * Copyright 2016 devemux86
  *
  * This file is part of the OpenScienceMap project (http://www.opensciencemap.org).
  *
@@ -45,10 +46,30 @@ public class MapFileTileSource extends TileSource {
 	File mapFile;
 	RandomAccessFile mInputFile;
 
+	/**
+	 * The preferred language when extracting labels from this tile source.
+	 */
+	private String preferredLanguage;
+	private Callback callback;
+
 	public MapFileTileSource() {
 		super(0, 17);
 	}
 
+	/**
+	 * Extracts substring of preferred language from multilingual string using
+	 * the preferredLanguage setting.
+	 */
+	String extractLocalized(String s) {
+		if (callback != null)
+			return callback.extractLocalized(s);
+		return MapFileUtils.extract(s, preferredLanguage);
+	}
+
+	public void setCallback(Callback callback) {
+		this.callback = callback;
+	}
+
 	public boolean setMapFile(String filename) {
 		setOption("file", filename);
 
@@ -65,6 +86,10 @@ public boolean setMapFile(String filename) {
 		return true;
 	}
 
+	public void setPreferredLanguage(String preferredLanguage) {
+		this.preferredLanguage = preferredLanguage;
+	}
+
 	@Override
 	public OpenResult open() {
 		if (!options.containsKey("file"))
@@ -101,7 +126,8 @@ public OpenResult open() {
 			mapFile = file;
 			databaseIndexCache = new IndexCache(mInputFile, INDEX_CACHE_SIZE);
 
-			experimental = fileInfo.fileVersion == 4;
+			// Experimental?
+			//experimental = fileInfo.fileVersion == 4;
 
 			log.debug("File version: " + fileInfo.fileVersion);
 			return OpenResult.SUCCESS;
@@ -141,4 +167,10 @@ public MapInfo getMapInfo() {
 		return fileInfo;
 	}
 
+	public interface Callback {
+		/**
+		 * Extracts substring of preferred language from multilingual string.
+		 */
+		String extractLocalized(String s);
+	}
 }

vtm/src/org/oscim/tiling/source/mapfile/MapFileUtils.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2016 devemux86
+ *
+ * This program is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Lesser General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.oscim.tiling.source.mapfile;
+
+import java.util.Locale;
+
+public final class MapFileUtils {
+
+    /**
+     * Extracts substring of preferred language from multilingual string.<br/>
+     * Example multilingual string: "Base\ren\bEnglish\rjp\bJapan\rzh_py\bPin-yin".
+     * <p/>
+     * Use '\r' delimiter among names and '\b' delimiter between each language and name.
+     */
+    public static String extract(String s, String language) {
+        if (s == null || s.trim().isEmpty()) {
+            return null;
+        }
+
+        String[] langNames = s.split("\r");
+        if (language == null || language.trim().isEmpty()) {
+            return langNames[0];
+        }
+
+        String fallback = null;
+        for (int i = 1; i < langNames.length; i++) {
+            String[] langName = langNames[i].split("\b");
+            if (langName.length != 2) {
+                continue;
+            }
+
+            // Perfect match
+            if (langName[0].equalsIgnoreCase(language)) {
+                return langName[1];
+            }
+
+            // Fall back to base, e.g. zh-min-lan -> zh
+            if (fallback == null && !langName[0].contains("-") && (language.contains("-") || language.contains("_"))
+                    && language.toLowerCase(Locale.ENGLISH).startsWith(langName[0].toLowerCase(Locale.ENGLISH))) {
+                fallback = langName[1];
+            }
+        }
+        return (fallback != null) ? fallback : langNames[0];
+    }
+
+    private MapFileUtils() {
+    }
+}

vtm/src/org/oscim/tiling/source/mapfile/MapInfo.java

@@ -1,5 +1,6 @@
 /*
  * Copyright 2010, 2011, 2012 mapsforge.org
+ * Copyright 2016 devemux86
  *
  * This file is part of the OpenScienceMap project (http://www.opensciencemap.org).
  *
@@ -50,9 +51,9 @@ public class MapInfo {
 	public final int fileVersion;
 
 	/**
-	 * The preferred language for names as defined in ISO 3166-1 (may be null).
+	 * The preferred language(s) separated with ',' for names as defined in ISO 639-1 or ISO 639-2 (may be null).
 	 */
-	public final String languagePreference;
+	public final String languagesPreference;
 
 	/**
 	 * The center point of the map file.
@@ -120,7 +121,7 @@ public MapInfo(BoundingBox bbox, Byte zoom, GeoPoint start, String projection,
 		this.mapDate = date;
 		this.boundingBox = bbox;
 		this.mapCenter = bbox.getCenterPoint();
-		this.languagePreference = language;
+		this.languagesPreference = language;
 		this.fileSize = size;
 		this.fileVersion = version;
 

vtm/src/org/oscim/tiling/source/mapfile/header/MapFileInfo.java

@@ -1,5 +1,6 @@
 /*
  * Copyright 2010, 2011, 2012 mapsforge.org
+ * Copyright 2016 devemux86
  *
  * This file is part of the OpenScienceMap project (http://www.opensciencemap.org).
  *
@@ -57,7 +58,7 @@ public class MapFileInfo extends org.oscim.tiling.source.mapfile.MapInfo {
 		      mapFileInfoBuilder.mapDate,
 		      mapFileInfoBuilder.fileSize,
 		      mapFileInfoBuilder.fileVersion,
-		      mapFileInfoBuilder.optionalFields.languagePreference,
+		      mapFileInfoBuilder.optionalFields.languagesPreference,
 		      mapFileInfoBuilder.optionalFields.comment,
 		      mapFileInfoBuilder.optionalFields.createdBy,
 		      mapFileInfoBuilder.zoomLevel);

vtm/src/org/oscim/tiling/source/mapfile/header/OptionalFields.java

@@ -1,5 +1,6 @@
 /*
  * Copyright 2010, 2011, 2012 mapsforge.org
+ * Copyright 2016 devemux86
  *
  * This file is part of the OpenScienceMap project (http://www.opensciencemap.org).
  *
@@ -37,9 +38,9 @@ final class OptionalFields {
 	private static final int HEADER_BITMASK_DEBUG = 0x80;
 
 	/**
-	 * Bitmask for the language preference field in the file header.
+	 * Bitmask for the language(s) preference field in the file header.
 	 */
-	private static final int HEADER_BITMASK_LANGUAGE_PREFERENCE = 0x10;
+	private static final int HEADER_BITMASK_LANGUAGES_PREFERENCE = 0x10;
 
 	/**
 	 * Bitmask for the start position field in the file header.
@@ -51,11 +52,6 @@ final class OptionalFields {
 	 */
 	private static final int HEADER_BITMASK_START_ZOOM_LEVEL = 0x20;
 
-	/**
-	 * The length of the language preference string.
-	 */
-	private static final int LANGUAGE_PREFERENCE_LENGTH = 2;
-
 	/**
 	 * Maximum valid start zoom level.
 	 */
@@ -77,30 +73,26 @@ static OpenResult readOptionalFields(ReadBuffer readBuffer,
 	String createdBy;
 	final boolean hasComment;
 	final boolean hasCreatedBy;
-	final boolean hasLanguagePreference;
+	final boolean hasLanguagesPreference;
 	final boolean hasStartPosition;
 	final boolean hasStartZoomLevel;
 	final boolean isDebugFile;
-	String languagePreference;
+	String languagesPreference;
 	GeoPoint startPosition;
 	Byte startZoomLevel;
 
 	private OptionalFields(byte flags) {
 		this.isDebugFile = (flags & HEADER_BITMASK_DEBUG) != 0;
 		this.hasStartPosition = (flags & HEADER_BITMASK_START_POSITION) != 0;
 		this.hasStartZoomLevel = (flags & HEADER_BITMASK_START_ZOOM_LEVEL) != 0;
-		this.hasLanguagePreference = (flags & HEADER_BITMASK_LANGUAGE_PREFERENCE) != 0;
+		this.hasLanguagesPreference = (flags & HEADER_BITMASK_LANGUAGES_PREFERENCE) != 0;
 		this.hasComment = (flags & HEADER_BITMASK_COMMENT) != 0;
 		this.hasCreatedBy = (flags & HEADER_BITMASK_CREATED_BY) != 0;
 	}
 
-	private OpenResult readLanguagePreference(ReadBuffer readBuffer) {
-		if (this.hasLanguagePreference) {
-			String countryCode = readBuffer.readUTF8EncodedString();
-			if (countryCode.length() != LANGUAGE_PREFERENCE_LENGTH) {
-				return new OpenResult("invalid language preference: " + countryCode);
-			}
-			this.languagePreference = countryCode;
+	private OpenResult readLanguagesPreference(ReadBuffer readBuffer) {
+		if (this.hasLanguagesPreference) {
+			this.languagesPreference = readBuffer.readUTF8EncodedString();
 		}
 		return OpenResult.SUCCESS;
 	}
@@ -150,7 +142,7 @@ private OpenResult readOptionalFields(ReadBuffer readBuffer) {
 			return openResult;
 		}
 
-		openResult = readLanguagePreference(readBuffer);
+		openResult = readLanguagesPreference(readBuffer);
 		if (!openResult.isSuccess()) {
 			return openResult;
 		}

vtm/src/org/oscim/tiling/source/mapfile/header/RequiredFields.java

@@ -1,5 +1,6 @@
 /*
  * Copyright 2010, 2011, 2012 mapsforge.org
+ * Copyright 2016 devemux86
  *
  * This file is part of the OpenScienceMap project (http://www.opensciencemap.org).
  *
@@ -50,10 +51,14 @@ final class RequiredFields {
 	private static final char SPACE = ' ';
 
 	/**
-	 * Version of the map file format which is supported by this implementation.
+	 * Lowest version of the map file format supported by this implementation.
 	 */
-	private static final int FILE_VERSION_3 = 3;
-	private static final int FILE_VERSION_4 = 4;
+	private static final int SUPPORTED_FILE_VERSION_MIN = 3;
+
+	/**
+	 * Highest version of the map file format supported by this implementation.
+	 */
+	private static final int SUPPORTED_FILE_VERSION_MAX = 4;
 
 	/**
 	 * The maximum latitude values in microdegrees.
@@ -126,7 +131,7 @@ static OpenResult readFileSize(ReadBuffer readBuffer, long fileSize,
 	static OpenResult readFileVersion(ReadBuffer readBuffer, MapFileInfoBuilder mapFileInfoBuilder) {
 		// get and check the file version (4 bytes)
 		int fileVersion = readBuffer.readInt();
-		if (fileVersion != FILE_VERSION_3 && fileVersion != FILE_VERSION_4) {
+		if (fileVersion < SUPPORTED_FILE_VERSION_MIN || fileVersion > SUPPORTED_FILE_VERSION_MAX) {
 			return new OpenResult("unsupported file version: " + fileVersion);
 		}
 		mapFileInfoBuilder.fileVersion = fileVersion;