JAVA: Add support for loading short number metadata

debian/changelog

@@ -1,3 +1,11 @@
+libphonenumber (5.7) precise; urgency=low
+
+  * Code changes:
+    - Improve phone number extraction recall.
+    - Add support for loading short number metadata.
+
+ -- David Yonge-Mallo <davinci@google.com>  Wed, 17 Jul 2013 11:25:45 +0200
+
 libphonenumber (5.6.2) precise; urgency=low
 
   * Metadata changes:
@@ -10,14 +18,14 @@ libphonenumber (5.6.2) precise; urgency=low
    - Updated geocoding data for country calling code(s):
      49 (en), 98 (en, fa), 241 (en)
 
- -- Lara Rennie <lararennie@google.com>  Tue, Jul 16 2013 10:01:53 +0200
+ -- Lara Rennie <lararennie@google.com>  Tue, 16 Jul 2013 10:01:53 +0200
 
 libphonenumber (5.6.1) precise; urgency=low
 
   * Code changes:
    - Rename all instances of PhoneNumberMetadata to PhoneNumberMetadata for consistency.
 
- -- David Yonge-Mallo <davinci@google.com>  Fri, Jul 12 2013 10:26:53 +0200
+ -- David Yonge-Mallo <davinci@google.com>  Fri, 12 Jul 2013 10:26:53 +0200
 
 libphonenumber (5.6) precise; urgency=low
 

java/build.xml

@@ -27,7 +27,7 @@
     </fileset>
   </path>
 
-  <target name="build-metadata">
+  <target name="build-phone-metadata">
     <exec executable="java">
       <arg value="-jar" />
       <arg value="${build.tools.jar}"/>
@@ -129,7 +129,7 @@
   </target>
 
   <target name="compile" description="Compile Java source."
-          depends="build-metadata,build-alternate-metadata,build-geo-data">
+          depends="build-phone-metadata,build-short-metadata,build-alternate-metadata,build-geo-data">
     <mkdir dir="${classes.dir}"/>
     <javac srcdir="${libphonenumber.src.dir};${geocoder.src.dir}"
            destdir="${classes.dir}" classpathref="classpath" includeAntRuntime="false"/>
@@ -150,6 +150,7 @@
       <fileset dir="${libphonenumber.src.dir}">
         <include name="**/PhoneNumberMetadataProto*"/>
         <include name="**/PhoneNumberAlternateFormatsProto*"/>
+        <include name="**/ShortNumberMetadataProto*"/>
       </fileset>
     </jar>
     <jar destfile="${jar.dir}/offline-geocoder.jar">
@@ -175,6 +176,7 @@
       <fileset dir="${libphonenumber.src.dir}">
         <include name="**/PhoneNumberMetadataProto*"/>
         <include name="**/PhoneNumberAlternateFormatsProto*"/>
+        <include name="**/ShortNumberMetadataProto*"/>
       </fileset>
       <fileset dir="${libphonenumber.test.dir}">
         <include name="**/PhoneNumberMetadataProtoForTesting*"/>

java/libphonenumber/src/com/google/i18n/phonenumbers/MetadataManager.java

@@ -25,6 +25,7 @@ import java.io.ObjectInputStream;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.logging.Level;
@@ -40,17 +41,25 @@ import java.util.logging.Logger;
 class MetadataManager {
   private static final String ALTERNATE_FORMATS_FILE_PREFIX =
       "/com/google/i18n/phonenumbers/data/PhoneNumberAlternateFormatsProto";
+  private static final String SHORT_NUMBER_METADATA_FILE_PREFIX =
+      "/com/google/i18n/phonenumbers/data/ShortNumberMetadataProto";
 
   private static final Logger LOGGER = Logger.getLogger(MetadataManager.class.getName());
 
   private static final Map<Integer, PhoneMetadata> callingCodeToAlternateFormatsMap =
       Collections.synchronizedMap(new HashMap<Integer, PhoneMetadata>());
+  private static final Map<String, PhoneMetadata> regionCodeToShortNumberMetadataMap =
+      Collections.synchronizedMap(new HashMap<String, PhoneMetadata>());
 
   // A set of which country calling codes there are alternate format data for. If the set has an
   // entry for a code, then there should be data for that code linked into the resources.
   private static final Set<Integer> countryCodeSet =
       AlternateFormatsCountryCodeSet.getCountryCodeSet();
 
+  // A set of which region codes there are short number data for. If the set has an entry for a
+  // code, then there should be data for that code linked into the resources.
+  private static final Set<String> regionCodeSet = ShortNumbersRegionCodeSet.getRegionCodeSet();
+
   private MetadataManager() {
   }
 
@@ -64,7 +73,7 @@ class MetadataManager {
     }
   }
 
-  private static void loadMetadataFromFile(int countryCallingCode) {
+  private static void loadAlternateFormatsMetadataFromFile(int countryCallingCode) {
     InputStream source = PhoneNumberMatcher.class.getResourceAsStream(
         ALTERNATE_FORMATS_FILE_PREFIX + "_" + countryCallingCode);
     ObjectInputStream in = null;
@@ -88,9 +97,44 @@ class MetadataManager {
     }
     synchronized (callingCodeToAlternateFormatsMap) {
       if (!callingCodeToAlternateFormatsMap.containsKey(countryCallingCode)) {
-        loadMetadataFromFile(countryCallingCode);
+        loadAlternateFormatsMetadataFromFile(countryCallingCode);
       }
     }
     return callingCodeToAlternateFormatsMap.get(countryCallingCode);
   }
+
+  private static void loadShortNumberMetadataFromFile(String regionCode) {
+    InputStream source = PhoneNumberMatcher.class.getResourceAsStream(
+        SHORT_NUMBER_METADATA_FILE_PREFIX + "_" + regionCode);
+    ObjectInputStream in = null;
+    try {
+      in = new ObjectInputStream(source);
+      PhoneMetadataCollection shortNumberMetadata = new PhoneMetadataCollection();
+      shortNumberMetadata.readExternal(in);
+      for (PhoneMetadata metadata : shortNumberMetadata.getMetadataList()) {
+        regionCodeToShortNumberMetadataMap.put(regionCode, metadata);
+      }
+    } catch (IOException e) {
+      LOGGER.log(Level.WARNING, e.toString());
+    } finally {
+      close(in);
+    }
+  }
+
+  static PhoneMetadata getShortNumberMetadataForRegion(String regionCode) {
+    if (!regionCodeSet.contains(regionCode)) {
+      return null;
+    }
+    synchronized (regionCodeToShortNumberMetadataMap) {
+      if (!regionCodeToShortNumberMetadataMap.containsKey(regionCode)) {
+        loadShortNumberMetadataFromFile(regionCode);
+      }
+    }
+    return regionCodeToShortNumberMetadataMap.get(regionCode);
+  }
+
+  // @VisibleForTesting
+  static Set<String> getShortNumberMetadataSupportedRegions() {
+    return regionCodeSet;
+  }
 }

java/libphonenumber/src/com/google/i18n/phonenumbers/PhoneNumberMatcher.java

@@ -568,9 +568,30 @@ final class PhoneNumberMatcher implements Iterator<PhoneNumberMatch> {
     return false;
   }
 
-  static boolean containsMoreThanOneSlash(String candidate) {
-    int firstSlashIndex = candidate.indexOf('/');
-    return (firstSlashIndex > 0 && candidate.substring(firstSlashIndex + 1).contains("/"));
+  static boolean containsMoreThanOneSlashInNationalNumber(PhoneNumber number, String candidate) {
+    int firstSlashInBodyIndex = candidate.indexOf('/');
+    if (firstSlashInBodyIndex < 0) {
+      // No slashes, this is okay.
+      return false;
+    }
+    // Now look for a second one.
+    int secondSlashInBodyIndex = candidate.indexOf('/', firstSlashInBodyIndex + 1);
+    if (secondSlashInBodyIndex < 0) {
+      // Only one slash, this is okay.
+      return false;
+    }
+
+    // If the first slash is after the country calling code, this is permitted.
+    boolean candidateHasCountryCode =
+        (number.getCountryCodeSource() == CountryCodeSource.FROM_NUMBER_WITH_PLUS_SIGN ||
+         number.getCountryCodeSource() == CountryCodeSource.FROM_NUMBER_WITHOUT_PLUS_SIGN);
+    if (candidateHasCountryCode &&
+        PhoneNumberUtil.normalizeDigitsOnly(candidate.substring(0, firstSlashInBodyIndex))
+            .equals(Integer.toString(number.getCountryCode()))) {
+      // Any more slashes and this is illegal.
+      return candidate.substring(secondSlashInBodyIndex + 1).contains("/");
+    }
+    return true;
   }
 
   static boolean containsOnlyValidXChars(

java/libphonenumber/src/com/google/i18n/phonenumbers/PhoneNumberUtil.java

@@ -451,7 +451,8 @@ public class PhoneNumberUtil {
      * are grouped in a possible way for this locale. For example, a US number written as
      * "65 02 53 00 00" and "650253 0000" are not accepted at this leniency level, whereas
      * "650 253 0000", "650 2530000" or "6502530000" are.
-     * Numbers with more than one '/' symbol are also dropped at this level.
+     * Numbers with more than one '/' symbol in the national significant number are also dropped at
+     * this level.
      * <p>
      * Warning: This level might result in lower coverage especially for regions outside of country
      * code "+1". If you are not sure about which level to use, email the discussion group
@@ -462,7 +463,7 @@ public class PhoneNumberUtil {
       boolean verify(PhoneNumber number, String candidate, PhoneNumberUtil util) {
         if (!util.isValidNumber(number) ||
             !PhoneNumberMatcher.containsOnlyValidXChars(number, candidate, util) ||
-            PhoneNumberMatcher.containsMoreThanOneSlash(candidate) ||
+            PhoneNumberMatcher.containsMoreThanOneSlashInNationalNumber(number, candidate) ||
             !PhoneNumberMatcher.isNationalPrefixPresentIfRequired(number, util)) {
           return false;
         }
@@ -493,7 +494,7 @@ public class PhoneNumberUtil {
       boolean verify(PhoneNumber number, String candidate, PhoneNumberUtil util) {
         if (!util.isValidNumber(number) ||
             !PhoneNumberMatcher.containsOnlyValidXChars(number, candidate, util) ||
-            PhoneNumberMatcher.containsMoreThanOneSlash(candidate) ||
+            PhoneNumberMatcher.containsMoreThanOneSlashInNationalNumber(number, candidate) ||
             !PhoneNumberMatcher.isNationalPrefixPresentIfRequired(number, util)) {
           return false;
         }
@@ -747,6 +748,17 @@ public class PhoneNumberUtil {
     return normalizedDigits;
   }
 
+  /**
+   * Normalizes a string of characters representing a phone number. This strips all characters which
+   * are not diallable on a mobile phone keypad (including all non-ASCII digits).
+   *
+   * @param number  a string of characters representing a phone number
+   * @return        the normalized string version of the phone number
+   */
+  static String normalizeDiallableCharsOnly(String number) {
+    return normalizeHelper(number, DIALLABLE_CHAR_MAPPINGS, true /* remove non matches */);
+  }
+
   /**
    * Converts all alpha characters in a number to their respective digits on a keypad, but retains
    * existing formatting.
@@ -1267,8 +1279,7 @@ public class PhoneNumberUtil {
                             : format(numberNoExt, PhoneNumberFormat.E164);
     }
     return withFormatting ? formattedNumber
-                          : normalizeHelper(formattedNumber, DIALLABLE_CHAR_MAPPINGS,
-                                            true /* remove non matches */);
+                          : normalizeDiallableCharsOnly(formattedNumber);
   }
 
   /**
@@ -1453,10 +1464,8 @@ public class PhoneNumberUtil {
     // If no digit is inserted/removed/modified as a result of our formatting, we return the
     // formatted phone number; otherwise we return the raw input the user entered.
     if (formattedNumber != null && rawInput.length() > 0) {
-      String normalizedFormattedNumber =
-          normalizeHelper(formattedNumber, DIALLABLE_CHAR_MAPPINGS, true /* remove non matches */);
-      String normalizedRawInput =
-          normalizeHelper(rawInput, DIALLABLE_CHAR_MAPPINGS, true /* remove non matches */);
+      String normalizedFormattedNumber = normalizeDiallableCharsOnly(formattedNumber);
+      String normalizedRawInput = normalizeDiallableCharsOnly(rawInput);
       if (!normalizedFormattedNumber.equals(normalizedRawInput)) {
         formattedNumber = rawInput;
       }
@@ -1961,14 +1970,21 @@ public class PhoneNumberUtil {
     return countryCodeToNonGeographicalMetadataMap.get(countryCallingCode);
   }
 
-  private boolean isNumberMatchingDesc(String nationalNumber, PhoneNumberDesc numberDesc) {
+  // @VisibleForTesting
+  boolean isNumberPossibleForDesc(String nationalNumber, PhoneNumberDesc numberDesc) {
     Matcher possibleNumberPatternMatcher =
         regexCache.getPatternForRegex(numberDesc.getPossibleNumberPattern())
             .matcher(nationalNumber);
+    return possibleNumberPatternMatcher.matches();
+  }
+
+  // @VisibleForTesting
+  boolean isNumberMatchingDesc(String nationalNumber, PhoneNumberDesc numberDesc) {
     Matcher nationalNumberPatternMatcher =
         regexCache.getPatternForRegex(numberDesc.getNationalNumberPattern())
             .matcher(nationalNumber);
-    return possibleNumberPatternMatcher.matches() && nationalNumberPatternMatcher.matches();
+    return isNumberPossibleForDesc(nationalNumber, numberDesc) &&
+        nationalNumberPatternMatcher.matches();
   }
 
   /**

java/libphonenumber/src/com/google/i18n/phonenumbers/Phonemetadata.java

@@ -447,6 +447,20 @@ public final class Phonemetadata {
       return this;
     }
 
+    // required PhoneNumberDesc emergency = 27;
+    private boolean hasEmergency;
+    private PhoneNumberDesc emergency_ = null;
+    public boolean hasEmergency() { return hasEmergency; }
+    public PhoneNumberDesc getEmergency() { return emergency_; }
+    public PhoneMetadata setEmergency(PhoneNumberDesc value) {
+      if (value == null) {
+        throw new NullPointerException();
+      }
+      hasEmergency = true;
+      emergency_ = value;
+      return this;
+    }
+
     // required PhoneNumberDesc voicemail = 28;
     private boolean hasVoicemail;
     private PhoneNumberDesc voicemail_ = null;
@@ -461,17 +475,31 @@ public final class Phonemetadata {
       return this;
     }
 
-    // required PhoneNumberDesc emergency = 27;
-    private boolean hasEmergency;
-    private PhoneNumberDesc emergency_ = null;
-    public boolean hasEmergency() { return hasEmergency; }
-    public PhoneNumberDesc getEmergency() { return emergency_; }
-    public PhoneMetadata setEmergency(PhoneNumberDesc value) {
+    // required PhoneNumberDesc short_code = 29;
+    private boolean hasShortCode;
+    private PhoneNumberDesc shortCode_ = null;
+    public boolean hasShortCode() { return hasShortCode; }
+    public PhoneNumberDesc getShortCode() { return shortCode_; }
+    public PhoneMetadata setShortCode(PhoneNumberDesc value) {
       if (value == null) {
         throw new NullPointerException();
       }
-      hasEmergency = true;
-      emergency_ = value;
+      hasShortCode = true;
+      shortCode_ = value;
+      return this;
+    }
+
+    // required PhoneNumberDesc standard_rate = 30;
+    private boolean hasStandardRate;
+    private PhoneNumberDesc standardRate_ = null;
+    public boolean hasStandardRate() { return hasStandardRate; }
+    public PhoneNumberDesc getStandardRate() { return standardRate_; }
+    public PhoneMetadata setStandardRate(PhoneNumberDesc value) {
+      if (value == null) {
+        throw new NullPointerException();
+      }
+      hasStandardRate = true;
+      standardRate_ = value;
       return this;
     }
 
@@ -705,13 +733,21 @@ public final class Phonemetadata {
       if (hasUan) {
         uan_.writeExternal(objectOutput);
       }
+      objectOutput.writeBoolean(hasEmergency);
+      if (hasEmergency) {
+        emergency_.writeExternal(objectOutput);
+      }
       objectOutput.writeBoolean(hasVoicemail);
       if (hasVoicemail) {
         voicemail_.writeExternal(objectOutput);
       }
-      objectOutput.writeBoolean(hasEmergency);
-      if (hasEmergency) {
-        emergency_.writeExternal(objectOutput);
+      objectOutput.writeBoolean(hasShortCode);
+      if (hasShortCode) {
+        shortCode_.writeExternal(objectOutput);
+      }
+      objectOutput.writeBoolean(hasStandardRate);
+      if (hasStandardRate) {
+        standardRate_.writeExternal(objectOutput);
       }
       objectOutput.writeBoolean(hasNoInternationalDialling);
       if (hasNoInternationalDialling) {
@@ -833,6 +869,12 @@ public final class Phonemetadata {
         setUan(desc);
       }
       hasDesc = objectInput.readBoolean();
+      if (hasDesc) {
+        PhoneNumberDesc desc = new PhoneNumberDesc();
+        desc.readExternal(objectInput);
+        setEmergency(desc);
+      }
+      hasDesc = objectInput.readBoolean();
       if (hasDesc) {
         PhoneNumberDesc desc = new PhoneNumberDesc();
         desc.readExternal(objectInput);
@@ -842,7 +884,13 @@ public final class Phonemetadata {
       if (hasDesc) {
         PhoneNumberDesc desc = new PhoneNumberDesc();
         desc.readExternal(objectInput);
-        setEmergency(desc);
+        setShortCode(desc);
+      }
+      hasDesc = objectInput.readBoolean();
+      if (hasDesc) {
+        PhoneNumberDesc desc = new PhoneNumberDesc();
+        desc.readExternal(objectInput);
+        setStandardRate(desc);
       }
       hasDesc = objectInput.readBoolean();
       if (hasDesc) {

java/libphonenumber/src/com/google/i18n/phonenumbers/ShortNumberUtil.java

@@ -17,7 +17,12 @@
 package com.google.i18n.phonenumbers;
 
 import com.google.i18n.phonenumbers.Phonemetadata.PhoneMetadata;
+import com.google.i18n.phonenumbers.Phonemetadata.PhoneNumberDesc;
 
+import java.util.Collections;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import java.util.regex.Pattern;
 
 /*
@@ -25,10 +30,22 @@ import java.util.regex.Pattern;
  * most commercial short numbers are not handled here, but by the PhoneNumberUtil.
  *
  * @author Shaopeng Jia
+ * @author David Yonge-Mallo
  */
 public class ShortNumberUtil {
 
   private final PhoneNumberUtil phoneUtil;
+  private static final Logger LOGGER = Logger.getLogger(ShortNumberUtil.class.getName());
+
+  /**
+   * Cost categories of short numbers.
+   */
+  public enum ShortNumberCost {
+    TOLL_FREE,
+    STANDARD_RATE,
+    PREMIUM_RATE,
+    UNKNOWN_COST
+  }
 
   public ShortNumberUtil() {
     phoneUtil = PhoneNumberUtil.getInstance();
@@ -39,6 +56,71 @@ public class ShortNumberUtil {
     phoneUtil = util;
   }
 
+  /**
+   * Convenience method to get a list of what regions the library has metadata for.
+   */
+  public Set<String> getSupportedRegions() {
+    return Collections.unmodifiableSet(MetadataManager.getShortNumberMetadataSupportedRegions());
+  }
+
+  /**
+   * Gets a valid short number for the specified region.
+   *
+   * @param regionCode  the region for which an example short number is needed
+   * @return  a valid short number for the specified region. Returns an empty string when the
+   *    metadata does not contain such information.
+   */
+  // @VisibleForTesting
+  String getExampleShortNumber(String regionCode) {
+    PhoneMetadata phoneMetadata = MetadataManager.getShortNumberMetadataForRegion(regionCode);
+    if (null == phoneMetadata) {
+      LOGGER.log(Level.WARNING, "Unable to get short number metadata for region: " + regionCode);
+      return "";
+    }
+    PhoneNumberDesc desc = phoneMetadata.getShortCode();
+    if (desc.hasExampleNumber()) {
+      return desc.getExampleNumber();
+    }
+    return "";
+  }
+
+  /**
+   * Gets a valid short number for the specified cost category.
+   *
+   * @param regionCode  the region for which an example short number is needed
+   * @param cost  the cost category of number that is needed
+   * @return  a valid short number for the specified region and cost category. Returns an empty
+   *    string when the metadata does not contain such information, or the cost is UNKNOWN_COST.
+   */
+  // @VisibleForTesting
+  String getExampleShortNumberForCost(String regionCode, ShortNumberCost cost) {
+    PhoneMetadata phoneMetadata = MetadataManager.getShortNumberMetadataForRegion(regionCode);
+    if (null == phoneMetadata) {
+      LOGGER.log(Level.WARNING, "Unable to get short number metadata for region: " + regionCode);
+      return "";
+    }
+    PhoneNumberDesc desc = getShortNumberDescByCost(phoneMetadata, cost);
+    if (desc != null && desc.hasExampleNumber()) {
+      return desc.getExampleNumber();
+    }
+    return "";
+  }
+
+  private PhoneNumberDesc getShortNumberDescByCost(PhoneMetadata metadata, ShortNumberCost cost) {
+    switch (cost) {
+      case TOLL_FREE:
+        return metadata.getTollFree();
+      case STANDARD_RATE:
+        return metadata.getStandardRate();
+      case PREMIUM_RATE:
+        return metadata.getPremiumRate();
+      default:
+        // UNKNOWN_COST numbers are computed by the process of elimination from the other cost
+        // categories.
+        return null;
+    }
+  }
+
   /**
    * Returns true if the number might be used to connect to an emergency service in the given
    * region.

java/libphonenumber/src/com/google/i18n/phonenumbers/ShortNumbersRegionCodeSet.java

@@ -26,9 +26,9 @@ import java.util.Set;
 public class ShortNumbersRegionCodeSet {
   // A set of all region codes for which data is available.
   static Set<String> getRegionCodeSet() {
-    // The capacity is set to 1 as there are 50 different entries,
+    // The capacity is set to 66 as there are 50 different entries,
     // and this offers a load factor of roughly 0.75.
-    Set<String> regionCodeSet = new HashSet<String>(1);
+    Set<String> regionCodeSet = new HashSet<String>(66);
 
     regionCodeSet.add("AM");
     regionCodeSet.add("AR");