Browse Source

Merge 12c112f02a into cf446565d9

pull/3185/merge
dponomarenko-firstorion 4 days ago
committed by GitHub
parent
commit
4cf84b8b2b
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
9 changed files with 382 additions and 32 deletions
  1. +53
    -2
      cpp/CMakeLists.txt
  2. +68
    -0
      cpp/src/phonenumbers/metadata_convertor.cc
  3. +25
    -0
      cpp/src/phonenumbers/metadataconvertor.h
  4. +107
    -19
      cpp/src/phonenumbers/phonenumberutil.cc
  5. +8
    -0
      cpp/src/phonenumbers/phonenumberutil.h
  6. +89
    -8
      cpp/src/phonenumbers/shortnumberinfo.cc
  7. +8
    -2
      cpp/src/phonenumbers/shortnumberinfo.h
  8. +15
    -0
      cpp/test/phonenumbers/phonenumberutil_test.cc
  9. +9
    -1
      cpp/test/phonenumbers/shortnumberinfo_test.cc

+ 53
- 2
cpp/CMakeLists.txt View File

@ -94,6 +94,15 @@ option (BUILD_TESTING "Build testing" ON)
option (BUILD_TOOLS_ONLY "Limit build to targets in ../tools/cpp" OFF)
option (USE_STDMUTEX "Use C++ 2011 std::mutex for multi-threading" OFF)
option (USE_POSIX_THREAD "Use Posix api for multi-threading" OFF)
option (ISTREAM_DATA_PROVIDER "Use protobuf binary file as input for metadata" OFF)
if(ISTREAM_DATA_PROVIDER)
add_definitions ("-DISTREAM_DATA_PROVIDER")
if(NOT METADATA_PATH)
set(METADATA_PATH ${CMAKE_CURRENT_SOURCE_DIR})
endif()
add_definitions(-DMETADATA_PATH="${METADATA_PATH}")
endif()
if (USE_ALTERNATE_FORMATS)
add_definitions ("-DI18N_PHONENUMBERS_USE_ALTERNATE_FORMATS")
@ -145,7 +154,7 @@ if (USE_BOOST)
if (WIN32)
set (Boost_USE_STATIC_LIBS ON)
endif ()
find_package (Boost 1.40.0 COMPONENTS date_time system thread)
find_package (Boost 1.40.0 COMPONENTS date_time system filesystem thread)
if (NOT Boost_FOUND)
print_error ("Boost Date_Time/System/Thread" "Boost")
endif ()
@ -427,7 +436,7 @@ include_directories ("src")
# Collate dependencies
#----------------------------------------------------------------
set (LIBRARY_DEPS ${ICU_LIB} ${PROTOBUF_LIB} absl::node_hash_set absl::strings absl::synchronization)
set (LIBRARY_DEPS ${ICU_LIB} ${PROTOBUF_LIB} absl::node_hash_set absl::strings absl::synchronization absl::flags absl::flags_parse)
if (USE_BOOST)
list (APPEND LIBRARY_DEPS ${Boost_LIBRARIES})
@ -507,6 +516,25 @@ if (BUILD_SHARED_LIBS)
endif ()
endif ()
if(ISTREAM_DATA_PROVIDER)
if(USE_LITE_METADATA)
set (METADATA_CONVERTOR_SOURCES
"src/phonenumbers/metadata_convertor.cc"
"src/phonenumbers/short_metadata.cc"
"src/phonenumbers/lite_metadata.cc")
else()
set (METADATA_CONVERTOR_SOURCES
"src/phonenumbers/metadata_convertor.cc"
"src/phonenumbers/metadata.cc"
"src/phonenumbers/short_metadata.cc")
endif()
endif()
if(ISTREAM_DATA_PROVIDER)
add_executable(metadata_convertor ${METADATA_CONVERTOR_SOURCES})
target_link_libraries (metadata_convertor ${PROTOBUF_LIB} phonenumber)
endif()
#----------------------------------------------------------------
# Build testing library
#----------------------------------------------------------------
@ -557,6 +585,16 @@ if(BUILD_TESTING)
list (APPEND TEST_SOURCES ${GEOCODING_TEST_SOURCES})
endif ()
if(ISTREAM_DATA_PROVIDER)
set (METADATA_CONVERTOR_TEST_SOURCES
"src/phonenumbers/metadata_convertor.cc"
"src/phonenumbers/short_metadata.cc"
"src/phonenumbers/test_metadata.cc")
add_executable(metadata_convertor_test ${METADATA_CONVERTOR_TEST_SOURCES})
target_link_libraries (metadata_convertor_test ${PROTOBUF_LIB} phonenumber)
endif()
if (USE_ICU_REGEXP)
# Add the phone number matcher tests.
list (APPEND TEST_SOURCES "test/phonenumbers/phonenumbermatch_test.cc")
@ -597,6 +635,19 @@ if(BUILD_TESTING)
endif ()
endif()
# Generate metadata protobuf files
add_custom_command(
OUTPUT ${METADATA_PATH}/metadata.dat
COMMAND metadata_convertor_test "--output_file" ${METADATA_PATH}/metadata.dat
COMMENT "generate metadata .dat files"
VERBATIM
)
add_custom_target(
"generate_metadata_files" ALL
DEPENDS ${METADATA_PATH}/metadata.dat
)
#----------------------------------------------------------------
# Install built libraries
#----------------------------------------------------------------


+ 68
- 0
cpp/src/phonenumbers/metadata_convertor.cc View File

@ -0,0 +1,68 @@
#include <fstream>
#include <string>
#include <iostream>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "phonenumbers/phonemetadata.pb.h"
#include "phonenumbers/phonenumber.pb.h"
#include "phonenumbers/metadataconvertor.h"
#include "phonenumbers/short_metadata.h"
ABSL_FLAG(std::string, output_file, "metadata.dat", "output file name");
bool LoadCompiledInMetadataShort(i18n::phonenumbers::PhoneMetadataCollection* metadata) {
if (!metadata->ParseFromArray(i18n::phonenumbers::short_metadata_get(), i18n::phonenumbers::short_metadata_size())) {
std::cerr << "Could not parse binary data.";
return false;
}
return true;
}
std::string get_short_file_name(const std::string& filename) {
std::string::size_type idx = filename.rfind('.');
if(idx != std::string::npos){
std::string ext = filename.substr(idx+1);
std::string name = filename.substr(0, idx);
std::string short_name = name + "_short." + ext;
return short_name;
}
return filename + "_short";
}
int main(int argc, char *argv[]) {
absl::ParseCommandLine(argc, argv);
std::string output_file = absl::GetFlag(FLAGS_output_file);
i18n::phonenumbers::PhoneMetadataCollection metadata_collection;
MetadataConvertor convertor;
if (!convertor.LoadCompiledInMetadata(&metadata_collection)) {
std::cerr << "Could not parse compiled-in metadata." << std::endl;
return -1;
}
std::fstream output(output_file, std::ios::out | std::ios::trunc | std::ios::binary);
if (!metadata_collection.SerializeToOstream(&output)) {
std::cerr << "Failed to write metadata output file." << std::endl;
return -1;
}
i18n::phonenumbers::PhoneMetadataCollection short_metadata_collection;
std::string short_output_file = get_short_file_name(output_file);
if (!LoadCompiledInMetadataShort(&short_metadata_collection)) {
std::cerr << "Could not parse compiled-in metadata." << std::endl;
return -1;
}
std::fstream output_short(short_output_file, std::ios::out | std::ios::trunc | std::ios::binary);
if (!short_metadata_collection.SerializeToOstream(&output_short)) {
std::cerr << "Failed to write metadata output file." << std::endl;
return -1;
}
return 0;
}

+ 25
- 0
cpp/src/phonenumbers/metadataconvertor.h View File

@ -0,0 +1,25 @@
#ifndef I18N_PHONENUMBERS_METADATACONVERTOR_H_
#define I18N_PHONENUMBERS_METADATACONVERTOR_H_
#ifdef USE_LITE_METADATA
#include "phonenumbers/metadata_lite.h"
#else
#include "phonenumbers/metadata.h"
#endif
class MetadataConvertor {
public:
bool LoadCompiledInMetadata(i18n::phonenumbers::PhoneMetadataCollection* metadata)const {
if (!metadata->ParseFromArray(i18n::phonenumbers::metadata_get(), i18n::phonenumbers::metadata_size())) {
std::cerr << "ShortNumberConvertor : Could not parse binary data.";
return false;
}
return true;
}
public:
MetadataConvertor(){}
~MetadataConvertor(){}
};
#endif // I18N_PHONENUMBERS_METADATACONVERTOR_H_

+ 107
- 19
cpp/src/phonenumbers/phonenumberutil.cc View File

@ -21,7 +21,8 @@
#include <map>
#include <utility>
#include <vector>
#include <fstream>
#include <boost/filesystem.hpp>
#include <unicode/uchar.h>
#include <unicode/utf8.h>
@ -34,7 +35,6 @@
#include "phonenumbers/matcher_api.h"
#include "phonenumbers/metadata.h"
#include "phonenumbers/normalize_utf8.h"
#include "phonenumbers/phonemetadata.pb.h"
#include "phonenumbers/phonenumber.h"
#include "phonenumbers/phonenumber.pb.h"
#include "phonenumbers/regex_based_matcher.h"
@ -121,6 +121,21 @@ const char kPossibleCharsAfterExtLabel[] =
"[:\\.\xEF\xBC\x8E]?[ \xC2\xA0\\t,-]*";
const char kOptionalExtSuffix[] = "#?";
const string METADATA_FILE_NAME = "metadata.dat";
#ifdef ISTREAM_DATA_PROVIDER
bool LoadMetadataFromFile(const string& fileName, PhoneMetadataCollection* metadata) {
std::fstream input(fileName.c_str(), std::ios::in | std::ios::binary);
if (!input) {
LOG(ERROR) << "metadata file not found.";
} else if (!metadata->ParseFromIstream(&input)) {
LOG(ERROR) << "Could not parse binary data from file.";
return false;
}
return true;
}
#else
bool LoadCompiledInMetadata(PhoneMetadataCollection* metadata) {
if (!metadata->ParseFromArray(metadata_get(), metadata_size())) {
LOG(ERROR) << "Could not parse binary data.";
@ -128,6 +143,7 @@ bool LoadCompiledInMetadata(PhoneMetadataCollection* metadata) {
}
return true;
}
#endif // ISTREAM_DATA_PROVIDER
// Returns a pointer to the description inside the metadata of the appropriate
// type.
@ -854,25 +870,65 @@ class PhoneNumberRegExpsAndMappings {
const PhoneNumberRegExpsAndMappings&) = delete;
};
// Private constructor. Also takes care of initialisation.
PhoneNumberUtil::PhoneNumberUtil()
: logger_(Logger::set_logger_impl(new NullLogger())),
matcher_api_(new RegexBasedMatcher()),
reg_exps_(new PhoneNumberRegExpsAndMappings),
country_calling_code_to_region_code_map_(
new std::vector<IntRegionsPair>()),
nanpa_regions_(new absl::node_hash_set<string>()),
region_to_metadata_map_(new absl::node_hash_map<string, PhoneMetadata>()),
country_code_to_non_geographical_metadata_map_(
new absl::node_hash_map<int, PhoneMetadata>) {
Logger::set_logger_impl(logger_.get());
// TODO: Update the java version to put the contents of the init
// method inside the constructor as well to keep both in sync.
#ifdef ISTREAM_DATA_PROVIDER
void PhoneNumberUtil::ClearMetadata() {
country_code_to_non_geographical_metadata_map_->clear();
region_to_metadata_map_->clear();
gtl::STLDeleteContainerPairSecondPointers(
country_calling_code_to_region_code_map_->begin(),
country_calling_code_to_region_code_map_->end());
country_calling_code_to_region_code_map_->clear();
nanpa_regions_->clear();
}
bool UpdateMetadataFile(const string& filepath) {
boost::filesystem::path path_to(METADATA_PATH);
path_to /= METADATA_FILE_NAME;
boost::filesystem::path path_bk = path_to;
path_bk += ".bak";
boost::system::error_code ec;
if (path_to.string() == filepath) {
return true;
}
boost::filesystem::copy(path_to.string(), path_bk.string(), ec);
if (ec) {
LOG(DFATAL) << "Could not create backup copy of metadata file." << ec.message();
return false;
}
boost::filesystem::rename(filepath, path_to.string(), ec);
if(ec) {
LOG(DFATAL) << "Could not rename metadata file." << ec.message();
return false;
}
boost::filesystem::remove(path_bk, ec);
if(ec) {
LOG(DFATAL) << "Could not remove metadata file." << ec.message();
}
return true;
}
bool PhoneNumberUtil::ReloadMetadata(const string& filepath) {
ClearMetadata();
PhoneMetadataCollection metadata_collection;
if (!LoadCompiledInMetadata(&metadata_collection)) {
LOG(DFATAL) << "Could not parse compiled-in metadata.";
return;
if (!LoadMetadataFromFile(filepath, &metadata_collection)) {
LOG(DFATAL) << "Could not parse metadata from file.";
return false;
}
LoadMetadataFromCollection(metadata_collection);
if(!UpdateMetadataFile(filepath)){
LOG(DFATAL) << "Could not update metadata file.";
return false;
}
return true;
}
#endif // ISTREAM_DATA_PROVIDER
void PhoneNumberUtil::LoadMetadataFromCollection(const PhoneMetadataCollection& metadata_collection) {
// Storing data in a temporary map to make it easier to find other regions
// that share a country calling code when inserting data.
std::map<int, std::list<string>* > country_calling_code_to_region_map;
@ -921,6 +977,38 @@ PhoneNumberUtil::PhoneNumberUtil()
country_calling_code_to_region_code_map_->end(), OrderByFirst());
}
// Private constructor. Also takes care of initialisation.
PhoneNumberUtil::PhoneNumberUtil()
: logger_(Logger::set_logger_impl(new NullLogger())),
matcher_api_(new RegexBasedMatcher()),
reg_exps_(new PhoneNumberRegExpsAndMappings),
country_calling_code_to_region_code_map_(
new std::vector<IntRegionsPair>()),
nanpa_regions_(new absl::node_hash_set<string>()),
region_to_metadata_map_(new absl::node_hash_map<string, PhoneMetadata>()),
country_code_to_non_geographical_metadata_map_(
new absl::node_hash_map<int, PhoneMetadata>) {
Logger::set_logger_impl(logger_.get());
// TODO: Update the java version to put the contents of the init
// method inside the constructor as well to keep both in sync.
PhoneMetadataCollection metadata_collection;
#ifdef ISTREAM_DATA_PROVIDER
boost::filesystem::path path_to(METADATA_PATH);
path_to += "/";
path_to += METADATA_FILE_NAME;
if (!LoadMetadataFromFile(path_to.string(), &metadata_collection)) {
LOG(DFATAL) << "Could not parse metadata from file.";
return;
}
#else
if (!LoadCompiledInMetadata(&metadata_collection)) {
LOG(DFATAL) << "Could not parse compiled-in metadata.";
return;
}
#endif // ISTREAM_DATA_PROVIDER
LoadMetadataFromCollection(metadata_collection);
}
PhoneNumberUtil::~PhoneNumberUtil() {
gtl::STLDeleteContainerPairSecondPointers(
country_calling_code_to_region_code_map_->begin(),


+ 8
- 0
cpp/src/phonenumbers/phonenumberutil.h View File

@ -30,6 +30,7 @@
#include "phonenumbers/base/memory/singleton.h"
#include "phonenumbers/phonenumber.pb.h"
#include "phonenumbers/phonemetadata.pb.h"
#include "absl/container/node_hash_set.h"
#include "absl/container/node_hash_map.h"
@ -67,6 +68,11 @@ class PhoneNumberUtil : public Singleton<PhoneNumberUtil> {
friend class ShortNumberInfoTest;
friend class Singleton<PhoneNumberUtil>;
#ifdef ISTREAM_DATA_PROVIDER
void ClearMetadata();
public:
bool ReloadMetadata(const string& fileName);
#endif // ISTREAM_DATA_PROVIDER
public:
// This type is neither copyable nor movable.
PhoneNumberUtil(const PhoneNumberUtil&) = delete;
@ -836,10 +842,12 @@ class PhoneNumberUtil : public Singleton<PhoneNumberUtil> {
// PhoneMetadata for that country calling code. Examples of the country
// calling codes include 800 (International Toll Free Service) and 808
// (International Shared Cost Service).
scoped_ptr<absl::node_hash_map<int, PhoneMetadata> >
country_code_to_non_geographical_metadata_map_;
PhoneNumberUtil();
void LoadMetadataFromCollection(const PhoneMetadataCollection& metadata_collection);
// Returns a regular expression for the possible extensions that may be found
// in a number, for use when matching.


+ 89
- 8
cpp/src/phonenumbers/shortnumberinfo.cc View File

@ -19,13 +19,15 @@
#include <iterator>
#include <map>
#include <boost/filesystem.hpp>
#include "phonenumbers/default_logger.h"
#include "phonenumbers/matcher_api.h"
#include "phonenumbers/phonemetadata.pb.h"
#include "phonenumbers/phonenumberutil.h"
#include "phonenumbers/regex_based_matcher.h"
#include "phonenumbers/region_code.h"
#include "phonenumbers/short_metadata.h"
#include <fstream>
namespace i18n {
namespace phonenumbers {
@ -34,6 +36,69 @@ using google::protobuf::RepeatedField;
using std::map;
using std::string;
#ifdef ISTREAM_DATA_PROVIDER
const string METADATA_SHORT_FILE_NAME = "metadata_short.dat";
bool LoadMetadataFromFile(string fileName, PhoneMetadataCollection* metadata) {
std::fstream input(fileName.c_str(), std::ios::in | std::ios::binary);
if (!input) {
LOG(ERROR) << "metadata file not found.";
} else if (!metadata->ParseFromIstream(&input)) {
LOG(ERROR) << "Could not parse binary data from file.";
return false;
}
return true;
}
void ShortNumberInfo::ClearMetadata(){
region_to_short_metadata_map_->clear();
regions_where_emergency_numbers_must_be_exact_->clear();
}
bool UpdateShortMetadataFile(const string& filepath) {
boost::filesystem::path path_to(METADATA_PATH);
path_to += "/";
path_to += METADATA_SHORT_FILE_NAME;
boost::filesystem::path path_bk = path_to;
path_bk += ".bak";
boost::system::error_code ec;
boost::filesystem::copy(path_to.string(), path_bk.string(), ec);
if (ec) {
LOG(DFATAL) << "Could not create backup copy of metadata file." << ec.message();
return false;
}
boost::filesystem::rename(filepath, path_to.string(), ec);
if(ec) {
LOG(DFATAL) << "Could not rename metadata file." << ec.message();
return false;
}
boost::filesystem::remove(path_bk, ec);
if(ec) {
LOG(DFATAL) << "Could not remove metadata file." << ec.message();
}
return true;
}
bool ShortNumberInfo::ReloadMetadata(const string& filename) {
ClearMetadata();
PhoneMetadataCollection metadata_collection;
if (!LoadMetadataFromFile(filename, &metadata_collection)) {
LOG(DFATAL) << "Could not parse metadata from file.";
return false;
}
LoadMetadataFromCollection(metadata_collection);
if(!UpdateShortMetadataFile(filename)){
LOG(DFATAL) << "Could not update metadata file.";
return false;
}
return true;
}
#else
bool LoadCompiledInMetadata(PhoneMetadataCollection* metadata) {
if (!metadata->ParseFromArray(short_metadata_get(), short_metadata_size())) {
LOG(ERROR) << "Could not parse binary data.";
@ -41,24 +106,40 @@ bool LoadCompiledInMetadata(PhoneMetadataCollection* metadata) {
}
return true;
}
#endif // ISTREAM_DATA_PROVIDER
void ShortNumberInfo::LoadMetadataFromCollection(const PhoneMetadataCollection& metadata_collection) {
for (const auto& metadata : metadata_collection.metadata()) {
const string& region_code = metadata.id();
region_to_short_metadata_map_->insert(std::make_pair(region_code, metadata));
}
regions_where_emergency_numbers_must_be_exact_->insert("BR");
regions_where_emergency_numbers_must_be_exact_->insert("CL");
regions_where_emergency_numbers_must_be_exact_->insert("NI");
}
ShortNumberInfo::ShortNumberInfo()
: phone_util_(*PhoneNumberUtil::GetInstance()),
matcher_api_(new RegexBasedMatcher()),
region_to_short_metadata_map_(new absl::flat_hash_map<string, PhoneMetadata>()),
regions_where_emergency_numbers_must_be_exact_(new absl::flat_hash_set<string>()) {
PhoneMetadataCollection metadata_collection;
#ifdef ISTREAM_DATA_PROVIDER
boost::filesystem::path path_to(METADATA_PATH);
path_to += "/";
path_to += METADATA_SHORT_FILE_NAME;
if (!LoadMetadataFromFile(path_to.string(), &metadata_collection)) {
LOG(DFATAL) << "Could not parse metadata from file.";
return;
}
#else
if (!LoadCompiledInMetadata(&metadata_collection)) {
LOG(DFATAL) << "Could not parse compiled-in metadata.";
return;
}
for (const auto& metadata : metadata_collection.metadata()) {
const string& region_code = metadata.id();
region_to_short_metadata_map_->insert(std::make_pair(region_code, metadata));
}
regions_where_emergency_numbers_must_be_exact_->insert("BR");
regions_where_emergency_numbers_must_be_exact_->insert("CL");
regions_where_emergency_numbers_must_be_exact_->insert("NI");
#endif // ISTREAM_DATA_PROVIDER
LoadMetadataFromCollection(metadata_collection);
}
ShortNumberInfo::~ShortNumberInfo() {}


+ 8
- 2
cpp/src/phonenumbers/shortnumberinfo.h View File

@ -23,7 +23,7 @@
#include <map>
#include <set>
#include <string>
#include "phonenumbers/phonemetadata.pb.h"
#include "phonenumbers/base/basictypes.h"
#include "phonenumbers/base/memory/scoped_ptr.h"
#include "absl/container/flat_hash_set.h"
@ -43,6 +43,7 @@ class PhoneNumber;
class PhoneNumberUtil;
class ShortNumberInfo {
void LoadMetadataFromCollection(const PhoneMetadataCollection& metadata_collection);
public:
ShortNumberInfo();
@ -51,7 +52,12 @@ class ShortNumberInfo {
ShortNumberInfo& operator=(const ShortNumberInfo&) = delete;
~ShortNumberInfo();
#ifdef ISTREAM_DATA_PROVIDER
private:
void ClearMetadata();
public:
bool ReloadMetadata(const string& filename);
#endif // ISTREAM_DATA_PROVIDER
// Cost categories of short numbers.
enum ShortNumberCost {
TOLL_FREE,


+ 15
- 0
cpp/test/phonenumbers/phonenumberutil_test.cc View File

@ -123,7 +123,14 @@ class PhoneNumberUtilTest : public testing::Test {
phone_util_.Parse(number_to_parse, RegionCode::ZZ(), &actual_number));
}
#ifdef ISTREAM_DATA_PROVIDER
void ReloadMetadata(const string fileName) {
phone_util_.ReloadMetadata(fileName);
}
PhoneNumberUtil& phone_util_;
#else
const PhoneNumberUtil& phone_util_;
#endif
};
@ -162,6 +169,14 @@ TEST_F(PhoneNumberUtilTest, InterchangeInvalidCodepoints) {
}
}
#ifdef ISTREAM_DATA_PROVIDER
TEST_F(PhoneNumberUtilTest, ReloadMetadata) {
string path(METADATA_PATH);
path.append("/metadata.dat");
EXPECT_TRUE(phone_util_.ReloadMetadata(path));
}
#endif
TEST_F(PhoneNumberUtilTest, GetSupportedRegions) {
std::set<string> regions;


+ 9
- 1
cpp/test/phonenumbers/shortnumberinfo_test.cc View File

@ -52,7 +52,7 @@ class ShortNumberInfoTest : public testing::Test {
}
const PhoneNumberUtil phone_util_;
const ShortNumberInfo short_info_;
ShortNumberInfo short_info_;
};
@ -99,6 +99,14 @@ TEST_F(ShortNumberInfoTest, IsValidShortNumber) {
EXPECT_TRUE(short_info_.IsValidShortNumber(shared_number));
}
#ifdef ISTREAM_DATA_PROVIDER
TEST_F(ShortNumberInfoTest, ReloadMetadata) {
string path(METADATA_PATH);
path.append("/metadata_short.dat");
EXPECT_TRUE(short_info_.ReloadMetadata(path));
}
#endif
TEST_F(ShortNumberInfoTest, IsCarrierSpecific) {
PhoneNumber carrier_specific_number;
carrier_specific_number.set_country_code(1);


Loading…
Cancel
Save