From f20f72b20e787843b89a70fc03a55c94bc4994dd Mon Sep 17 00:00:00 2001 From: Denys Ponomarenko Date: Tue, 15 Aug 2023 02:28:15 +0000 Subject: [PATCH] loading metadata from file added --- cpp/CMakeLists.txt | 2 +- cpp/src/phonenumbers/phonenumberutil.cc | 121 +++++++++++++++--- cpp/src/phonenumbers/phonenumberutil.h | 8 ++ cpp/src/phonenumbers/shortnumberinfo.cc | 48 ++++++- cpp/src/phonenumbers/shortnumberinfo.h | 8 +- cpp/test/phonenumbers/phonenumberutil_test.cc | 11 +- 6 files changed, 171 insertions(+), 27 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cdd7a0813..1de908571 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -151,7 +151,7 @@ if (USE_BOOST) if (WIN32) set (Boost_USE_STATIC_LIBS ON) endif () - find_package (Boost 1.40.0 COMPONENTS date_time system thread) + find_package (Boost 1.40.0 COMPONENTS date_time system filesystem thread) if (NOT Boost_FOUND) print_error ("Boost Date_Time/System/Thread" "Boost") endif () diff --git a/cpp/src/phonenumbers/phonenumberutil.cc b/cpp/src/phonenumbers/phonenumberutil.cc index ec53db1b8..85649c568 100644 --- a/cpp/src/phonenumbers/phonenumberutil.cc +++ b/cpp/src/phonenumbers/phonenumberutil.cc @@ -21,7 +21,7 @@ #include #include #include - +#include #include #include @@ -34,7 +34,6 @@ #include "phonenumbers/matcher_api.h" #include "phonenumbers/metadata.h" #include "phonenumbers/normalize_utf8.h" -#include "phonenumbers/phonemetadata.pb.h" #include "phonenumbers/phonenumber.h" #include "phonenumbers/phonenumber.pb.h" #include "phonenumbers/regex_based_matcher.h" @@ -47,6 +46,8 @@ #include "phonenumbers/utf/unicodetext.h" #include "phonenumbers/utf/utf.h" +#include + namespace i18n { namespace phonenumbers { @@ -121,6 +122,21 @@ const char kPossibleCharsAfterExtLabel[] = "[:\\.\xEF\xBC\x8E]?[ \xC2\xA0\\t,-]*"; const char kOptionalExtSuffix[] = "#?"; +const string METADATA_FILE_NAME = "metadata.dat"; + +#ifdef ISTREAM_DATA_PROVIDER +bool LoadMetadataFromFile(const string& fileName, PhoneMetadataCollection* metadata) { + std::fstream input(fileName.c_str(), std::ios::in | std::ios::binary); + if (!input) { + LOG(ERROR) << "metadata file not found."; + } else if (!metadata->ParseFromIstream(&input)) { + LOG(ERROR) << "Could not parse binary data from file."; + return false; + } + + return true; +} +#else bool LoadCompiledInMetadata(PhoneMetadataCollection* metadata) { if (!metadata->ParseFromArray(metadata_get(), metadata_size())) { LOG(ERROR) << "Could not parse binary data."; @@ -128,6 +144,7 @@ bool LoadCompiledInMetadata(PhoneMetadataCollection* metadata) { } return true; } +#endif // ISTREAM_DATA_PROVIDER // Returns a pointer to the description inside the metadata of the appropriate // type. @@ -846,25 +863,62 @@ class PhoneNumberRegExpsAndMappings { DISALLOW_COPY_AND_ASSIGN(PhoneNumberRegExpsAndMappings); }; -// Private constructor. Also takes care of initialisation. -PhoneNumberUtil::PhoneNumberUtil() - : logger_(Logger::set_logger_impl(new NullLogger())), - matcher_api_(new RegexBasedMatcher()), - reg_exps_(new PhoneNumberRegExpsAndMappings), - country_calling_code_to_region_code_map_( - new std::vector()), - nanpa_regions_(new absl::node_hash_set()), - region_to_metadata_map_(new absl::node_hash_map()), - country_code_to_non_geographical_metadata_map_( - new absl::node_hash_map) { - Logger::set_logger_impl(logger_.get()); - // TODO: Update the java version to put the contents of the init - // method inside the constructor as well to keep both in sync. +#ifdef ISTREAM_DATA_PROVIDER +void PhoneNumberUtil::ClearMetadata() { + country_code_to_non_geographical_metadata_map_->clear(); + region_to_metadata_map_->clear(); + + gtl::STLDeleteContainerPairSecondPointers( + country_calling_code_to_region_code_map_->begin(), + country_calling_code_to_region_code_map_->end()); + + country_calling_code_to_region_code_map_->clear(); + nanpa_regions_->clear(); +} + +bool UpdateMetadataFile(const string& filepath) { + + boost::filesystem::path path_to = boost::filesystem::current_path(); + path_to += "/"; + path_to += METADATA_FILE_NAME; + boost::filesystem::path path_bk = path_to; + path_bk += ".bak"; + boost::system::error_code ec; + + boost::filesystem::copy(path_to.string(), path_bk.string(), ec); + if (ec) { + LOG(DFATAL) << "Could not create backup copy of metadata file." << ec.message(); + return false; + } + boost::filesystem::rename(filepath, path_to.string(), ec); + if(ec) { + LOG(DFATAL) << "Could not rename metadata file." << ec.message(); + return false; + } + boost::filesystem::remove(path_bk, ec); + if(ec) { + LOG(DFATAL) << "Could not remove metadata file." << ec.message(); + } + return true; +} + +bool PhoneNumberUtil::ReloadMetadata(const string& filepath) { + ClearMetadata(); PhoneMetadataCollection metadata_collection; - if (!LoadCompiledInMetadata(&metadata_collection)) { - LOG(DFATAL) << "Could not parse compiled-in metadata."; - return; + if (!LoadMetadataFromFile(filepath, &metadata_collection)) { + LOG(DFATAL) << "Could not parse metadata from file."; + return false; + } + LoadMetadataFromCollection(metadata_collection); + if(!UpdateMetadataFile(filepath)){ + LOG(DFATAL) << "Could not update metadata file."; + return false; } + return true; +} +#endif // ISTREAM_DATA_PROVIDER + +void PhoneNumberUtil::LoadMetadataFromCollection(const PhoneMetadataCollection& metadata_collection) { // Storing data in a temporary map to make it easier to find other regions // that share a country calling code when inserting data. std::map* > country_calling_code_to_region_map; @@ -913,6 +967,35 @@ PhoneNumberUtil::PhoneNumberUtil() country_calling_code_to_region_code_map_->end(), OrderByFirst()); } +// Private constructor. Also takes care of initialisation. +PhoneNumberUtil::PhoneNumberUtil() + : logger_(Logger::set_logger_impl(new NullLogger())), + matcher_api_(new RegexBasedMatcher()), + reg_exps_(new PhoneNumberRegExpsAndMappings), + country_calling_code_to_region_code_map_( + new std::vector()), + nanpa_regions_(new absl::node_hash_set()), + region_to_metadata_map_(new absl::node_hash_map()), + country_code_to_non_geographical_metadata_map_( + new absl::node_hash_map) { + Logger::set_logger_impl(logger_.get()); + // TODO: Update the java version to put the contents of the init + // method inside the constructor as well to keep both in sync. + PhoneMetadataCollection metadata_collection; +#ifdef ISTREAM_DATA_PROVIDER + if (!LoadMetadataFromFile(METADATA_FILE_NAME, &metadata_collection)) { + LOG(DFATAL) << "Could not parse metadata from file."; + return; + } +#else + if (!LoadCompiledInMetadata(&metadata_collection)) { + LOG(DFATAL) << "Could not parse compiled-in metadata."; + return; + } +#endif // ISTREAM_DATA_PROVIDER + LoadMetadataFromCollection(metadata_collection); +} + PhoneNumberUtil::~PhoneNumberUtil() { gtl::STLDeleteContainerPairSecondPointers( country_calling_code_to_region_code_map_->begin(), diff --git a/cpp/src/phonenumbers/phonenumberutil.h b/cpp/src/phonenumbers/phonenumberutil.h index 1a3ddd920..cb9e31795 100644 --- a/cpp/src/phonenumbers/phonenumberutil.h +++ b/cpp/src/phonenumbers/phonenumberutil.h @@ -30,6 +30,7 @@ #include "phonenumbers/base/memory/singleton.h" #include "phonenumbers/phonenumber.pb.h" +#include "phonenumbers/phonemetadata.pb.h" #include "absl/container/node_hash_set.h" #include "absl/container/node_hash_map.h" @@ -67,6 +68,11 @@ class PhoneNumberUtil : public Singleton { friend class ShortNumberInfoTest; friend class Singleton; +#ifdef ISTREAM_DATA_PROVIDER + void ClearMetadata(); + public: + bool ReloadMetadata(const string& fileName); +#endif // ISTREAM_DATA_PROVIDER public: ~PhoneNumberUtil(); static const char kRegionCodeForNonGeoEntity[]; @@ -818,10 +824,12 @@ class PhoneNumberUtil : public Singleton { // PhoneMetadata for that country calling code. Examples of the country // calling codes include 800 (International Toll Free Service) and 808 // (International Shared Cost Service). + scoped_ptr > country_code_to_non_geographical_metadata_map_; PhoneNumberUtil(); + void LoadMetadataFromCollection(const PhoneMetadataCollection& metadata_collection); // Returns a regular expression for the possible extensions that may be found // in a number, for use when matching. diff --git a/cpp/src/phonenumbers/shortnumberinfo.cc b/cpp/src/phonenumbers/shortnumberinfo.cc index 8e5d12df9..6c1daeb95 100644 --- a/cpp/src/phonenumbers/shortnumberinfo.cc +++ b/cpp/src/phonenumbers/shortnumberinfo.cc @@ -26,6 +26,7 @@ #include "phonenumbers/regex_based_matcher.h" #include "phonenumbers/region_code.h" #include "phonenumbers/short_metadata.h" +#include namespace i18n { namespace phonenumbers { @@ -34,6 +35,30 @@ using google::protobuf::RepeatedField; using std::map; using std::string; + +#ifdef ISTREAM_DATA_PROVIDER +bool LoadMetadataFromFile(string fileName, PhoneMetadataCollection* metadata) { + std::fstream input(fileName.c_str(), std::ios::in | std::ios::binary); + if (!input) { + LOG(ERROR) << "metadata file not found."; + } else if (!metadata->ParseFromIstream(&input)) { + LOG(ERROR) << "Could not parse binary data from file."; + return false; + } + + return true; +} + +void ShortNumberInfo::ClearMetadata(){ + region_to_short_metadata_map_->clear(); + regions_where_emergency_numbers_must_be_exact_->clear(); +} + +void ShortNumberInfo::ReloadMetadata() { + ClearMetadata(); + LoadMetadata(); +} +#else bool LoadCompiledInMetadata(PhoneMetadataCollection* metadata) { if (!metadata->ParseFromArray(short_metadata_get(), short_metadata_size())) { LOG(ERROR) << "Could not parse binary data."; @@ -41,17 +66,22 @@ bool LoadCompiledInMetadata(PhoneMetadataCollection* metadata) { } return true; } +#endif // ISTREAM_DATA_PROVIDER -ShortNumberInfo::ShortNumberInfo() - : phone_util_(*PhoneNumberUtil::GetInstance()), - matcher_api_(new RegexBasedMatcher()), - region_to_short_metadata_map_(new absl::flat_hash_map()), - regions_where_emergency_numbers_must_be_exact_(new absl::flat_hash_set()) { +void ShortNumberInfo::LoadMetadata() { PhoneMetadataCollection metadata_collection; +#ifdef ISTREAM_DATA_PROVIDER + const string metadataInputFile = "metadata_short.dat"; + if (!LoadMetadataFromFile(metadataInputFile, &metadata_collection)) { + LOG(DFATAL) << "Could not parse metadata from file."; + return; + } +#else if (!LoadCompiledInMetadata(&metadata_collection)) { LOG(DFATAL) << "Could not parse compiled-in metadata."; return; } +#endif // ISTREAM_DATA_PROVIDER for (const auto& metadata : metadata_collection.metadata()) { const string& region_code = metadata.id(); region_to_short_metadata_map_->insert(std::make_pair(region_code, metadata)); @@ -61,6 +91,14 @@ ShortNumberInfo::ShortNumberInfo() regions_where_emergency_numbers_must_be_exact_->insert("NI"); } +ShortNumberInfo::ShortNumberInfo() + : phone_util_(*PhoneNumberUtil::GetInstance()), + matcher_api_(new RegexBasedMatcher()), + region_to_short_metadata_map_(new absl::flat_hash_map()), + regions_where_emergency_numbers_must_be_exact_(new absl::flat_hash_set()) { + LoadMetadata(); +} + ShortNumberInfo::~ShortNumberInfo() {} // Returns a pointer to the phone metadata for the appropriate region or NULL diff --git a/cpp/src/phonenumbers/shortnumberinfo.h b/cpp/src/phonenumbers/shortnumberinfo.h index db54b48ed..b67858e08 100644 --- a/cpp/src/phonenumbers/shortnumberinfo.h +++ b/cpp/src/phonenumbers/shortnumberinfo.h @@ -46,7 +46,12 @@ class ShortNumberInfo { public: ShortNumberInfo(); ~ShortNumberInfo(); - +#ifdef ISTREAM_DATA_PROVIDER + private: + void ClearMetadata(); + public: + void ReloadMetadata(); +#endif // ISTREAM_DATA_PROVIDER // Cost categories of short numbers. enum ShortNumberCost { TOLL_FREE, @@ -206,6 +211,7 @@ class ShortNumberInfo { bool MatchesEmergencyNumberHelper(const string& number, const string& region_code, bool allow_prefix_match) const; + void LoadMetadata(); DISALLOW_COPY_AND_ASSIGN(ShortNumberInfo); }; diff --git a/cpp/test/phonenumbers/phonenumberutil_test.cc b/cpp/test/phonenumbers/phonenumberutil_test.cc index 9bc28185e..18cf2db6b 100644 --- a/cpp/test/phonenumbers/phonenumberutil_test.cc +++ b/cpp/test/phonenumbers/phonenumberutil_test.cc @@ -118,7 +118,11 @@ class PhoneNumberUtilTest : public testing::Test { phone_util_.Parse(number_to_parse, RegionCode::ZZ(), &actual_number)); } - const PhoneNumberUtil& phone_util_; + void ReloadMetadata(const string fileName) { + phone_util_.ReloadMetadata(fileName); + } + + PhoneNumberUtil& phone_util_; private: DISALLOW_COPY_AND_ASSIGN(PhoneNumberUtilTest); @@ -159,6 +163,11 @@ TEST_F(PhoneNumberUtilTest, InterchangeInvalidCodepoints) { } } +TEST_F(PhoneNumberUtilTest, ReloadMetadata) { + const string filename = "metadata.dat"; + EXPECT_TRUE(phone_util_.ReloadMetadata(filename)); +} + TEST_F(PhoneNumberUtilTest, GetSupportedRegions) { std::set regions;