From 780345bc2764753aae8fbf277dfe477375eff59e Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Tue, 9 Jun 2020 02:24:58 -0400 Subject: [PATCH 01/12] put file stream inside scope to force buffer flush This was already fixed for serializeXML but not for serializeToXMLFile or deserializeFromXMLFile. --- gtsam/base/serialization.h | 12 ++++-- gtsam/base/serializationTestHelpers.h | 57 +++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/gtsam/base/serialization.h b/gtsam/base/serialization.h index e475465de..088029903 100644 --- a/gtsam/base/serialization.h +++ b/gtsam/base/serialization.h @@ -106,8 +106,10 @@ bool serializeToXMLFile(const T& input, const std::string& filename, const std:: std::ofstream out_archive_stream(filename.c_str()); if (!out_archive_stream.is_open()) return false; - boost::archive::xml_oarchive out_archive(out_archive_stream); - out_archive << boost::serialization::make_nvp(name.c_str(), input);; + { + boost::archive::xml_oarchive out_archive(out_archive_stream); + out_archive << boost::serialization::make_nvp(name.c_str(), input);; + } out_archive_stream.close(); return true; } @@ -117,8 +119,10 @@ bool deserializeFromXMLFile(const std::string& filename, T& output, const std::s std::ifstream in_archive_stream(filename.c_str()); if (!in_archive_stream.is_open()) return false; - boost::archive::xml_iarchive in_archive(in_archive_stream); - in_archive >> boost::serialization::make_nvp(name.c_str(), output); + { + boost::archive::xml_iarchive in_archive(in_archive_stream); + in_archive >> boost::serialization::make_nvp(name.c_str(), output); + } in_archive_stream.close(); return true; } diff --git a/gtsam/base/serializationTestHelpers.h b/gtsam/base/serializationTestHelpers.h index 06b3462e9..178b256a6 100644 --- a/gtsam/base/serializationTestHelpers.h +++ b/gtsam/base/serializationTestHelpers.h @@ -26,6 +26,7 @@ #include #include +#include // whether to print the serialized text to stdout @@ -40,6 +41,13 @@ T create() { return T(); } +std::string resetFilesystem() { + // Create files in folder in build folder + boost::filesystem::remove_all("actual"); + boost::filesystem::create_directory("actual"); + return "actual/"; +} + // Templated round-trip serialization template void roundtrip(const T& input, T& output) { @@ -50,11 +58,22 @@ void roundtrip(const T& input, T& output) { deserialize(serialized, output); } +// Templated round-trip serialization using a file +template +void roundtripFile(const T& input, T& output) { + std::string path = resetFilesystem(); + + serializeToFile(input, path + "graph.dat"); + deserializeFromFile(path + "graph.dat", output); +} + // This version requires equality operator template bool equality(const T& input = T()) { T output = create(); roundtrip(input,output); + if (input != output) return false; + roundtripFile(input,output); return input==output; } @@ -63,6 +82,8 @@ template bool equalsObj(const T& input = T()) { T output = create(); roundtrip(input,output); + if (!assert_equal(input, output)) return false; + roundtripFile(input,output); return assert_equal(input, output); } @@ -71,6 +92,8 @@ template bool equalsDereferenced(const T& input) { T output = create(); roundtrip(input,output); + if (!input->equals(*output)) return false; + roundtripFile(input,output); return input->equals(*output); } @@ -85,11 +108,25 @@ void roundtripXML(const T& input, T& output) { deserializeXML(serialized, output); } +// Templated round-trip serialization using XML File +template +void roundtripXMLFile(const T& input, T& output) { + std::string path = resetFilesystem(); + + // Serialize + serializeToXMLFile(input, path + "graph.xml"); + + // De-serialize + deserializeFromXMLFile(path + "graph.xml", output); +} + // This version requires equality operator template bool equalityXML(const T& input = T()) { T output = create(); roundtripXML(input,output); + if (input != output) return false; + roundtripXMLFile(input,output); return input==output; } @@ -98,6 +135,8 @@ template bool equalsXML(const T& input = T()) { T output = create(); roundtripXML(input,output); + if (!assert_equal(input, output)) return false; + roundtripXMLFile(input, output); return assert_equal(input, output); } @@ -106,6 +145,8 @@ template bool equalsDereferencedXML(const T& input = T()) { T output = create(); roundtripXML(input,output); + if (!input->equals(*output)) return false; + roundtripXMLFile(input, output); return input->equals(*output); } @@ -120,11 +161,23 @@ void roundtripBinary(const T& input, T& output) { deserializeBinary(serialized, output); } +// Templated round-trip serialization using XML +template +void roundtripBinaryFile(const T& input, T& output) { + std::string path = resetFilesystem(); + // Serialize + serializeToBinaryFile(input, path + "graph.bin"); + // De-serialize + deserializeFromBinaryFile(path + "graph.bin", output); +} + // This version requires equality operator template bool equalityBinary(const T& input = T()) { T output = create(); roundtripBinary(input,output); + if (input != output) return false; + roundtripBinaryFile(input,output); return input==output; } @@ -133,6 +186,8 @@ template bool equalsBinary(const T& input = T()) { T output = create(); roundtripBinary(input,output); + if (!assert_equal(input, output)) return false; + roundtripBinaryFile(input,output); return assert_equal(input, output); } @@ -141,6 +196,8 @@ template bool equalsDereferencedBinary(const T& input = T()) { T output = create(); roundtripBinary(input,output); + if (!input->equals(*output)) return false; + roundtripBinaryFile(input,output); return input->equals(*output); } From c94736b6b6959820efe628d52cc8cd17c0269cdc Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Tue, 9 Jun 2020 02:39:54 -0400 Subject: [PATCH 02/12] bypass assert_equal tests for file roundtrips --- gtsam/base/serializationTestHelpers.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/gtsam/base/serializationTestHelpers.h b/gtsam/base/serializationTestHelpers.h index 178b256a6..d29899072 100644 --- a/gtsam/base/serializationTestHelpers.h +++ b/gtsam/base/serializationTestHelpers.h @@ -82,8 +82,6 @@ template bool equalsObj(const T& input = T()) { T output = create(); roundtrip(input,output); - if (!assert_equal(input, output)) return false; - roundtripFile(input,output); return assert_equal(input, output); } @@ -135,8 +133,6 @@ template bool equalsXML(const T& input = T()) { T output = create(); roundtripXML(input,output); - if (!assert_equal(input, output)) return false; - roundtripXMLFile(input, output); return assert_equal(input, output); } @@ -186,8 +182,6 @@ template bool equalsBinary(const T& input = T()) { T output = create(); roundtripBinary(input,output); - if (!assert_equal(input, output)) return false; - roundtripBinaryFile(input,output); return assert_equal(input, output); } From 686b0effbf4e6d222ed26c17d2c00bb7dc878551 Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Tue, 9 Jun 2020 02:48:53 -0400 Subject: [PATCH 03/12] better comments on serializationTestHelper functions --- gtsam/base/serializationTestHelpers.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gtsam/base/serializationTestHelpers.h b/gtsam/base/serializationTestHelpers.h index d29899072..031a6278f 100644 --- a/gtsam/base/serializationTestHelpers.h +++ b/gtsam/base/serializationTestHelpers.h @@ -41,8 +41,8 @@ T create() { return T(); } +// Creates or empties a folder in the build folder and returns the relative path std::string resetFilesystem() { - // Create files in folder in build folder boost::filesystem::remove_all("actual"); boost::filesystem::create_directory("actual"); return "actual/"; @@ -157,7 +157,7 @@ void roundtripBinary(const T& input, T& output) { deserializeBinary(serialized, output); } -// Templated round-trip serialization using XML +// Templated round-trip serialization using Binary file template void roundtripBinaryFile(const T& input, T& output) { std::string path = resetFilesystem(); From 8a9ade00254e403f15471b17ce9fe75fcf0da084 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Thu, 11 Jun 2020 10:58:44 -0500 Subject: [PATCH 04/12] remove extra semicolon --- gtsam/base/serialization.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtsam/base/serialization.h b/gtsam/base/serialization.h index 088029903..191e0ecfb 100644 --- a/gtsam/base/serialization.h +++ b/gtsam/base/serialization.h @@ -108,7 +108,7 @@ bool serializeToXMLFile(const T& input, const std::string& filename, const std:: return false; { boost::archive::xml_oarchive out_archive(out_archive_stream); - out_archive << boost::serialization::make_nvp(name.c_str(), input);; + out_archive << boost::serialization::make_nvp(name.c_str(), input); } out_archive_stream.close(); return true; From a796f74b8004d7cc12acaf3d20c9b3827a590770 Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Mon, 22 Jun 2020 20:24:32 -0400 Subject: [PATCH 05/12] use boost paths append to have platform agnostic path separators --- gtsam/base/serializationTestHelpers.h | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/gtsam/base/serializationTestHelpers.h b/gtsam/base/serializationTestHelpers.h index 031a6278f..602fd68f6 100644 --- a/gtsam/base/serializationTestHelpers.h +++ b/gtsam/base/serializationTestHelpers.h @@ -42,10 +42,11 @@ T create() { } // Creates or empties a folder in the build folder and returns the relative path -std::string resetFilesystem() { - boost::filesystem::remove_all("actual"); - boost::filesystem::create_directory("actual"); - return "actual/"; +boost::filesystem::path resetFilesystem( + boost::filesystem::path folder = "actual") { + boost::filesystem::remove_all(folder); + boost::filesystem::create_directory(folder); + return folder; } // Templated round-trip serialization @@ -61,10 +62,10 @@ void roundtrip(const T& input, T& output) { // Templated round-trip serialization using a file template void roundtripFile(const T& input, T& output) { - std::string path = resetFilesystem(); + boost::filesystem::path path = resetFilesystem()/"graph.dat"; - serializeToFile(input, path + "graph.dat"); - deserializeFromFile(path + "graph.dat", output); + serializeToFile(input, path.string()); + deserializeFromFile(path.string(), output); } // This version requires equality operator @@ -109,13 +110,13 @@ void roundtripXML(const T& input, T& output) { // Templated round-trip serialization using XML File template void roundtripXMLFile(const T& input, T& output) { - std::string path = resetFilesystem(); + boost::filesystem::path path = resetFilesystem()/"graph.xml"; // Serialize - serializeToXMLFile(input, path + "graph.xml"); + serializeToXMLFile(input, path.string()); // De-serialize - deserializeFromXMLFile(path + "graph.xml", output); + deserializeFromXMLFile(path.string(), output); } // This version requires equality operator @@ -160,11 +161,11 @@ void roundtripBinary(const T& input, T& output) { // Templated round-trip serialization using Binary file template void roundtripBinaryFile(const T& input, T& output) { - std::string path = resetFilesystem(); + boost::filesystem::path path = resetFilesystem()/"graph.bin"; // Serialize - serializeToBinaryFile(input, path + "graph.bin"); + serializeToBinaryFile(input, path.string()); // De-serialize - deserializeFromBinaryFile(path + "graph.bin", output); + deserializeFromBinaryFile(path.string(), output); } // This version requires equality operator From 327cbc515ff93db6f6547ed3deead1bdf9cfdb2e Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Mon, 22 Jun 2020 21:15:07 -0400 Subject: [PATCH 06/12] Separate stream creation and serialization Recommended by @ProfFan in #343 with the objective of making (de)serialize to string and to file more similar --- gtsam/base/serialization.h | 86 +++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 33 deletions(-) diff --git a/gtsam/base/serialization.h b/gtsam/base/serialization.h index 191e0ecfb..031ee3a3e 100644 --- a/gtsam/base/serialization.h +++ b/gtsam/base/serialization.h @@ -44,18 +44,28 @@ namespace gtsam { // Serialization directly to strings in compressed format template -std::string serialize(const T& input) { - std::ostringstream out_archive_stream; +void serialize(const T& input, std::ostream & out_archive_stream) { boost::archive::text_oarchive out_archive(out_archive_stream); out_archive << input; +} + +template +void deserialize(std::istream & in_archive_stream, T& output) { + boost::archive::text_iarchive in_archive(in_archive_stream); + in_archive >> output; +} + +template +std::string serialize(const T& input) { + std::ostringstream out_archive_stream; + serialize(input, out_archive_stream); return out_archive_stream.str(); } template void deserialize(const std::string& serialized, T& output) { std::istringstream in_archive_stream(serialized); - boost::archive::text_iarchive in_archive(in_archive_stream); - in_archive >> output; + deserialize(in_archive_stream, output); } template @@ -63,8 +73,7 @@ bool serializeToFile(const T& input, const std::string& filename) { std::ofstream out_archive_stream(filename.c_str()); if (!out_archive_stream.is_open()) return false; - boost::archive::text_oarchive out_archive(out_archive_stream); - out_archive << input; + serialize(input, out_archive_stream); out_archive_stream.close(); return true; } @@ -74,31 +83,37 @@ bool deserializeFromFile(const std::string& filename, T& output) { std::ifstream in_archive_stream(filename.c_str()); if (!in_archive_stream.is_open()) return false; - boost::archive::text_iarchive in_archive(in_archive_stream); - in_archive >> output; + deserialize(in_archive_stream, output); in_archive_stream.close(); return true; } // Serialization to XML format with named structures +template +void serializeXML(const T& input, std::ostream& out_archive_stream, + const std::string& name = "data") { + boost::archive::xml_oarchive out_archive(out_archive_stream); + out_archive << boost::serialization::make_nvp(name.c_str(), input); +} + +template +void deserializeXML(std::istream& in_archive_stream, T& output, + const std::string& name = "data") { + boost::archive::xml_iarchive in_archive(in_archive_stream); + in_archive >> boost::serialization::make_nvp(name.c_str(), output); +} + template std::string serializeXML(const T& input, const std::string& name="data") { std::ostringstream out_archive_stream; - // braces to flush out_archive as it goes out of scope before taking str() - // fixes crash with boost 1.66-1.68 - // see https://github.com/boostorg/serialization/issues/82 - { - boost::archive::xml_oarchive out_archive(out_archive_stream); - out_archive << boost::serialization::make_nvp(name.c_str(), input); - } + serializeXML(input, out_archive_stream, name); return out_archive_stream.str(); } template void deserializeXML(const std::string& serialized, T& output, const std::string& name="data") { std::istringstream in_archive_stream(serialized); - boost::archive::xml_iarchive in_archive(in_archive_stream); - in_archive >> boost::serialization::make_nvp(name.c_str(), output); + deserializeXML(in_archive_stream, output, name); } template @@ -106,10 +121,7 @@ bool serializeToXMLFile(const T& input, const std::string& filename, const std:: std::ofstream out_archive_stream(filename.c_str()); if (!out_archive_stream.is_open()) return false; - { - boost::archive::xml_oarchive out_archive(out_archive_stream); - out_archive << boost::serialization::make_nvp(name.c_str(), input); - } + serializeXML(input, out_archive_stream, name); out_archive_stream.close(); return true; } @@ -119,28 +131,38 @@ bool deserializeFromXMLFile(const std::string& filename, T& output, const std::s std::ifstream in_archive_stream(filename.c_str()); if (!in_archive_stream.is_open()) return false; - { - boost::archive::xml_iarchive in_archive(in_archive_stream); - in_archive >> boost::serialization::make_nvp(name.c_str(), output); - } + deserializeXML(in_archive_stream, output, name); in_archive_stream.close(); return true; } +// Serialization to binary format with named structures +template +void serializeBinary(const T& input, std::ostream& out_archive_stream, + const std::string& name = "data") { + boost::archive::binary_oarchive out_archive(out_archive_stream); + out_archive << boost::serialization::make_nvp(name.c_str(), input); +} + +template +void deserializeBinary(std::istream& in_archive_stream, T& output, + const std::string& name = "data") { + boost::archive::binary_iarchive in_archive(in_archive_stream); + in_archive >> boost::serialization::make_nvp(name.c_str(), output); +} + // Serialization to binary format with named structures template std::string serializeBinary(const T& input, const std::string& name="data") { std::ostringstream out_archive_stream; - boost::archive::binary_oarchive out_archive(out_archive_stream); - out_archive << boost::serialization::make_nvp(name.c_str(), input); + serializeBinary(input, out_archive_stream, name); return out_archive_stream.str(); } template void deserializeBinary(const std::string& serialized, T& output, const std::string& name="data") { std::istringstream in_archive_stream(serialized); - boost::archive::binary_iarchive in_archive(in_archive_stream); - in_archive >> boost::serialization::make_nvp(name.c_str(), output); + deserializeBinary(in_archive_stream, output, name); } template @@ -148,8 +170,7 @@ bool serializeToBinaryFile(const T& input, const std::string& filename, const st std::ofstream out_archive_stream(filename.c_str()); if (!out_archive_stream.is_open()) return false; - boost::archive::binary_oarchive out_archive(out_archive_stream); - out_archive << boost::serialization::make_nvp(name.c_str(), input); + serializeBinary(input, out_archive_stream, name); out_archive_stream.close(); return true; } @@ -159,8 +180,7 @@ bool deserializeFromBinaryFile(const std::string& filename, T& output, const std std::ifstream in_archive_stream(filename.c_str()); if (!in_archive_stream.is_open()) return false; - boost::archive::binary_iarchive in_archive(in_archive_stream); - in_archive >> boost::serialization::make_nvp(name.c_str(), output); + deserializeBinary(in_archive_stream, output, name); in_archive_stream.close(); return true; } From a4737d47066c0ee0e533bea393f4da14347e9abf Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Mon, 22 Jun 2020 21:18:43 -0400 Subject: [PATCH 07/12] formatting to Google style --- gtsam/base/serialization.h | 78 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/gtsam/base/serialization.h b/gtsam/base/serialization.h index 031ee3a3e..aaf06275c 100644 --- a/gtsam/base/serialization.h +++ b/gtsam/base/serialization.h @@ -43,46 +43,44 @@ namespace gtsam { // Serialization directly to strings in compressed format -template -void serialize(const T& input, std::ostream & out_archive_stream) { +template +void serialize(const T& input, std::ostream& out_archive_stream) { boost::archive::text_oarchive out_archive(out_archive_stream); out_archive << input; } -template -void deserialize(std::istream & in_archive_stream, T& output) { +template +void deserialize(std::istream& in_archive_stream, T& output) { boost::archive::text_iarchive in_archive(in_archive_stream); in_archive >> output; } -template +template std::string serialize(const T& input) { std::ostringstream out_archive_stream; serialize(input, out_archive_stream); return out_archive_stream.str(); } -template +template void deserialize(const std::string& serialized, T& output) { std::istringstream in_archive_stream(serialized); deserialize(in_archive_stream, output); } -template +template bool serializeToFile(const T& input, const std::string& filename) { std::ofstream out_archive_stream(filename.c_str()); - if (!out_archive_stream.is_open()) - return false; + if (!out_archive_stream.is_open()) return false; serialize(input, out_archive_stream); out_archive_stream.close(); return true; } -template +template bool deserializeFromFile(const std::string& filename, T& output) { std::ifstream in_archive_stream(filename.c_str()); - if (!in_archive_stream.is_open()) - return false; + if (!in_archive_stream.is_open()) return false; deserialize(in_archive_stream, output); in_archive_stream.close(); return true; @@ -103,34 +101,36 @@ void deserializeXML(std::istream& in_archive_stream, T& output, in_archive >> boost::serialization::make_nvp(name.c_str(), output); } -template -std::string serializeXML(const T& input, const std::string& name="data") { +template +std::string serializeXML(const T& input, + const std::string& name = "data") { std::ostringstream out_archive_stream; serializeXML(input, out_archive_stream, name); return out_archive_stream.str(); } -template -void deserializeXML(const std::string& serialized, T& output, const std::string& name="data") { +template +void deserializeXML(const std::string& serialized, T& output, + const std::string& name = "data") { std::istringstream in_archive_stream(serialized); deserializeXML(in_archive_stream, output, name); } -template -bool serializeToXMLFile(const T& input, const std::string& filename, const std::string& name="data") { +template +bool serializeToXMLFile(const T& input, const std::string& filename, + const std::string& name = "data") { std::ofstream out_archive_stream(filename.c_str()); - if (!out_archive_stream.is_open()) - return false; + if (!out_archive_stream.is_open()) return false; serializeXML(input, out_archive_stream, name); out_archive_stream.close(); return true; } -template -bool deserializeFromXMLFile(const std::string& filename, T& output, const std::string& name="data") { +template +bool deserializeFromXMLFile(const std::string& filename, T& output, + const std::string& name = "data") { std::ifstream in_archive_stream(filename.c_str()); - if (!in_archive_stream.is_open()) - return false; + if (!in_archive_stream.is_open()) return false; deserializeXML(in_archive_stream, output, name); in_archive_stream.close(); return true; @@ -139,7 +139,7 @@ bool deserializeFromXMLFile(const std::string& filename, T& output, const std::s // Serialization to binary format with named structures template void serializeBinary(const T& input, std::ostream& out_archive_stream, - const std::string& name = "data") { + const std::string& name = "data") { boost::archive::binary_oarchive out_archive(out_archive_stream); out_archive << boost::serialization::make_nvp(name.c_str(), input); } @@ -152,37 +152,39 @@ void deserializeBinary(std::istream& in_archive_stream, T& output, } // Serialization to binary format with named structures -template -std::string serializeBinary(const T& input, const std::string& name="data") { +template +std::string serializeBinary(const T& input, + const std::string& name = "data") { std::ostringstream out_archive_stream; serializeBinary(input, out_archive_stream, name); return out_archive_stream.str(); } -template -void deserializeBinary(const std::string& serialized, T& output, const std::string& name="data") { +template +void deserializeBinary(const std::string& serialized, T& output, + const std::string& name = "data") { std::istringstream in_archive_stream(serialized); deserializeBinary(in_archive_stream, output, name); } -template -bool serializeToBinaryFile(const T& input, const std::string& filename, const std::string& name="data") { +template +bool serializeToBinaryFile(const T& input, const std::string& filename, + const std::string& name = "data") { std::ofstream out_archive_stream(filename.c_str()); - if (!out_archive_stream.is_open()) - return false; + if (!out_archive_stream.is_open()) return false; serializeBinary(input, out_archive_stream, name); out_archive_stream.close(); return true; } -template -bool deserializeFromBinaryFile(const std::string& filename, T& output, const std::string& name="data") { +template +bool deserializeFromBinaryFile(const std::string& filename, T& output, + const std::string& name = "data") { std::ifstream in_archive_stream(filename.c_str()); - if (!in_archive_stream.is_open()) - return false; + if (!in_archive_stream.is_open()) return false; deserializeBinary(in_archive_stream, output, name); in_archive_stream.close(); return true; } -} // \namespace gtsam +} // namespace gtsam From 82db82bbf5d46bb64dc50ba52507b39ee7f061ed Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Mon, 22 Jun 2020 23:08:39 -0400 Subject: [PATCH 08/12] fixed unit test failure on `testSerializationBase` object `output` was getting reused, but should be re-loaded into a "blank" object each time. --- gtsam/base/serializationTestHelpers.h | 46 ++++++++++++--------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/gtsam/base/serializationTestHelpers.h b/gtsam/base/serializationTestHelpers.h index 602fd68f6..bac166e5b 100644 --- a/gtsam/base/serializationTestHelpers.h +++ b/gtsam/base/serializationTestHelpers.h @@ -44,8 +44,8 @@ T create() { // Creates or empties a folder in the build folder and returns the relative path boost::filesystem::path resetFilesystem( boost::filesystem::path folder = "actual") { - boost::filesystem::remove_all(folder); - boost::filesystem::create_directory(folder); + // boost::filesystem::remove_all(folder); + // boost::filesystem::create_directory(folder); return folder; } @@ -71,11 +71,10 @@ void roundtripFile(const T& input, T& output) { // This version requires equality operator template bool equality(const T& input = T()) { - T output = create(); + T output = create(), outputf = create(); roundtrip(input,output); - if (input != output) return false; - roundtripFile(input,output); - return input==output; + roundtripFile(input,outputf); + return (input==output) && (input==outputf); } // This version requires Testable @@ -89,11 +88,10 @@ bool equalsObj(const T& input = T()) { // De-referenced version for pointers, requires equals method template bool equalsDereferenced(const T& input) { - T output = create(); + T output = create(), outputf = create(); roundtrip(input,output); - if (!input->equals(*output)) return false; - roundtripFile(input,output); - return input->equals(*output); + roundtripFile(input,outputf); + return (input->equals(*output)) && (input->equals(*outputf)); } // Templated round-trip serialization using XML @@ -122,11 +120,10 @@ void roundtripXMLFile(const T& input, T& output) { // This version requires equality operator template bool equalityXML(const T& input = T()) { - T output = create(); + T output = create(), outputf = create(); roundtripXML(input,output); - if (input != output) return false; - roundtripXMLFile(input,output); - return input==output; + roundtripXMLFile(input,outputf); + return (input==output) && (input==outputf); } // This version requires Testable @@ -140,11 +137,10 @@ bool equalsXML(const T& input = T()) { // This version is for pointers, requires equals method template bool equalsDereferencedXML(const T& input = T()) { - T output = create(); + T output = create(), outputf = create(); roundtripXML(input,output); - if (!input->equals(*output)) return false; - roundtripXMLFile(input, output); - return input->equals(*output); + roundtripXMLFile(input, outputf); + return (input->equals(*output)) && (input->equals(*outputf)); } // Templated round-trip serialization using XML @@ -171,11 +167,10 @@ void roundtripBinaryFile(const T& input, T& output) { // This version requires equality operator template bool equalityBinary(const T& input = T()) { - T output = create(); + T output = create(), outputf = create(); roundtripBinary(input,output); - if (input != output) return false; - roundtripBinaryFile(input,output); - return input==output; + roundtripBinaryFile(input,outputf); + return (input==output) && (input==outputf); } // This version requires Testable @@ -189,11 +184,10 @@ bool equalsBinary(const T& input = T()) { // This version is for pointers, requires equals method template bool equalsDereferencedBinary(const T& input = T()) { - T output = create(); + T output = create(), outputf = create(); roundtripBinary(input,output); - if (!input->equals(*output)) return false; - roundtripBinaryFile(input,output); - return input->equals(*output); + roundtripBinaryFile(input,outputf); + return (input->equals(*output)) && (input->equals(*outputf)); } } // \namespace serializationTestHelpers From a0a3b8f459df37349a7b4af7d6b52016142204cb Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Tue, 23 Jun 2020 09:52:29 -0400 Subject: [PATCH 09/12] reset filesystem - forgot to uncomment these after debugging --- gtsam/base/serializationTestHelpers.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gtsam/base/serializationTestHelpers.h b/gtsam/base/serializationTestHelpers.h index bac166e5b..5d67b2687 100644 --- a/gtsam/base/serializationTestHelpers.h +++ b/gtsam/base/serializationTestHelpers.h @@ -44,8 +44,8 @@ T create() { // Creates or empties a folder in the build folder and returns the relative path boost::filesystem::path resetFilesystem( boost::filesystem::path folder = "actual") { - // boost::filesystem::remove_all(folder); - // boost::filesystem::create_directory(folder); + boost::filesystem::remove_all(folder); + boost::filesystem::create_directory(folder); return folder; } From de7332fceac3224d4ea693a8f1631300b3345c74 Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Wed, 24 Jun 2020 02:39:44 -0400 Subject: [PATCH 10/12] remove file roundtrip test for pointers --- gtsam/base/serializationTestHelpers.h | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/gtsam/base/serializationTestHelpers.h b/gtsam/base/serializationTestHelpers.h index 5d67b2687..78f9d6d10 100644 --- a/gtsam/base/serializationTestHelpers.h +++ b/gtsam/base/serializationTestHelpers.h @@ -88,10 +88,9 @@ bool equalsObj(const T& input = T()) { // De-referenced version for pointers, requires equals method template bool equalsDereferenced(const T& input) { - T output = create(), outputf = create(); + T output = create(); roundtrip(input,output); - roundtripFile(input,outputf); - return (input->equals(*output)) && (input->equals(*outputf)); + return input->equals(*output); } // Templated round-trip serialization using XML @@ -137,10 +136,9 @@ bool equalsXML(const T& input = T()) { // This version is for pointers, requires equals method template bool equalsDereferencedXML(const T& input = T()) { - T output = create(), outputf = create(); + T output = create(); roundtripXML(input,output); - roundtripXMLFile(input, outputf); - return (input->equals(*output)) && (input->equals(*outputf)); + return input->equals(*output); } // Templated round-trip serialization using XML @@ -184,10 +182,9 @@ bool equalsBinary(const T& input = T()) { // This version is for pointers, requires equals method template bool equalsDereferencedBinary(const T& input = T()) { - T output = create(), outputf = create(); + T output = create(); roundtripBinary(input,output); - roundtripBinaryFile(input,outputf); - return (input->equals(*output)) && (input->equals(*outputf)); + return input->equals(*output); } } // \namespace serializationTestHelpers From 6d75e992e896f54a1bd3956ff5f8cd1dddf5e814 Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Wed, 24 Jun 2020 14:16:00 -0400 Subject: [PATCH 11/12] serialization docstrings --- gtsam/base/serialization.h | 37 ++++++++++++++++++++++++--- gtsam/base/serializationTestHelpers.h | 17 +----------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/gtsam/base/serialization.h b/gtsam/base/serialization.h index aaf06275c..1a319ab17 100644 --- a/gtsam/base/serialization.h +++ b/gtsam/base/serialization.h @@ -42,19 +42,25 @@ namespace gtsam { -// Serialization directly to strings in compressed format +/** @name Standard serialization + * Serialization in default compressed format + */ +///@{ +/// serializes to a stream template void serialize(const T& input, std::ostream& out_archive_stream) { boost::archive::text_oarchive out_archive(out_archive_stream); out_archive << input; } +/// deserializes from a stream template void deserialize(std::istream& in_archive_stream, T& output) { boost::archive::text_iarchive in_archive(in_archive_stream); in_archive >> output; } +/// serializes to a string template std::string serialize(const T& input) { std::ostringstream out_archive_stream; @@ -62,12 +68,14 @@ std::string serialize(const T& input) { return out_archive_stream.str(); } +/// deserializes from a string template void deserialize(const std::string& serialized, T& output) { std::istringstream in_archive_stream(serialized); deserialize(in_archive_stream, output); } +/// serializes to a file template bool serializeToFile(const T& input, const std::string& filename) { std::ofstream out_archive_stream(filename.c_str()); @@ -77,6 +85,7 @@ bool serializeToFile(const T& input, const std::string& filename) { return true; } +/// deserializes from a file template bool deserializeFromFile(const std::string& filename, T& output) { std::ifstream in_archive_stream(filename.c_str()); @@ -85,8 +94,13 @@ bool deserializeFromFile(const std::string& filename, T& output) { in_archive_stream.close(); return true; } +///@} -// Serialization to XML format with named structures +/** @name XML Serialization + * Serialization to XML format with named structures + */ +///@{ +/// serializes to a stream in XML template void serializeXML(const T& input, std::ostream& out_archive_stream, const std::string& name = "data") { @@ -94,6 +108,7 @@ void serializeXML(const T& input, std::ostream& out_archive_stream, out_archive << boost::serialization::make_nvp(name.c_str(), input); } +/// deserializes from a stream in XML template void deserializeXML(std::istream& in_archive_stream, T& output, const std::string& name = "data") { @@ -101,6 +116,7 @@ void deserializeXML(std::istream& in_archive_stream, T& output, in_archive >> boost::serialization::make_nvp(name.c_str(), output); } +/// serializes to a string in XML template std::string serializeXML(const T& input, const std::string& name = "data") { @@ -109,6 +125,7 @@ std::string serializeXML(const T& input, return out_archive_stream.str(); } +/// deserializes from a string in XML template void deserializeXML(const std::string& serialized, T& output, const std::string& name = "data") { @@ -116,6 +133,7 @@ void deserializeXML(const std::string& serialized, T& output, deserializeXML(in_archive_stream, output, name); } +/// serializes to an XML file template bool serializeToXMLFile(const T& input, const std::string& filename, const std::string& name = "data") { @@ -126,6 +144,7 @@ bool serializeToXMLFile(const T& input, const std::string& filename, return true; } +/// deserializes from an XML file template bool deserializeFromXMLFile(const std::string& filename, T& output, const std::string& name = "data") { @@ -135,8 +154,13 @@ bool deserializeFromXMLFile(const std::string& filename, T& output, in_archive_stream.close(); return true; } +///@} -// Serialization to binary format with named structures +/** @name Binary Serialization + * Serialization to binary format with named structures + */ +///@{ +/// serializes to a stream in binary template void serializeBinary(const T& input, std::ostream& out_archive_stream, const std::string& name = "data") { @@ -144,6 +168,7 @@ void serializeBinary(const T& input, std::ostream& out_archive_stream, out_archive << boost::serialization::make_nvp(name.c_str(), input); } +/// deserializes from a stream in binary template void deserializeBinary(std::istream& in_archive_stream, T& output, const std::string& name = "data") { @@ -151,7 +176,7 @@ void deserializeBinary(std::istream& in_archive_stream, T& output, in_archive >> boost::serialization::make_nvp(name.c_str(), output); } -// Serialization to binary format with named structures +/// serializes to a string in binary template std::string serializeBinary(const T& input, const std::string& name = "data") { @@ -160,6 +185,7 @@ std::string serializeBinary(const T& input, return out_archive_stream.str(); } +/// deserializes from a string in binary template void deserializeBinary(const std::string& serialized, T& output, const std::string& name = "data") { @@ -167,6 +193,7 @@ void deserializeBinary(const std::string& serialized, T& output, deserializeBinary(in_archive_stream, output, name); } +/// serializes to a binary file template bool serializeToBinaryFile(const T& input, const std::string& filename, const std::string& name = "data") { @@ -177,6 +204,7 @@ bool serializeToBinaryFile(const T& input, const std::string& filename, return true; } +/// deserializes from a binary file template bool deserializeFromBinaryFile(const std::string& filename, T& output, const std::string& name = "data") { @@ -186,5 +214,6 @@ bool deserializeFromBinaryFile(const std::string& filename, T& output, in_archive_stream.close(); return true; } +///@} } // namespace gtsam diff --git a/gtsam/base/serializationTestHelpers.h b/gtsam/base/serializationTestHelpers.h index 78f9d6d10..5994a5e51 100644 --- a/gtsam/base/serializationTestHelpers.h +++ b/gtsam/base/serializationTestHelpers.h @@ -52,10 +52,8 @@ boost::filesystem::path resetFilesystem( // Templated round-trip serialization template void roundtrip(const T& input, T& output) { - // Serialize std::string serialized = serialize(input); if (verbose) std::cout << serialized << std::endl << std::endl; - deserialize(serialized, output); } @@ -63,12 +61,11 @@ void roundtrip(const T& input, T& output) { template void roundtripFile(const T& input, T& output) { boost::filesystem::path path = resetFilesystem()/"graph.dat"; - serializeToFile(input, path.string()); deserializeFromFile(path.string(), output); } -// This version requires equality operator +// This version requires equality operator and uses string and file round-trips template bool equality(const T& input = T()) { T output = create(), outputf = create(); @@ -96,11 +93,8 @@ bool equalsDereferenced(const T& input) { // Templated round-trip serialization using XML template void roundtripXML(const T& input, T& output) { - // Serialize std::string serialized = serializeXML(input); if (verbose) std::cout << serialized << std::endl << std::endl; - - // De-serialize deserializeXML(serialized, output); } @@ -108,11 +102,7 @@ void roundtripXML(const T& input, T& output) { template void roundtripXMLFile(const T& input, T& output) { boost::filesystem::path path = resetFilesystem()/"graph.xml"; - - // Serialize serializeToXMLFile(input, path.string()); - - // De-serialize deserializeFromXMLFile(path.string(), output); } @@ -144,11 +134,8 @@ bool equalsDereferencedXML(const T& input = T()) { // Templated round-trip serialization using XML template void roundtripBinary(const T& input, T& output) { - // Serialize std::string serialized = serializeBinary(input); if (verbose) std::cout << serialized << std::endl << std::endl; - - // De-serialize deserializeBinary(serialized, output); } @@ -156,9 +143,7 @@ void roundtripBinary(const T& input, T& output) { template void roundtripBinaryFile(const T& input, T& output) { boost::filesystem::path path = resetFilesystem()/"graph.bin"; - // Serialize serializeToBinaryFile(input, path.string()); - // De-serialize deserializeFromBinaryFile(path.string(), output); } From b37be7d640567be9b78c699d158c88035d4f789e Mon Sep 17 00:00:00 2001 From: Gerry Chen Date: Wed, 24 Jun 2020 14:33:08 -0400 Subject: [PATCH 12/12] rename serialization functions with less ambiguous names According to Varun's suggestion. Note: string functions should be automatically inlined by compiler to avoid passing big strings. --- gtsam/base/serialization.h | 88 +++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/gtsam/base/serialization.h b/gtsam/base/serialization.h index 1a319ab17..f589ecc5e 100644 --- a/gtsam/base/serialization.h +++ b/gtsam/base/serialization.h @@ -48,31 +48,31 @@ namespace gtsam { ///@{ /// serializes to a stream template -void serialize(const T& input, std::ostream& out_archive_stream) { +void serializeToStream(const T& input, std::ostream& out_archive_stream) { boost::archive::text_oarchive out_archive(out_archive_stream); out_archive << input; } /// deserializes from a stream template -void deserialize(std::istream& in_archive_stream, T& output) { +void deserializeFromStream(std::istream& in_archive_stream, T& output) { boost::archive::text_iarchive in_archive(in_archive_stream); in_archive >> output; } /// serializes to a string template -std::string serialize(const T& input) { +std::string serializeToString(const T& input) { std::ostringstream out_archive_stream; - serialize(input, out_archive_stream); + serializeToStream(input, out_archive_stream); return out_archive_stream.str(); } /// deserializes from a string template -void deserialize(const std::string& serialized, T& output) { +void deserializeFromString(const std::string& serialized, T& output) { std::istringstream in_archive_stream(serialized); - deserialize(in_archive_stream, output); + deserializeFromStream(in_archive_stream, output); } /// serializes to a file @@ -80,7 +80,7 @@ template bool serializeToFile(const T& input, const std::string& filename) { std::ofstream out_archive_stream(filename.c_str()); if (!out_archive_stream.is_open()) return false; - serialize(input, out_archive_stream); + serializeToStream(input, out_archive_stream); out_archive_stream.close(); return true; } @@ -90,10 +90,22 @@ template bool deserializeFromFile(const std::string& filename, T& output) { std::ifstream in_archive_stream(filename.c_str()); if (!in_archive_stream.is_open()) return false; - deserialize(in_archive_stream, output); + deserializeFromStream(in_archive_stream, output); in_archive_stream.close(); return true; } + +/// serializes to a string +template +std::string serialize(const T& input) { + return serializeToString(input); +} + +/// deserializes from a string +template +void deserialize(const std::string& serialized, T& output) { + deserializeFromString(serialized, output); +} ///@} /** @name XML Serialization @@ -102,7 +114,7 @@ bool deserializeFromFile(const std::string& filename, T& output) { ///@{ /// serializes to a stream in XML template -void serializeXML(const T& input, std::ostream& out_archive_stream, +void serializeToXMLStream(const T& input, std::ostream& out_archive_stream, const std::string& name = "data") { boost::archive::xml_oarchive out_archive(out_archive_stream); out_archive << boost::serialization::make_nvp(name.c_str(), input); @@ -110,7 +122,7 @@ void serializeXML(const T& input, std::ostream& out_archive_stream, /// deserializes from a stream in XML template -void deserializeXML(std::istream& in_archive_stream, T& output, +void deserializeFromXMLStream(std::istream& in_archive_stream, T& output, const std::string& name = "data") { boost::archive::xml_iarchive in_archive(in_archive_stream); in_archive >> boost::serialization::make_nvp(name.c_str(), output); @@ -118,19 +130,19 @@ void deserializeXML(std::istream& in_archive_stream, T& output, /// serializes to a string in XML template -std::string serializeXML(const T& input, +std::string serializeToXMLString(const T& input, const std::string& name = "data") { std::ostringstream out_archive_stream; - serializeXML(input, out_archive_stream, name); + serializeToXMLStream(input, out_archive_stream, name); return out_archive_stream.str(); } /// deserializes from a string in XML template -void deserializeXML(const std::string& serialized, T& output, +void deserializeFromXMLString(const std::string& serialized, T& output, const std::string& name = "data") { std::istringstream in_archive_stream(serialized); - deserializeXML(in_archive_stream, output, name); + deserializeFromXMLStream(in_archive_stream, output, name); } /// serializes to an XML file @@ -139,7 +151,7 @@ bool serializeToXMLFile(const T& input, const std::string& filename, const std::string& name = "data") { std::ofstream out_archive_stream(filename.c_str()); if (!out_archive_stream.is_open()) return false; - serializeXML(input, out_archive_stream, name); + serializeToXMLStream(input, out_archive_stream, name); out_archive_stream.close(); return true; } @@ -150,10 +162,24 @@ bool deserializeFromXMLFile(const std::string& filename, T& output, const std::string& name = "data") { std::ifstream in_archive_stream(filename.c_str()); if (!in_archive_stream.is_open()) return false; - deserializeXML(in_archive_stream, output, name); + deserializeFromXMLStream(in_archive_stream, output, name); in_archive_stream.close(); return true; } + +/// serializes to a string in XML +template +std::string serializeXML(const T& input, + const std::string& name = "data") { + return serializeToXMLString(input, name); +} + +/// deserializes from a string in XML +template +void deserializeXML(const std::string& serialized, T& output, + const std::string& name = "data") { + deserializeFromXMLString(serialized, output, name); +} ///@} /** @name Binary Serialization @@ -162,7 +188,7 @@ bool deserializeFromXMLFile(const std::string& filename, T& output, ///@{ /// serializes to a stream in binary template -void serializeBinary(const T& input, std::ostream& out_archive_stream, +void serializeToBinaryStream(const T& input, std::ostream& out_archive_stream, const std::string& name = "data") { boost::archive::binary_oarchive out_archive(out_archive_stream); out_archive << boost::serialization::make_nvp(name.c_str(), input); @@ -170,7 +196,7 @@ void serializeBinary(const T& input, std::ostream& out_archive_stream, /// deserializes from a stream in binary template -void deserializeBinary(std::istream& in_archive_stream, T& output, +void deserializeFromBinaryStream(std::istream& in_archive_stream, T& output, const std::string& name = "data") { boost::archive::binary_iarchive in_archive(in_archive_stream); in_archive >> boost::serialization::make_nvp(name.c_str(), output); @@ -178,19 +204,19 @@ void deserializeBinary(std::istream& in_archive_stream, T& output, /// serializes to a string in binary template -std::string serializeBinary(const T& input, +std::string serializeToBinaryString(const T& input, const std::string& name = "data") { std::ostringstream out_archive_stream; - serializeBinary(input, out_archive_stream, name); + serializeToBinaryStream(input, out_archive_stream, name); return out_archive_stream.str(); } /// deserializes from a string in binary template -void deserializeBinary(const std::string& serialized, T& output, +void deserializeFromBinaryString(const std::string& serialized, T& output, const std::string& name = "data") { std::istringstream in_archive_stream(serialized); - deserializeBinary(in_archive_stream, output, name); + deserializeFromBinaryStream(in_archive_stream, output, name); } /// serializes to a binary file @@ -199,7 +225,7 @@ bool serializeToBinaryFile(const T& input, const std::string& filename, const std::string& name = "data") { std::ofstream out_archive_stream(filename.c_str()); if (!out_archive_stream.is_open()) return false; - serializeBinary(input, out_archive_stream, name); + serializeToBinaryStream(input, out_archive_stream, name); out_archive_stream.close(); return true; } @@ -210,10 +236,24 @@ bool deserializeFromBinaryFile(const std::string& filename, T& output, const std::string& name = "data") { std::ifstream in_archive_stream(filename.c_str()); if (!in_archive_stream.is_open()) return false; - deserializeBinary(in_archive_stream, output, name); + deserializeFromBinaryStream(in_archive_stream, output, name); in_archive_stream.close(); return true; } + +/// serializes to a string in binary +template +std::string serializeBinary(const T& input, + const std::string& name = "data") { + return serializeToBinaryString(input, name); +} + +/// deserializes from a string in binary +template +void deserializeBinary(const std::string& serialized, T& output, + const std::string& name = "data") { + deserializeFromBinaryString(serialized, output, name); +} ///@} } // namespace gtsam