[protozero] 01/05: Imported Upstream version 1.2.0
Sebastiaan Couwenberg
sebastic at moszumanska.debian.org
Mon Oct 12 06:31:09 UTC 2015
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository protozero.
commit 06ff0d8700f16239670a039ebaa769587639bf33
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Mon Oct 12 08:15:32 2015 +0200
Imported Upstream version 1.2.0
---
CHANGELOG.md | 17 +-
Makefile | 2 +-
README.md | 2 -
include/protozero/byteswap.hpp | 30 +++-
include/protozero/pbf_builder.hpp | 28 +++-
include/protozero/pbf_message.hpp | 44 +++++
include/protozero/pbf_reader.hpp | 12 +-
include/protozero/version.hpp | 4 +-
test/include/test.hpp | 3 +
test/t/basic/test_cases.cpp | 3 +-
test/t/bool/test_cases.cpp | 75 ++++++++-
test/t/complex/test_cases.cpp | 342 +++++++++++++++++++++++++++++++++++++-
test/t/endian/test_cases.cpp | 62 +++++++
test/t/skip/test_cases.cpp | 5 +-
tutorial.md | 139 ++++++++++++++--
15 files changed, 736 insertions(+), 32 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 218d50b..dfe0dbd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,20 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Fixed
+## [1.2.0] - 2015-10-08
+
+### Added
+
+- pbf_message and pbf_builder template classes wrapping pbf_reader
+ and pbf_writer, respectively. The new classes are the preferred
+ interface now.
+
+### Changed
+
+- Improved byte swapping operation.
+- Detect some types of data corruption earlier and throw.
+
+
## [1.1.0] - 2015-08-22
### Changed
@@ -20,6 +34,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
- Make pbf reader and writer code endianess-aware.
-[unreleased]: https://github.com/osmcode/libosmium/compare/v1.1.0...HEAD
+[unreleased]: https://github.com/osmcode/libosmium/compare/v1.2.0...HEAD
+[1.2.0]: https://github.com/osmcode/libosmium/compare/v1.1.0...v1.2.0
[1.1.0]: https://github.com/osmcode/libosmium/compare/v1.0.0...v1.1.0
diff --git a/Makefile b/Makefile
index c771dc9..1141d07 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ CXX := $(CXX)
CXXFLAGS := $(CXXFLAGS)
LDFLAGS := $(LDFLAGS)
-WARNING_FLAGS := -Wall -Wextra -pedantic -Wsign-compare -Wsign-conversion -Wunused-parameter -Wno-float-equal
+WARNING_FLAGS := -Wall -Wextra -pedantic -Wsign-compare -Wsign-conversion -Wunused-parameter -Wno-float-equal -Wno-covered-switch-default
ifneq ($(findstring clang,$(CXX)),)
WARNING_FLAGS += -Wno-reserved-id-macro -Weverything -Wno-weak-vtables -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-exit-time-destructors -Wno-switch-enum -Wno-padded
diff --git a/README.md b/README.md
index db4c994..695442a 100644
--- a/README.md
+++ b/README.md
@@ -50,8 +50,6 @@ Call `make doc` to build the Doxygen documentation. (You'll need
## Limitations
-* The current implementation does not support big-endian machines. Fixed sized
- integers and floats/doubles will not decode properly.
* A protobuf message has to fit into memory completely, otherwise it can not
be parsed with this library. There is no streaming support.
* The length of a string, bytes, or submessage can't be more than 2^31-1.
diff --git a/include/protozero/byteswap.hpp b/include/protozero/byteswap.hpp
index d019c28..2082952 100644
--- a/include/protozero/byteswap.hpp
+++ b/include/protozero/byteswap.hpp
@@ -10,30 +10,57 @@ documentation.
*****************************************************************************/
+/**
+ * @file byteswap.hpp
+ *
+ * @brief Contains functions to swap bytes in values (for different endianness).
+ */
+
+#include <cstdint>
#include <cassert>
namespace protozero {
+/**
+ * Swap N byte value between endianness formats. This template function must
+ * be specialized to actually work.
+ */
template <int N>
inline void byteswap(const char* /*data*/, char* /*result*/) {
- assert(false);
+ static_assert(N == 1, "Can only swap 1, 4, or 8 byte values");
}
+/**
+ * Swap 1 byte value between endianness formats. (Basically just a copy).
+ */
template <>
inline void byteswap<1>(const char* data, char* result) {
result[0] = data[0];
}
+/**
+ * Swap 4 byte value (int32_t, uint32_t, float) between endianness formats.
+ */
template <>
inline void byteswap<4>(const char* data, char* result) {
+# if defined(__GNUC__) || defined(__clang__)
+ *reinterpret_cast<uint32_t*>(result) = __builtin_bswap32(*reinterpret_cast<const uint32_t*>(data));
+# else
result[3] = data[0];
result[2] = data[1];
result[1] = data[2];
result[0] = data[3];
+#endif
}
+/**
+ * Swap 8 byte value (int64_t, uint64_t, double) between endianness formats.
+ */
template <>
inline void byteswap<8>(const char* data, char* result) {
+# if defined(__GNUC__) || defined(__clang__)
+ *reinterpret_cast<uint64_t*>(result) = __builtin_bswap64(*reinterpret_cast<const uint64_t*>(data));
+# else
result[7] = data[0];
result[6] = data[1];
result[5] = data[2];
@@ -42,6 +69,7 @@ inline void byteswap<8>(const char* data, char* result) {
result[2] = data[5];
result[1] = data[6];
result[0] = data[7];
+#endif
}
} // end namespace protozero
diff --git a/include/protozero/pbf_builder.hpp b/include/protozero/pbf_builder.hpp
index d49a7ba..063fa9c 100644
--- a/include/protozero/pbf_builder.hpp
+++ b/include/protozero/pbf_builder.hpp
@@ -10,6 +10,12 @@ documentation.
*****************************************************************************/
+/**
+ * @file pbf_builder.hpp
+ *
+ * @brief Contains the pbf_builder template class.
+ */
+
#include <type_traits>
#include <protozero/pbf_types.hpp>
@@ -17,10 +23,22 @@ documentation.
namespace protozero {
+/**
+ * The pbf_builder is used to write PBF formatted messages into a buffer. It
+ * is based on the pbf_writer class and has all the same methods. The
+ * difference is that whereever the pbf_writer class takes an integer tag,
+ * this template class takes a tag of the template type T.
+ *
+ * Almost all methods in this class can throw an std::bad_alloc exception if
+ * the std::string used as a buffer wants to resize.
+ *
+ * Read the tutorial to understand how this class is used.
+ */
template <typename T>
class pbf_builder : public pbf_writer {
- static_assert(std::is_same<pbf_tag_type, typename std::underlying_type<T>::type>::value, "T must be enum with underlying type protozero::pbf_tag_type");
+ static_assert(std::is_same<pbf_tag_type, typename std::underlying_type<T>::type>::value,
+ "T must be enum with underlying type protozero::pbf_tag_type");
public:
@@ -35,6 +53,7 @@ public:
pbf_writer(parent_writer, pbf_tag_type(tag)) {
}
+/// @cond INTERNAL
#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \
inline void add_##name(T tag, type value) { \
pbf_writer::add_##name(pbf_tag_type(tag), value); \
@@ -55,6 +74,9 @@ public:
PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double)
+#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR
+/// @endcond
+
inline void add_bytes(T tag, const char* value, size_t size) {
pbf_writer::add_bytes(pbf_tag_type(tag), value, size);
}
@@ -83,6 +105,7 @@ public:
pbf_writer::add_message(pbf_tag_type(tag), value);
}
+/// @cond INTERNAL
#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \
template <typename InputIterator> \
inline void add_packed_##name(T tag, InputIterator first, InputIterator last) { \
@@ -104,6 +127,9 @@ public:
PROTOZERO_WRITER_WRAP_ADD_PACKED(float)
PROTOZERO_WRITER_WRAP_ADD_PACKED(double)
+#undef PROTOZERO_WRITER_WRAP_ADD_PACKED
+/// @endcond
+
};
} // end namespace protozero
diff --git a/include/protozero/pbf_message.hpp b/include/protozero/pbf_message.hpp
index af29a00..7fef06f 100644
--- a/include/protozero/pbf_message.hpp
+++ b/include/protozero/pbf_message.hpp
@@ -10,6 +10,12 @@ documentation.
*****************************************************************************/
+/**
+ * @file pbf_message.hpp
+ *
+ * @brief Contains the pbf_message class.
+ */
+
#include <type_traits>
#include <protozero/pbf_reader.hpp>
@@ -17,6 +23,44 @@ documentation.
namespace protozero {
+/**
+ * This class represents a protobuf message. Either a top-level message or
+ * a nested sub-message. Top-level messages can be created from any buffer
+ * with a pointer and length:
+ *
+ * @code
+ * enum class Message : protozero::pbf_tag_type {
+ * ...
+ * };
+ *
+ * std::string buffer;
+ * // fill buffer...
+ * pbf_message<Message> message(buffer.data(), buffer.size());
+ * @endcode
+ *
+ * Sub-messages are created using get_message():
+ *
+ * @code
+ * enum class SubMessage : protozero::pbf_tag_type {
+ * ...
+ * };
+ *
+ * pbf_message<Message> message(...);
+ * message.next();
+ * pbf_message<SubMessage> submessage = message.get_message();
+ * @endcode
+ *
+ * All methods of the pbf_message class except get_bytes() and get_string()
+ * provide the strong exception guarantee, ie they either succeed or do not
+ * change the pbf_message object they are called on. Use the get_data() method
+ * instead of get_bytes() or get_string(), if you need this guarantee.
+ *
+ * This template class is based on the pbf_reader class and has all the same
+ * methods. The difference is that whereever the pbf_reader class takes an
+ * integer tag, this template class takes a tag of the template type T.
+ *
+ * Read the tutorial to understand how this class is used.
+ */
template <typename T>
class pbf_message : public pbf_reader {
diff --git a/include/protozero/pbf_reader.hpp b/include/protozero/pbf_reader.hpp
index 1c5ed0d..ac3220c 100644
--- a/include/protozero/pbf_reader.hpp
+++ b/include/protozero/pbf_reader.hpp
@@ -866,8 +866,16 @@ bool pbf_reader::next() {
protozero_assert(((m_tag > 0 && m_tag < 19000) || (m_tag > 19999 && m_tag <= ((1 << 29) - 1))) && "tag out of range");
m_wire_type = pbf_wire_type(value & 0x07);
-// XXX do we want this check? or should it throw an exception?
-// protozero_assert((m_wire_type <=2 || m_wire_type == 5) && "illegal wire type");
+ switch (m_wire_type) {
+ case pbf_wire_type::varint:
+ case pbf_wire_type::fixed64:
+ case pbf_wire_type::length_delimited:
+ case pbf_wire_type::fixed32:
+ break;
+ default:
+ throw unknown_pbf_wire_type_exception();
+ }
+
return true;
}
diff --git a/include/protozero/version.hpp b/include/protozero/version.hpp
index 098492e..c8e45bb 100644
--- a/include/protozero/version.hpp
+++ b/include/protozero/version.hpp
@@ -11,12 +11,12 @@ documentation.
*****************************************************************************/
#define PROTOZERO_VERSION_MAJOR 1
-#define PROTOZERO_VERSION_MINOR 1
+#define PROTOZERO_VERSION_MINOR 2
#define PROTOZERO_VERSION_PATCH 0
#define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH)
-#define PROTOZERO_VERSION_STRING "1.1.0"
+#define PROTOZERO_VERSION_STRING "1.2.0"
#endif // PROTOZERO_VERSION_HPP
diff --git a/test/include/test.hpp b/test/include/test.hpp
index 33527a2..057a946 100644
--- a/test/include/test.hpp
+++ b/test/include/test.hpp
@@ -11,7 +11,10 @@ struct assert_error : public std::runtime_error {
#define protozero_assert(x) if (!(x)) { throw(assert_error(#x)); }
#include <protozero/pbf_reader.hpp>
+#include <protozero/pbf_message.hpp>
+
#include <protozero/pbf_writer.hpp>
+#include <protozero/pbf_builder.hpp>
extern std::string load_data(const std::string& filename);
diff --git a/test/t/basic/test_cases.cpp b/test/t/basic/test_cases.cpp
index 89c342c..ac3c7e3 100644
--- a/test/t/basic/test_cases.cpp
+++ b/test/t/basic/test_cases.cpp
@@ -39,8 +39,7 @@ TEST_CASE("basic") {
char buffer[1] = { 1 << 3 | 7 };
protozero::pbf_reader item(buffer, 1);
- REQUIRE(item.next());
- REQUIRE_THROWS_AS(item.skip(), protozero::unknown_pbf_wire_type_exception);
+ REQUIRE_THROWS_AS(item.next(), protozero::unknown_pbf_wire_type_exception);
}
}
diff --git a/test/t/bool/test_cases.cpp b/test/t/bool/test_cases.cpp
index 06cc02c..b73a8fd 100644
--- a/test/t/bool/test_cases.cpp
+++ b/test/t/bool/test_cases.cpp
@@ -1,7 +1,15 @@
#include <test.hpp>
-TEST_CASE("read bool field") {
+namespace TestBoolean {
+
+enum class Test : protozero::pbf_tag_type {
+ required_bool_b = 1
+};
+
+} // end namespace TestBoolean
+
+TEST_CASE("read bool field using pbf_reader") {
SECTION("false") {
std::string buffer = load_data("bool/data-false");
@@ -28,7 +36,7 @@ TEST_CASE("read bool field") {
protozero::pbf_reader item(buffer);
- REQUIRE(item.next());
+ REQUIRE(item.next(1));
REQUIRE(item.get_bool());
REQUIRE(!item.next());
}
@@ -38,14 +46,58 @@ TEST_CASE("read bool field") {
protozero::pbf_reader item(buffer);
+ REQUIRE(item.next(1));
+ REQUIRE(item.get_bool());
+ REQUIRE(!item.next());
+ }
+
+}
+
+TEST_CASE("read bool field using pbf_message") {
+
+ SECTION("false") {
+ std::string buffer = load_data("bool/data-false");
+
+ protozero::pbf_message<TestBoolean::Test> item(buffer);
+
+ REQUIRE(item.next());
+ REQUIRE(!item.get_bool());
+ REQUIRE(!item.next());
+ }
+
+ SECTION("true") {
+ std::string buffer = load_data("bool/data-true");
+
+ protozero::pbf_message<TestBoolean::Test> item(buffer);
+
REQUIRE(item.next());
REQUIRE(item.get_bool());
REQUIRE(!item.next());
}
+ SECTION("also true") {
+ std::string buffer = load_data("bool/data-also-true");
+
+ protozero::pbf_message<TestBoolean::Test> item(buffer);
+
+ REQUIRE(item.next(TestBoolean::Test::required_bool_b));
+ REQUIRE(item.get_bool());
+ REQUIRE(!item.next());
+ }
+
+ SECTION("still true") {
+ std::string buffer = load_data("bool/data-still-true");
+
+ protozero::pbf_message<TestBoolean::Test> item(buffer);
+
+ REQUIRE(item.next(TestBoolean::Test::required_bool_b));
+ REQUIRE(item.get_bool());
+ REQUIRE(!item.next());
+ }
+
}
-TEST_CASE("write bool field") {
+TEST_CASE("write bool field using pbf_writer") {
std::string buffer;
protozero::pbf_writer pw(buffer);
@@ -62,3 +114,20 @@ TEST_CASE("write bool field") {
}
+TEST_CASE("write bool field using pbf_builder") {
+
+ std::string buffer;
+ protozero::pbf_builder<TestBoolean::Test> pw(buffer);
+
+ SECTION("false") {
+ pw.add_bool(TestBoolean::Test::required_bool_b, false);
+ REQUIRE(buffer == load_data("bool/data-false"));
+ }
+
+ SECTION("true") {
+ pw.add_bool(TestBoolean::Test::required_bool_b, true);
+ REQUIRE(buffer == load_data("bool/data-true"));
+ }
+
+}
+
diff --git a/test/t/complex/test_cases.cpp b/test/t/complex/test_cases.cpp
index e74a11f..ab1b60d 100644
--- a/test/t/complex/test_cases.cpp
+++ b/test/t/complex/test_cases.cpp
@@ -1,7 +1,25 @@
#include <test.hpp>
-TEST_CASE("read complex data") {
+namespace TestComplex {
+
+enum class Test : protozero::pbf_tag_type {
+ required_fixed32_f = 1,
+ optional_int64_i = 2,
+ optional_int64_j = 3,
+ required_Sub_submessage = 5,
+ optional_string_s = 8,
+ repeated_uint32_u = 4,
+ packed_sint32_d = 7
+};
+
+enum class Sub : protozero::pbf_tag_type {
+ required_string_s = 1
+};
+
+} // end namespace TestComplex
+
+TEST_CASE("read complex data using pbf_reader") {
SECTION("minimal") {
std::string buffer = load_data("complex/data-minimal");
@@ -147,7 +165,153 @@ TEST_CASE("read complex data") {
}
-TEST_CASE("write complex data") {
+TEST_CASE("read complex data using pbf_message") {
+
+ SECTION("minimal") {
+ std::string buffer = load_data("complex/data-minimal");
+
+ protozero::pbf_message<TestComplex::Test> item(buffer);
+
+ while (item.next()) {
+ switch (item.tag()) {
+ case TestComplex::Test::required_fixed32_f: {
+ REQUIRE(item.get_fixed32() == 12345678L);
+ break;
+ }
+ case TestComplex::Test::required_Sub_submessage: {
+ protozero::pbf_message<TestComplex::Sub> subitem = item.get_message();
+ REQUIRE(subitem.next());
+ REQUIRE(subitem.get_string() == "foobar");
+ REQUIRE(!subitem.next());
+ break;
+ }
+ default: {
+ REQUIRE(false); // should not be here
+ break;
+ }
+ }
+ }
+ }
+
+ SECTION("some") {
+ std::string buffer = load_data("complex/data-some");
+
+ protozero::pbf_message<TestComplex::Test> item(buffer);
+
+ uint32_t sum_of_u = 0;
+ while (item.next()) {
+ switch (item.tag()) {
+ case TestComplex::Test::required_fixed32_f: {
+ REQUIRE(item.get_fixed32() == 12345678L);
+ break;
+ }
+ case TestComplex::Test::optional_int64_i: {
+ REQUIRE(true);
+ item.skip();
+ break;
+ }
+ case TestComplex::Test::repeated_uint32_u: {
+ sum_of_u += item.get_uint32();
+ break;
+ }
+ case TestComplex::Test::required_Sub_submessage: {
+ protozero::pbf_message<TestComplex::Sub> subitem = item.get_message();
+ REQUIRE(subitem.next());
+ REQUIRE(subitem.get_string() == "foobar");
+ REQUIRE(!subitem.next());
+ break;
+ }
+ default: {
+ REQUIRE(false); // should not be here
+ break;
+ }
+ }
+ }
+ REQUIRE(sum_of_u == 66);
+ }
+
+ SECTION("all") {
+ std::string buffer = load_data("complex/data-all");
+
+ protozero::pbf_message<TestComplex::Test> item(buffer);
+
+ int number_of_u = 0;
+ while (item.next()) {
+ switch (item.tag()) {
+ case TestComplex::Test::required_fixed32_f: {
+ REQUIRE(item.get_fixed32() == 12345678L);
+ break;
+ }
+ case TestComplex::Test::optional_int64_i: {
+ REQUIRE(true);
+ item.skip();
+ break;
+ }
+ case TestComplex::Test::optional_int64_j: {
+ REQUIRE(item.get_int64() == 555555555LL);
+ break;
+ }
+ case TestComplex::Test::repeated_uint32_u: {
+ item.skip();
+ ++number_of_u;
+ break;
+ }
+ case TestComplex::Test::required_Sub_submessage: {
+ protozero::pbf_message<TestComplex::Sub> subitem = item.get_message();
+ REQUIRE(subitem.next());
+ REQUIRE(subitem.get_string() == "foobar");
+ REQUIRE(!subitem.next());
+ break;
+ }
+ case TestComplex::Test::packed_sint32_d: {
+ auto pi = item.get_packed_sint32();
+ int32_t sum = 0;
+ for (auto it = pi.first; it != pi.second; ++it) {
+ sum += *it;
+ }
+ REQUIRE(sum == 5);
+ break;
+ }
+ case TestComplex::Test::optional_string_s: {
+ REQUIRE(item.get_string() == "optionalstring");
+ break;
+ }
+ default: {
+ REQUIRE(false); // should not be here
+ break;
+ }
+ }
+ }
+ REQUIRE(number_of_u == 5);
+ }
+
+ SECTION("skip everything") {
+ std::string buffer = load_data("complex/data-all");
+
+ protozero::pbf_message<TestComplex::Test> item(buffer);
+
+ while (item.next()) {
+ switch (item.tag()) {
+ case TestComplex::Test::required_fixed32_f:
+ case TestComplex::Test::optional_int64_i:
+ case TestComplex::Test::optional_int64_j:
+ case TestComplex::Test::repeated_uint32_u:
+ case TestComplex::Test::required_Sub_submessage:
+ case TestComplex::Test::packed_sint32_d:
+ case TestComplex::Test::optional_string_s:
+ item.skip();
+ break;
+ default: {
+ REQUIRE(false); // should not be here
+ break;
+ }
+ }
+ }
+ }
+
+}
+
+TEST_CASE("write complex data using pbf_writer") {
SECTION("minimal") {
std::string buffer;
@@ -306,6 +470,165 @@ TEST_CASE("write complex data") {
}
}
+TEST_CASE("write complex data using pbf_builder") {
+
+ SECTION("minimal") {
+ std::string buffer;
+ protozero::pbf_builder<TestComplex::Test> pw(buffer);
+ pw.add_fixed32(TestComplex::Test::required_fixed32_f, 12345678);
+
+ std::string submessage;
+ protozero::pbf_builder<TestComplex::Sub> pws(submessage);
+ pws.add_string(TestComplex::Sub::required_string_s, "foobar");
+
+ pw.add_message(TestComplex::Test::required_Sub_submessage, submessage);
+
+ protozero::pbf_reader item(buffer);
+
+ while (item.next()) {
+ switch (item.tag()) {
+ case 1: {
+ REQUIRE(item.get_fixed32() == 12345678L);
+ break;
+ }
+ case 5: {
+ protozero::pbf_reader subitem = item.get_message();
+ REQUIRE(subitem.next());
+ REQUIRE(subitem.get_string() == "foobar");
+ REQUIRE(!subitem.next());
+ break;
+ }
+ default: {
+ REQUIRE(false); // should not be here
+ break;
+ }
+ }
+ }
+ }
+
+ SECTION("some") {
+ std::string buffer;
+ protozero::pbf_builder<TestComplex::Test> pw(buffer);
+ pw.add_fixed32(TestComplex::Test::required_fixed32_f, 12345678);
+
+ std::string submessage;
+ protozero::pbf_builder<TestComplex::Sub> pws(submessage);
+ pws.add_string(TestComplex::Sub::required_string_s, "foobar");
+
+ pw.add_uint32(TestComplex::Test::repeated_uint32_u, 22);
+ pw.add_uint32(TestComplex::Test::repeated_uint32_u, 44);
+ pw.add_int64(TestComplex::Test::optional_int64_i, -9876543);
+ pw.add_message(TestComplex::Test::required_Sub_submessage, submessage);
+
+ protozero::pbf_reader item(buffer);
+
+ uint32_t sum_of_u = 0;
+ while (item.next()) {
+ switch (item.tag()) {
+ case 1: {
+ REQUIRE(item.get_fixed32() == 12345678L);
+ break;
+ }
+ case 2: {
+ REQUIRE(true);
+ item.skip();
+ break;
+ }
+ case 4: {
+ sum_of_u += item.get_uint32();
+ break;
+ }
+ case 5: {
+ protozero::pbf_reader subitem = item.get_message();
+ REQUIRE(subitem.next());
+ REQUIRE(subitem.get_string() == "foobar");
+ REQUIRE(!subitem.next());
+ break;
+ }
+ default: {
+ REQUIRE(false); // should not be here
+ break;
+ }
+ }
+ }
+ REQUIRE(sum_of_u == 66);
+ }
+
+ SECTION("all") {
+ std::string buffer;
+ protozero::pbf_builder<TestComplex::Test> pw(buffer);
+ pw.add_fixed32(TestComplex::Test::required_fixed32_f, 12345678);
+
+ std::string submessage;
+ protozero::pbf_builder<TestComplex::Sub> pws(submessage);
+ pws.add_string(TestComplex::Sub::required_string_s, "foobar");
+ pw.add_message(TestComplex::Test::required_Sub_submessage, submessage);
+
+ pw.add_uint32(TestComplex::Test::repeated_uint32_u, 22);
+ pw.add_uint32(TestComplex::Test::repeated_uint32_u, 44);
+ pw.add_int64(TestComplex::Test::optional_int64_i, -9876543);
+ pw.add_uint32(TestComplex::Test::repeated_uint32_u, 44);
+ pw.add_uint32(TestComplex::Test::repeated_uint32_u, 66);
+ pw.add_uint32(TestComplex::Test::repeated_uint32_u, 66);
+
+ int32_t d[] = { -17, 22 };
+ pw.add_packed_sint32(TestComplex::Test::packed_sint32_d, std::begin(d), std::end(d));
+
+ pw.add_int64(TestComplex::Test::optional_int64_j, 555555555);
+
+ protozero::pbf_reader item(buffer);
+
+ int number_of_u = 0;
+ while (item.next()) {
+ switch (item.tag()) {
+ case 1: {
+ REQUIRE(item.get_fixed32() == 12345678L);
+ break;
+ }
+ case 2: {
+ REQUIRE(true);
+ item.skip();
+ break;
+ }
+ case 3: {
+ REQUIRE(item.get_int64() == 555555555LL);
+ break;
+ }
+ case 4: {
+ item.skip();
+ ++number_of_u;
+ break;
+ }
+ case 5: {
+ protozero::pbf_reader subitem = item.get_message();
+ REQUIRE(subitem.next());
+ REQUIRE(subitem.get_string() == "foobar");
+ REQUIRE(!subitem.next());
+ break;
+ }
+ case 7: {
+ auto pi = item.get_packed_sint32();
+ int32_t sum = 0;
+ for (auto it = pi.first; it != pi.second; ++it) {
+ sum += *it;
+ }
+ REQUIRE(sum == 5);
+ break;
+ }
+ case 8: {
+ REQUIRE(item.get_string() == "optionalstring");
+ break;
+ }
+ default: {
+ REQUIRE(false); // should not be here
+ break;
+ }
+ }
+ }
+ REQUIRE(number_of_u == 5);
+ }
+}
+
static void check_message(const std::string& buffer) {
protozero::pbf_reader item(buffer);
@@ -330,7 +653,7 @@ static void check_message(const std::string& buffer) {
}
}
-TEST_CASE("write complex with subwriter") {
+TEST_CASE("write complex with subwriter using pbf_writer") {
std::string buffer_test;
protozero::pbf_writer pbf_test(buffer_test);
pbf_test.add_fixed32(1, 42L);
@@ -343,3 +666,16 @@ TEST_CASE("write complex with subwriter") {
check_message(buffer_test);
}
+TEST_CASE("write complex with subwriter using pbf_builder") {
+ std::string buffer_test;
+ protozero::pbf_builder<TestComplex::Test> pbf_test(buffer_test);
+ pbf_test.add_fixed32(TestComplex::Test::required_fixed32_f, 42L);
+
+ SECTION("message in message") {
+ protozero::pbf_builder<TestComplex::Sub> pbf_submessage(pbf_test, TestComplex::Test::required_Sub_submessage);
+ pbf_submessage.add_string(TestComplex::Sub::required_string_s, "foobar");
+ }
+
+ check_message(buffer_test);
+}
+
diff --git a/test/t/endian/test_cases.cpp b/test/t/endian/test_cases.cpp
new file mode 100644
index 0000000..0c0feb0
--- /dev/null
+++ b/test/t/endian/test_cases.cpp
@@ -0,0 +1,62 @@
+
+#include <cstdint>
+
+#include <protozero/byteswap.hpp>
+
+#include <test.hpp>
+
+namespace {
+
+ char check_swap_1(char data) {
+ char swapped;
+ char back;
+
+ protozero::byteswap<1>(&data, &swapped);
+ protozero::byteswap<1>(&swapped, &back);
+
+ return back;
+ }
+
+ int32_t check_swap_4(int32_t data) {
+ int32_t swapped;
+ int32_t back;
+
+ protozero::byteswap<4>(reinterpret_cast<const char*>(&data), reinterpret_cast<char*>(&swapped));
+ protozero::byteswap<4>(reinterpret_cast<const char*>(&swapped), reinterpret_cast<char*>(&back));
+
+ return back;
+ }
+
+ int64_t check_swap_8(int64_t data) {
+ int64_t swapped;
+ int64_t back;
+
+ protozero::byteswap<8>(reinterpret_cast<const char*>(&data), reinterpret_cast<char*>(&swapped));
+ protozero::byteswap<8>(reinterpret_cast<const char*>(&swapped), reinterpret_cast<char*>(&back));
+
+ return back;
+ }
+
+}
+
+TEST_CASE("byte swapping") {
+ REQUIRE(0 == check_swap_1(0));
+ REQUIRE(1 == check_swap_1(1));
+ REQUIRE(-1 == check_swap_1(-1));
+ REQUIRE(127 == check_swap_1(127));
+
+ REQUIRE(0 == check_swap_4(0));
+ REQUIRE(1 == check_swap_4(1));
+ REQUIRE(-1 == check_swap_4(-1));
+ REQUIRE(395503 == check_swap_4(395503));
+ REQUIRE(-804022 == check_swap_4(-804022));
+
+ REQUIRE(0 == check_swap_8(0));
+ REQUIRE(1 == check_swap_8(1));
+ REQUIRE(-1 == check_swap_8(-1));
+ REQUIRE(395503 == check_swap_8(395503));
+ REQUIRE(-804022 == check_swap_8(-804022));
+ REQUIRE(3280329805 == check_swap_8(3280329805));
+ REQUIRE(-2489204041 == check_swap_8(-2489204041));
+}
+
diff --git a/test/t/skip/test_cases.cpp b/test/t/skip/test_cases.cpp
index 70f932f..14bd10f 100644
--- a/test/t/skip/test_cases.cpp
+++ b/test/t/skip/test_cases.cpp
@@ -110,7 +110,7 @@ TEST_CASE("skip") {
}
- SECTION("check that skip() throws on unknown field type") {
+ SECTION("check that next() throws on unknown field type") {
std::string buffer;
protozero::pbf_writer pw(buffer);
@@ -120,8 +120,7 @@ TEST_CASE("skip") {
protozero::pbf_reader item(buffer);
- REQUIRE(item.next());
- REQUIRE_THROWS_AS(item.skip(), protozero::unknown_pbf_wire_type_exception);
+ REQUIRE_THROWS_AS(item.next(), protozero::unknown_pbf_wire_type_exception);
}
SECTION("check that skip() throws on short buffer") {
diff --git a/tutorial.md b/tutorial.md
index 7084cac..e5de974 100644
--- a/tutorial.md
+++ b/tutorial.md
@@ -22,11 +22,16 @@ if you are getting lost.
## Prerequisites
You need a C++11-capable compiler for protozero to work. Copy the files in the
-`include` directory somewhere where your build system can find them.
+`include/protozero` directory somewhere where your build system can find them.
+Keep the `protozero` directory and include the files in the form
-You always need `protozero/pbf_types.hpp`, `protozero/varint.hpp`, and
-`protozero/exception.hpp`. For reading and writing support you need
-`protozero/pbf_reader.hpp` and `protozero/pbf_writer.hpp`, respectively.
+ #include <protozero/FILENAME.hpp>
+
+You always need `byteswap.hpp`, `pbf_types.hpp`, `varint.hpp`, and
+`exception.hpp`. For reading you need `pbf_reader.hpp` and probably
+`pbf_message.hpp`, for writing you need `pbf_writer.hpp` and probably
+`pbf_builder.hpp`. You only need `version.hpp` if you want access to the macros
+defining the library version.
## Parsing protobuf-encoded messages
@@ -46,8 +51,8 @@ errors. We encourage you to compile with asserts enabled in your debug builds.
Lets say you have a protocol description in a `.proto` file like this:
message Example1 {
- required uint32 x = 1;
- optional string s = 2;
+ required uint32 x = 1;
+ optional string s = 2;
repeated fixed64 r = 17;
}
@@ -211,10 +216,27 @@ the `get_enum()` function to get the value of the enum, you have to translate
this into the symbolic name yourself. See the `enum` test case for an example.
-### Exceptions
+### Asserts and exceptions in the protozero library
+
+Protozero uses `assert()` liberally to help you find bugs in your own code when
+compiled in debug mode (ie with `NDEBUG` not set). If such an assert "fires",
+this is a very strong indication that there is a bug in your code somewhere.
+
+(Protozero will disable those asserts and "convert" them into exception in its
+own test code. This is done to make sure the asserts actually work as intended.
+Your test code will not need this!)
+
+Exceptions, on the other hand, are thrown by protozero if some kind of data
+corruption was detected while it is trying to parse the data. This could also
+be an indicator for a bug in the user code, but because it can happen if the
+data was (intentionally or not intentionally) been messed with, it is reported
+to the user code using exceptions.
+
+Most of the functions on the writer side can throw a `std::bad_alloc`
+exception if there is no space to grow a buffer. Other than that no exceptions
+can occur on the writer side.
-All exceptions thrown by `pbf_reader.hpp` functions derive from
-`protozero::exception`.
+All exceptions thrown by the reader side derive from `protozero::exception`.
Note that all exceptions can also happen if you are expecting a data field of
a certain type in your code but the field actually has a different type. In
@@ -251,6 +273,91 @@ In all cases objects of the `pbf_reader` class store a pointer into the input
data that was given to the constructor. You have to make sure this pointer
stays valid for the duration of the objects lifetime.
+## Parsing Protobuf-Encoded Messages Using `pbf_message`
+
+One problem in the code above are the "magic numbers" used as tags for the
+different fields that you got from the `.proto` file. Instead of spreading
+these magic numbers around your code you can define them once in an `enum
+class` and then use the `pbf_message` template class instead of the
+`pbf_reader` class.
+
+Here is the first example again, this time using this new technique. So you
+have the following in a `.proto` file:
+
+ message Example1 {
+ required uint32 x = 1;
+ optional string s = 2;
+ repeated fixed64 r = 17;
+ }
+
+Add the following declaration in one of your header files:
+
+ enum class Example1 : protozero::pbf_tag_type {
+ required_uint32_x = 1,
+ optional_string_s = 2,
+ repeated_fixed64_r = 17
+ };
+
+The message name becomes the name of the `enum class` which is always built
+on top of the `protozero::pbf_tag_type` type. Each field in the message
+becomes one value of the enum. In this case the name is created from the
+type (including the modifiers like `required` or `optional`) and the name of
+the field. You can use any name you want, but this convention makes it easier
+later, to get everything right.
+
+To read messages created according to that description, you will have code that
+looks somewhat like this, this time using `pbf_message` instead of
+`pbf_reader`:
+
+ #include <protozero/pbf_message.hpp>
+
+ // get data from somewhere into the input string
+ std::string input = get_input_data();
+
+ // initialize pbf message with this data
+ protozero::pbf_message<Example1> message(input);
+
+ // iterate over fields in the message
+ while (message.next()) {
+
+ // switch depending on the field tag (the field name is not available)
+ switch (message.tag()) {
+ case Example1::required_uint32_x:
+ auto x = message.get_uint32();
+ break;
+ case Example1::optional_string_s:
+ std::string s = message.get_string();
+ break;
+ case Example1::repeated_fixed64_r:
+ message.skip();
+ break;
+ default:
+ // ignore data for unknown tags to allow for future extensions
+ message.skip();
+ }
+
+ }
+
+Note the correspondance between the enum value (for instance
+`required_uint32_x`) and the name of the getter function (for instance
+`get_uint32()`). This makes it easier to get the correct types. Also the
+naming makes it easier to keep different message types apart if you have
+multiple (or embedded) messages.
+
+See the `test/t/complex` test case for a complete example using this interface.
+
+Using `pbf_message` in favour of `pbf_reader` is recommended for all code.
+Note that `pbf_message` derives from `pbf_reader`, so you can always fall
+back to the more generic interface if necessary.
+
+One problem you might run into is the following: The enum class lists all
+possible values you know about and you'll have lots of `switch` statements
+checking those values. Some compilers will know that your `switch` covers
+all possible cases and warn you if you have a `default` case that looks
+unneccessary to the compiler. But you still want that `default` case to allow
+for future extension of those messages (and maybe also to detect corrupted
+data). You can switch of this warning with `-Wno-covered-switch-default`).
+
## Writing Protobuf-Encoded Messages
@@ -372,14 +479,24 @@ into it. It then adds the contents of the submessage to the buffer. When the
written in the reserved space. If less space was needed for the length field
than was available, the rest of the buffer is moved over a few bytes.
+## Writing Protobuf-Encoded Messages Using `pbf_builder`
+
+Just like the `pbf_message` template class wraps the `pbf_reader` class, there
+is a `pbf_builder` template class wrapping the `pbf_writer` class. It is
+instantiated using the same `enum class` described above and used exactly
+like the `pbf_writer` class but using the values of the enum instead of bare
+integers.
+
+See the `test/t/complex` test case for a complete example using this interface.
+
-## Using the low-level varint and zigzag encoding and decoding functions
+## Using the Low-Level Varint and Zigzag Encoding and Decoding Functions
Protozero gives you access to the low-level functions for encoding and
decoding varint and zigzag integer encodings, because these functions can
sometimes be useful outside the Protocol Buffer context.
-### Using low-level functions
+### Using Low-Level Functions
To use the low-level, add this include to your C++ program:
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/protozero.git
More information about the Pkg-grass-devel
mailing list