Commit 42171e25 authored by Giorgio Azzinnaro's avatar Giorgio Azzinnaro

Merge branch 'feature/test' into develop

parents ff64556f dc89b0fc
......@@ -36,4 +36,3 @@ public:
}
#endif // PROFANEDB_FORMAT_MARSHALLER_H
......@@ -33,10 +33,15 @@ using google::protobuf::DescriptorProto;
using google::protobuf::FieldDescriptor;
using google::protobuf::FieldDescriptorProto_Type;
using google::protobuf::DescriptorPool;
using google::protobuf::Message;
using profanedb::protobuf::Key;
profanedb::format::protobuf::Loader::RootSourceTree::RootSourceTree(std::initializer_list<path> paths)
namespace profanedb {
namespace format {
namespace protobuf {
Loader::RootSourceTree::RootSourceTree(std::initializer_list<path> paths)
: paths(paths)
{
if (paths.size() == 0)
......@@ -53,24 +58,25 @@ profanedb::format::protobuf::Loader::RootSourceTree::RootSourceTree(std::initial
throw std::runtime_error(this->GetLastErrorMessage());
}
profanedb::format::protobuf::Loader::Loader(
Loader::Loader(
std::unique_ptr<RootSourceTree> include,
std::unique_ptr<RootSourceTree> schema)
: includeSourceTree(std::move(include))
, schemaSourceTree(std::move(schema))
, includeDb(includeSourceTree.get())
, schemaDb(schemaSourceTree.get())
, schemaPool(new MergedDescriptorDatabase(&includeDb, &schemaDb))
, normalizedPool(&normalizedDescriptorDb)
, schemaPool(new MergedDescriptorDatabase(&includeDb, &schemaDb), &errorCollector)
, normalizedPool(new MergedDescriptorDatabase(&includeDb, &normalizedDescriptorDb), &errorCollector)
{
// profanedb.protobuf.options.key is defined in here
// It is used to mark the primary key on Messages
const FileDescriptor * optionsFile = schemaPool.FindFileByName("profanedb/protobuf/options.proto");
FileDescriptorProto optionsProto;
optionsFile->CopyTo(&optionsProto);
this->normalizedDescriptorDb.AddAndOwn(&optionsProto);
BOOST_LOG_TRIVIAL(debug) << "Loading profanedb/protobuf/options.proto and copying to normalized descriptor db";
// profanedb.protobuf.options.key is defined in here
// It is used to mark the primary key on Messages
// const FileDescriptor * optionsFile = schemaPool.FindFileByName("profanedb/protobuf/options.proto");
// FileDescriptorProto optionsProto;
// optionsFile->CopyTo(&optionsProto);
// normalizedDescriptorDb.AddAndOwn(&optionsProto);
// BOOST_LOG_TRIVIAL(debug) << "Loading profanedb/protobuf/options.proto and copying to normalized descriptor db";
// Just in case schema is defined in more than one place
for (const auto & path: schemaSourceTree->paths) {
......@@ -85,47 +91,53 @@ profanedb::format::protobuf::Loader::Loader(
// The file is now retrieved, and its path for Protobuf must be relative to the mapping
// it's parsed, normalized (nested keyable messages are removed)
FileDescriptorProto normalizedProto = this->ParseFile(schemaPool.FindFileByName(file.path().lexically_relative(path).string()));
FileDescriptorProto * normalizedProto = this->ParseFile(
schemaPool.FindFileByName(file.path().lexically_relative(path).string()));
BOOST_LOG_TRIVIAL(debug) << "Adding normalized proto " << normalizedProto.name();
BOOST_LOG_TRIVIAL(debug) << "Adding normalized proto " << normalizedProto->name();
BOOST_LOG_TRIVIAL(trace) << std::endl << normalizedProto->DebugString();
// The normalizedDescriptorDb keeps these new Descriptors
this->normalizedDescriptorDb.AddAndOwn(&normalizedProto);
normalizedDescriptorDb.AddAndOwn(normalizedProto);
}
}
}
}
FileDescriptorProto profanedb::format::protobuf::Loader::ParseFile(
FileDescriptorProto * Loader::ParseFile(
const FileDescriptor * fileDescriptor)
{
BOOST_LOG_TRIVIAL(debug) << "Parsing file " << fileDescriptor->name();
// BOOST_LOG_TRIVIAL(trace) << fileDescriptor->DebugString(); // Redundant, is done for each message later
// A FileDescriptorProto is needed to edit messages and populate the normalized descriptor database
FileDescriptorProto normFileDescProto;
fileDescriptor->CopyTo(&normFileDescProto);
FileDescriptorProto * normFileDescProto = new FileDescriptorProto;
fileDescriptor->CopyTo(normFileDescProto);
// storage.proto dependency must be added
// as profanedb.protobuf.Key is used in normalized messages
*normFileDescProto->add_dependency() = "profanedb/protobuf/storage.proto";
// For each message in the file...
for (int i = 0; i < fileDescriptor->message_type_count(); i++) {
// ... parse it, make nested messages of type profanedb.protobuf.Key
*normFileDescProto.mutable_message_type(i) = this->ParseAndNormalizeDescriptor(fileDescriptor->message_type(i));
*normFileDescProto->mutable_message_type(i) = *this->ParseAndNormalizeDescriptor(fileDescriptor->message_type(i));
}
return normFileDescProto;
}
DescriptorProto profanedb::format::protobuf::Loader::ParseAndNormalizeDescriptor(
DescriptorProto * Loader::ParseAndNormalizeDescriptor(
const Descriptor * descriptor)
{
BOOST_LOG_TRIVIAL(debug) << "Parsing descriptor " << descriptor->full_name();
BOOST_LOG_TRIVIAL(trace) << std::endl << descriptor->DebugString();
DescriptorProto normDescProto;
descriptor->CopyTo(&normDescProto);
DescriptorProto * normDescProto = new DescriptorProto;
descriptor->CopyTo(normDescProto);
// Recurse for all messages DEFINED within this message
for (int j = 0; j < descriptor->nested_type_count(); j++) {
*normDescProto.mutable_nested_type(j) = this->ParseAndNormalizeDescriptor(descriptor->nested_type(j));
*normDescProto->mutable_nested_type(j) = *this->ParseAndNormalizeDescriptor(descriptor->nested_type(j));
}
// Now the actual Descriptor normalization
......@@ -137,12 +149,12 @@ DescriptorProto profanedb::format::protobuf::Loader::ParseAndNormalizeDescriptor
// If this field is effectively a message,
// and that Message is keyable...
if(nestedMessage != nullptr && this->IsKeyable(nestedMessage)) {
if(nestedMessage != NULL && this->IsKeyable(nestedMessage)) {
// ... make the field in the normalized descriptor a Key object
// TODO Maybe we could use an option here as well
normDescProto.mutable_field(k)->set_type(
normDescProto->mutable_field(k)->set_type(
FieldDescriptorProto_Type::FieldDescriptorProto_Type_TYPE_MESSAGE); // Redundant, is message already
normDescProto.mutable_field(k)->set_type_name(Key::descriptor()->full_name()); // TODO Should include Key in normalizedPool
normDescProto->mutable_field(k)->set_type_name(Key::descriptor()->full_name()); // TODO Should include Key in normalizedPool
BOOST_LOG_TRIVIAL(trace) << "Message " << descriptor->name()
<< " has keyable nested message " << field->name()
......@@ -153,7 +165,7 @@ DescriptorProto profanedb::format::protobuf::Loader::ParseAndNormalizeDescriptor
return normDescProto;
}
bool profanedb::format::protobuf::Loader::IsKeyable(const Descriptor * descriptor) const
bool Loader::IsKeyable(const Descriptor * descriptor) const
{
for (int i = 0; i < descriptor->field_count(); i++) {
// If any field in message has profanedb::protobuf::options::key set
......@@ -163,12 +175,59 @@ bool profanedb::format::protobuf::Loader::IsKeyable(const Descriptor * descripto
return false;
}
const DescriptorPool & profanedb::format::protobuf::Loader::GetSchemaPool() const
const DescriptorPool & Loader::GetPool(PoolType poolType) const
{
return (poolType == SCHEMA) ? this->schemaPool : this->normalizedPool;
}
const Descriptor * Loader::GetDescriptor(PoolType poolType, std::string typeName) const
{
BOOST_LOG_TRIVIAL(debug) << "Getting descriptor "
<< typeName
<< " from "
<< ((poolType == SCHEMA)
? "SCHEMA"
: "NORMALIZED")
<< " pool";
const Descriptor * descriptor = this->GetPool(poolType).FindMessageTypeByName(typeName);
if (descriptor == NULL)
throw std::runtime_error(typeName + " doesn't exist");
BOOST_LOG_TRIVIAL(trace) << std::endl << descriptor->DebugString();
return descriptor;
}
const Message * Loader::CreateMessage(PoolType poolType, std::string typeName)
{
BOOST_LOG_TRIVIAL(debug) << "Creating message " << typeName;
return this->messageFactory.GetPrototype(
this->GetDescriptor(poolType, typeName));
}
void Loader::BoostLogErrorCollector::AddError(
const std::string & filename,
const std::string & element_name,
const Message * descriptor,
DescriptorPool::ErrorCollector::ErrorLocation location,
const std::string & message)
{
return this->schemaPool;
BOOST_LOG_TRIVIAL(error) << message;
}
const DescriptorPool & profanedb::format::protobuf::Loader::GetNormalizedPool() const
void Loader::BoostLogErrorCollector::AddWarning(
const std::string & filename,
const std::string & element_name,
const Message * descriptor,
DescriptorPool::ErrorCollector::ErrorLocation location,
const std::string & message)
{
return this->normalizedPool;
BOOST_LOG_TRIVIAL(warning) << message;
}
}
}
}
......@@ -24,8 +24,9 @@
#include <profanedb/protobuf/storage.pb.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/compiler/importer.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/dynamic_message.h>
#include <boost/filesystem.hpp>
#include <boost/log/trivial.hpp>
......@@ -43,7 +44,7 @@ public:
// mapping all the paths provided to the root ("/") path for easier import.
// Paths are available for Loader to populate its Pool;
class RootSourceTree : public google::protobuf::compiler::DiskSourceTree {
friend Loader;
friend class Loader;
public:
RootSourceTree(std::initializer_list<boost::filesystem::path> paths);
......@@ -58,24 +59,54 @@ public:
Loader(
std::unique_ptr<RootSourceTree> include,
std::unique_ptr<RootSourceTree> schema);
const google::protobuf::DescriptorPool & GetSchemaPool() const;
const google::protobuf::DescriptorPool & GetNormalizedPool() const;
enum PoolType {
SCHEMA,
NORMALIZED
};
// Get either Schema or Normalized pool
const google::protobuf::DescriptorPool & GetPool(PoolType poolType) const;
// Retrieve a Descriptor either from Schema or Normalized pool
const google::protobuf::Descriptor * GetDescriptor(
PoolType poolType,
std::string typeName) const;
// Create a prototype Message from defined pool
// (Should use `CreateMessage()->New() to get an empty Message from the prototype)
const google::protobuf::Message * CreateMessage(PoolType poolType, std::string typeName);
private:
// Given a Protobuf FileDescriptor from the pool, parse all of its messages,
// find the keyable messages, and return a FileDescriptorProto,
// ready to be put in the normalizedDescriptorDb
google::protobuf::FileDescriptorProto ParseFile(
google::protobuf::FileDescriptorProto * ParseFile(
const google::protobuf::FileDescriptor * fileDescriptor);
// Parse a Descriptor and its nested messages
google::protobuf::DescriptorProto ParseAndNormalizeDescriptor(
google::protobuf::DescriptorProto * ParseAndNormalizeDescriptor(
const google::protobuf::Descriptor * descriptor);
// Check whether a Descriptor has a field with key option set
bool IsKeyable(const google::protobuf::Descriptor * descriptor) const;
class BoostLogErrorCollector
: public google::protobuf::DescriptorPool::ErrorCollector {
virtual void AddError(const std::string & filename,
const std::string & element_name,
const google::protobuf::Message * descriptor,
google::protobuf::DescriptorPool::ErrorCollector::ErrorLocation location,
const std::string & message) override;
virtual void AddWarning(const std::string & filename,
const std::string & element_name,
const google::protobuf::Message * descriptor,
google::protobuf::DescriptorPool::ErrorCollector::ErrorLocation location,
const std::string & message) override;
};
BoostLogErrorCollector errorCollector;
std::unique_ptr<RootSourceTree> includeSourceTree;
std::unique_ptr<RootSourceTree> schemaSourceTree;
......@@ -90,6 +121,9 @@ private:
// For each keyable message in schema, there is a normalized version
// which has Key in place of nested keyable messages
google::protobuf::DescriptorPool normalizedPool;
// The message factory is used to create new messages from the pools
google::protobuf::DynamicMessageFactory messageFactory;
};
}
}
......
......@@ -50,7 +50,9 @@ MessageTreeNode Marshaller::Marshal(const Message & message)
// The normalized message will be filled with data coming from input message,
// replacing references to other objects with their keys.
// It will then be serialized and set as storable message payload in messageTree;
Message * normalizedMessage = this->CreateMessage(NORMALIZED, message.GetTypeName());
Message * normalizedMessage
= this->loader->CreateMessage(Loader::NORMALIZED,
message.GetTypeName())->New();
// Only fields which are set in the message are processed
std::vector< const FieldDescriptor * > setFields;
......@@ -102,10 +104,14 @@ MessageTreeNode Marshaller::Marshal(const Message & message)
const Message & Marshaller::Unmarshal(const StorableMessage & storable)
{
// An empty normalized message is generated using the Key
Message * normalizedMessage = this->CreateMessage(NORMALIZED, storable.key().message_type());
Message * normalizedMessage
= this->loader->CreateMessage(Loader::NORMALIZED,
storable.key().message_type())->New();
// The original message is also retrieved
Message * originalMessage = this->CreateMessage(SCHEMA, storable.key().message_type());
Message * originalMessage
= this->loader->CreateMessage(Loader::SCHEMA,
storable.key().message_type())->New();
// StorableMessage payload contains the serialized normalized message,
// as previously stored into the DB
......@@ -136,7 +142,8 @@ const Message & Marshaller::Unmarshal(const StorableMessage & storable)
->MutableMessage(originalMessage, originalField)
->MergeFrom(nestedMessage);
}
else {
else { // if field is not a reference
// Just like in Marshal, other fields are simply copied over,
// as normalized and original descriptors look the same except for nested keyable messages
this->CopyField(normalizedField, *normalizedMessage, originalMessage);
......@@ -147,18 +154,6 @@ const Message & Marshaller::Unmarshal(const StorableMessage & storable)
return *originalMessage;
}
Message * Marshaller::CreateMessage(Marshaller::MessagePool pool, std::string type)
{
// DescriptorPool is either from Schema or Normalized
const DescriptorPool & descriptorPool =
(pool == SCHEMA)
? loader->GetSchemaPool()
: loader->GetNormalizedPool();
// TODO Check whether type exists
return this->messageFactory.GetPrototype(descriptorPool.FindMessageTypeByName(type))->New();
}
void Marshaller::CopyField(
const FieldDescriptor * fromField,
const Message & from,
......
......@@ -29,6 +29,8 @@
#include <google/protobuf/descriptor.h>
#include <google/protobuf/dynamic_message.h>
#include <boost/log/trivial.hpp>
#include <profanedb/format/marshaller.h>
#include "loader.h"
......@@ -47,12 +49,6 @@ public:
virtual profanedb::protobuf::MessageTreeNode Marshal(const google::protobuf::Message & message) override;
virtual const google::protobuf::Message & Unmarshal(const profanedb::protobuf::StorableMessage & storable) override;
enum MessagePool {
SCHEMA,
NORMALIZED
};
google::protobuf::Message * CreateMessage(MessagePool pool, std::string type);
private:
// Loader contains the schemaPool and normalizedPool
const std::shared_ptr<Loader> loader;
......@@ -61,8 +57,6 @@ private:
// Storage is used to recursively retrieve them.
const std::shared_ptr<profanedb::vault::Storage> storage;
google::protobuf::DynamicMessageFactory messageFactory;
// Copy a field from a message to another.
// Differs from MergeFrom because it doesn't check whether Descriptors match.
void CopyField(
......
......@@ -19,21 +19,28 @@
#include "storage.h"
profanedb::vault::rocksdb::Storage::Storage(std::unique_ptr<DB> rocksDb)
using profanedb::protobuf::StorableMessage;
using profanedb::protobuf::Key;
namespace profanedb {
namespace vault {
namespace rocksdb {
Storage::Storage(std::unique_ptr<DB> rocksDb)
: rocksDb(std::move(rocksDb))
{
}
void profanedb::vault::rocksdb::Storage::Store(const profanedb::protobuf::StorableMessage & storable)
void Storage::Store(const StorableMessage & storable)
{
this->rocksDb->Put(::rocksdb::WriteOptions(),
storable.key().SerializeAsString(),
storable.payload());
}
profanedb::protobuf::StorableMessage profanedb::vault::rocksdb::Storage::Retrieve(const profanedb::protobuf::Key & key) const
StorableMessage Storage::Retrieve(const Key & key) const
{
protobuf::StorableMessage stored;
StorableMessage stored;
*stored.mutable_key() = key;
......@@ -43,3 +50,7 @@ profanedb::protobuf::StorableMessage profanedb::vault::rocksdb::Storage::Retriev
return stored;
}
}
}
}
......@@ -2,6 +2,7 @@
#include <boost/test/included/unit_test.hpp>
#include <profanedb/test/protobuf/schema/test.pb.h>
#include <profanedb/protobuf/storage.pb.h>
#include <profanedb/format/protobuf/marshaller.h>
#include <profanedb/vault/rocksdb/storage.h>
......@@ -10,6 +11,7 @@ using profanedb::format::protobuf::Loader;
using profanedb::format::Marshaller;
using ProtobufMarshaller = profanedb::format::protobuf::Marshaller;
using profanedb::vault::Storage;
using profanedb::protobuf::MessageTreeNode;
// FIXME Should mock this
using RocksStorage = profanedb::vault::rocksdb::Storage;
......@@ -18,7 +20,8 @@ using google::protobuf::Message;
struct Format
{
std::shared_ptr<Marshaller<Message>> marshaller;
std::shared_ptr<Loader> loader;
std::shared_ptr<ProtobufMarshaller> marshaller;
Format()
{
......@@ -31,23 +34,23 @@ struct Format
auto storage = std::make_shared<RocksStorage>(std::unique_ptr<rocksdb::DB>(rocks));
auto includeSourceTree = new Loader::RootSourceTree{
"/usr/include", "/home/giorgio/Documents/ProfaneDB/ProfaneDBLib/src"};
"/usr/include", "/home/giorgio/Documents/ProfaneDB/ProfaneDB/src"};
auto schemaSourceTree = new Loader::RootSourceTree{"/home/giorgio/Documents/ProfaneDB/ProfaneDBLib/test/profanedb/test/protobuf/schema"};
auto schemaSourceTree = new Loader::RootSourceTree{"/home/giorgio/Documents/ProfaneDB/ProfaneDB/test/profanedb/test/protobuf/schema"};
auto loader = std::make_shared<Loader>(
this->loader = std::make_shared<Loader>(
std::unique_ptr<Loader::RootSourceTree>(includeSourceTree),
std::unique_ptr<Loader::RootSourceTree>(schemaSourceTree));
std::unique_ptr<Loader::RootSourceTree>(schemaSourceTree));
this->marshaller = std::make_shared<ProtobufMarshaller>(storage, loader);
}
};
BOOST_FIXTURE_TEST_CASE(first, Format)
BOOST_FIXTURE_TEST_CASE(marshal, Format)
{
schema::Test message;
BOOST_TEST_MESSAGE(message.DebugString());
marshaller->Marshal(message);
MessageTreeNode tree = marshaller->Marshal(message);
BOOST_TEST_MESSAGE(tree.DebugString());
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment