Commit 90855b5c authored by Giorgio Azzinnaro's avatar Giorgio Azzinnaro

added Loader

parent b2b7aef5
add_subdirectory(protobuf) # Generated Protobuf (& gRPC if neeeded) code add_subdirectory(protobuf) # Generated Protobuf (& gRPC if neeeded) code
add_subdirectory(boot) add_subdirectory(format)
add_subdirectory(vault) add_subdirectory(vault)
add_library(profanedb db.cpp) add_library(profanedb db.cpp)
......
add_library(profanedb_boot protobuf/schema.cpp)
target_link_libraries(profanedb_boot profanedb_protobuf)
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "schema.h"
using google::protobuf::Message;
using google::protobuf::Reflection;
using google::protobuf::Descriptor;
using google::protobuf::FieldDescriptor;
using profanedb::protobuf::Key;
profanedb::boot::protobuf::Schema::Schema()
{
}
profanedb::boot::protobuf::Schema::~Schema()
{
}
bool profanedb::boot::protobuf::Schema::IsKeyable(const Message & message) const
{
const Descriptor * descriptor = message.GetDescriptor();
for (int i = 0; i < descriptor->field_count(); i++) {
if (descriptor->field(i)->options().GetExtension(profanedb::protobuf::options).key())
return true;
}
return false;
}
Key profanedb::boot::protobuf::Schema::GetKey(const Message & message) const
{
const Descriptor * descriptor = message.GetDescriptor();
for (int i = 0; i < descriptor->field_count(); i++) {
if (descriptor->field(i)->options().GetExtension(profanedb::protobuf::options).key()) {
return FieldToKey(message, descriptor->field(i));
}
}
throw std::runtime_error(message.GetTypeName() + " is not keyable");
}
std::vector<const google::protobuf::Message *> profanedb::boot::protobuf::Schema::GetNestedMessages(
const Message & message) const
{
std::vector<const Message *> nested;
const Reflection * reflection = message.GetReflection();
const Descriptor * descriptor = message.GetDescriptor();
for (int i = 0; i < descriptor->field_count(); i++) {
if (descriptor->field(i)->type() == FieldDescriptor::TYPE_MESSAGE) {
nested.push_back(&reflection->GetMessage(message,descriptor->field(i)));
}
}
return nested;
}
Key profanedb::boot::protobuf::Schema::FieldToKey(
const Message & message,
const FieldDescriptor * fd)
{
Key key;
*key.mutable_message_type() = message.GetTypeName();
*key.mutable_field() = fd->name();
const Reflection * reflection = message.GetReflection();
if (fd->is_repeated()) {
for (int y = 0; y < reflection->FieldSize(message, fd); y++) {
switch (fd->cpp_type()) {
#define HANDLE_TYPE(CPPTYPE, METHOD) \
case FieldDescriptor::CPPTYPE_##CPPTYPE: \
*key.mutable_value() += "$" + std::to_string(reflection->GetRepeated##METHOD(message, fd, y)); \
break;
HANDLE_TYPE(INT32 , Int32 );
HANDLE_TYPE(INT64 , Int64 );
HANDLE_TYPE(UINT32, UInt32);
HANDLE_TYPE(UINT64, UInt64);
HANDLE_TYPE(DOUBLE, Double);
HANDLE_TYPE(FLOAT , Float );
HANDLE_TYPE(BOOL , Bool );
#undef HANDLE_TYPE
case FieldDescriptor::CPPTYPE_ENUM:
*key.mutable_value() += "$" + std::to_string(reflection->GetRepeatedEnum(message, fd, y)->index());
break;
case FieldDescriptor::CPPTYPE_STRING:
*key.mutable_value() += "$" + reflection->GetRepeatedString(message, fd, y);
break;
case FieldDescriptor::CPPTYPE_MESSAGE:
*key.mutable_value() += "$" + reflection->GetRepeatedMessage(message, fd, y).SerializeAsString();
break;
}
}
} else {
switch (fd->cpp_type()) {
#define HANDLE_TYPE(CPPTYPE, METHOD) \
case FieldDescriptor::CPPTYPE_##CPPTYPE: \
*key.mutable_value() = std::to_string(reflection->Get##METHOD(message, fd)); \
break;
HANDLE_TYPE(INT32 , Int32 );
HANDLE_TYPE(INT64 , Int64 );
HANDLE_TYPE(UINT32, UInt32);
HANDLE_TYPE(UINT64, UInt64);
HANDLE_TYPE(DOUBLE, Double);
HANDLE_TYPE(FLOAT , Float );
HANDLE_TYPE(BOOL , Bool );
#undef HANDLE_TYPE
case FieldDescriptor::CPPTYPE_ENUM:
*key.mutable_value() = std::to_string(reflection->GetEnum(message, fd)->index());
break;
case FieldDescriptor::CPPTYPE_STRING:
*key.mutable_value() = reflection->GetString(message, fd);
break;
case FieldDescriptor::CPPTYPE_MESSAGE:
*key.mutable_value() = reflection->GetMessage(message, fd).SerializeAsString();
break;
}
}
return key;
}
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef PROFANEDB_BOOT_PROTOBUF_SCHEMA_H
#define PROFANEDB_BOOT_PROTOBUF_SCHEMA_H
#include <google/protobuf/message.h>
#include <profanedb/protobuf/storage.pb.h>
#include <profanedb/protobuf/options.pb.h>
#include <profanedb/boot/schema.h>
namespace profanedb {
namespace boot {
namespace protobuf {
// Redundant here, might be relevant for other kind of "Message" classes,
// to copy and paste from profanedb::boot::Schema interface
// Can be changed if collisions occur
using Message = google::protobuf::Message;
// Protobuf effectively stores all info regarding a Message in its Descriptor,
// so considering the Message was created before, we don't require any dependency here
class Schema : public profanedb::boot::Schema <Message>
{
public:
Schema();
~Schema();
// Check whether a Message has a key, therefore can be stored
virtual bool IsKeyable(const Message & message) const override;
// The Key is defined as an option of the given message
virtual profanedb::protobuf::Key GetKey(const Message & message) const override;
// Retrieve nested messages from a message
virtual std::vector<const Message *> GetNestedMessages(const Message & message) const override;
private:
static profanedb::protobuf::Key FieldToKey(const Message & message, const google::protobuf::FieldDescriptor * fd);
};
}
}
}
#endif // PROFANEDB_BOOT_PROTOBUFSCHEMA_H
/*
* ProfaneDB - A Protocol Buffers database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef PROFANEDB_BOOT_SCHEMA_H
#define PROFANEDB_BOOT_SCHEMA_H
#include <profanedb/protobuf/storage.pb.h>
namespace profanedb {
namespace boot {
// A Schema defines a Key for each message,
// a Key is a unique identifier for that given message.
// This data might be retrieved directly from the message,
// or the Schema might need to parse some definition to know what to do
template <typename Message>
class Schema {
public:
virtual ~Schema() = 0;
// Check whether a Message has a key, therefore can be stored
virtual bool IsKeyable(const Message & message) const = 0;
// Extract a Key from a Message
virtual profanedb::protobuf::Key GetKey(const Message & message) const = 0;
// Retrieve nested messages from a message
virtual std::vector<const Message *> GetNestedMessages(const Message & message) const = 0;
};
}
}
#endif // PROFANEDB_BOOT_SCHEMA_H
...@@ -21,9 +21,9 @@ ...@@ -21,9 +21,9 @@
template<typename Message> template<typename Message>
profanedb::Db<Message>::Db( profanedb::Db<Message>::Db(
std::shared_ptr< profanedb::boot::Schema<Message> > schema, std::shared_ptr< format::Marshaller<Message> > marshaller,
std::shared_ptr<profanedb::vault::Storage> storage) std::shared_ptr<profanedb::vault::Storage> storage)
: schema(schema) : marshaller(marshaller)
, storage(storage) , storage(storage)
{ {
} }
...@@ -36,21 +36,20 @@ profanedb::Db<Message>::~Db() ...@@ -36,21 +36,20 @@ profanedb::Db<Message>::~Db()
template<typename Message> template<typename Message>
const Message & profanedb::Db<Message>::Get(const protobuf::Key & key) const const Message & profanedb::Db<Message>::Get(const protobuf::Key & key) const
{ {
this->storage->Retrieve(key); return this->marshaller->Unmarshal(this->storage->Retrieve(key));
// Unmarshal message
// For each nested key retrieve and set message
// TODO Storable to Message and return
} }
template<typename Message> template<typename Message>
bool profanedb::Db<Message>::Put(const Message & message) bool profanedb::Db<Message>::Put(const Message & message)
{ {
// TODO Message to Storable this->storage->Store(this->marshaller->Marshal(message));
// storage->Store()
// TODO Check exceptions
return true;
} }
template<typename Message> template<typename Message>
bool profanedb::Db<Message>::Delete(const protobuf::Key & key) bool profanedb::Db<Message>::Delete(const protobuf::Key & key)
{ {
// TODO in Storage
} }
...@@ -17,16 +17,16 @@ ...@@ -17,16 +17,16 @@
* *
*/ */
#ifndef PROFANEDB_DB_H
#define PROFANEDB_DB_H
#include <memory> #include <memory>
#include <profanedb/boot/schema.h> #include <profanedb/format/marshaller.h>
#include <profanedb/vault/storage.h> #include <profanedb/vault/storage.h>
#include <profanedb/protobuf/storage.pb.h> #include <profanedb/protobuf/storage.pb.h>
#ifndef PROFANEDB_DB_H
#define PROFANEDB_DB_H
namespace profanedb { namespace profanedb {
// Db should be the main interface when embedding ProfaneDB // Db should be the main interface when embedding ProfaneDB
...@@ -35,7 +35,7 @@ class Db ...@@ -35,7 +35,7 @@ class Db
{ {
public: public:
Db( Db(
std::shared_ptr< boot::Schema<Message> > schema, std::shared_ptr< format::Marshaller<Message> > marshaller,
std::shared_ptr<vault::Storage> storage); std::shared_ptr<vault::Storage> storage);
~Db(); ~Db();
...@@ -44,7 +44,7 @@ public: ...@@ -44,7 +44,7 @@ public:
virtual bool Delete(const protobuf::Key & key); virtual bool Delete(const protobuf::Key & key);
private: private:
std::shared_ptr< boot::Schema<Message> > schema; std::shared_ptr< format::Marshaller<Message> > marshaller;
std::shared_ptr<vault::Storage> storage; std::shared_ptr<vault::Storage> storage;
}; };
} }
......
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "loader.h"
using boost::filesystem::path;
using boost::filesystem::recursive_directory_iterator;
using boost::filesystem::symlink_option;
using google::protobuf::io::ZeroCopyInputStream;
using google::protobuf::MergedDescriptorDatabase;
using google::protobuf::compiler::SourceTreeDescriptorDatabase;
using google::protobuf::FileDescriptor;
using google::protobuf::FileDescriptorProto;
using google::protobuf::Descriptor;
using google::protobuf::DescriptorProto;
using google::protobuf::FieldDescriptor;
using google::protobuf::FieldDescriptorProto_Type;
using profanedb::protobuf::Key;
profanedb::format::protobuf::Loader::RootSourceTree::RootSourceTree(std::initializer_list<path> paths)
{
for (const auto & path: paths)
this->MapPath("", path.string());
ZeroCopyInputStream * inputStream = this->Open("");
if (inputStream == nullptr)
throw std::runtime_error(this->GetLastErrorMessage());
}
profanedb::format::protobuf::Loader::Loader(
std::unique_ptr<RootSourceTree> include,
std::unique_ptr<RootSourceTree> schema)
: includeSourceTree(std::move(include))
, schemaSourceTree(std::move(schema))
, includeDb(includeSourceTree.get())
, schemaDb(schemaSourceTree.get())
, schemaPool(new MergedDescriptorDatabase(&includeDb, &schemaDb))
, normalizedPool(&normalizedDescriptorDb)
{
// profanedb.protobuf.options.key is defined in here
// It is used to mark the primary key on Messages
schemaPool.FindFileByName("profanedb/protobuf/options.proto");
// Just in case schema is defined in more than one place
for (const auto & path: schemaSourceTree->paths) {
// Iterate through all files in that mapped path
for (const auto & file: recursive_directory_iterator(path, symlink_option::recurse)) {
// Only consider files ending in .proto
// TODO This might be configured differently
if (file.path().extension() == ".proto") {
// The file is now retrieved, and its path for Protobuf must be relative to the mapping
this->ParseFile(schemaPool.FindFileByName(file.path().lexically_relative(path).string()));
}
}
}
}
FileDescriptorProto profanedb::format::protobuf::Loader::ParseFile(
const FileDescriptor * fileDescriptor)
{
// A FileDescriptorProto is needed to edit messages and populate the normalized descriptor database
FileDescriptorProto normFileDescProto;
fileDescriptor->CopyTo(&normFileDescProto);
// For each message in the file...
for (int i = 0; i < fileDescriptor->message_type_count(); i++) {
// ... parse it, make nested messages Key objects
*normFileDescProto.mutable_message_type(i) = this->ParseAndNormalizeDescriptor(fileDescriptor->message_type(i));
}
return normFileDescProto;
}
DescriptorProto profanedb::format::protobuf::Loader::ParseAndNormalizeDescriptor(
const Descriptor * descriptor)
{
DescriptorProto normDescProto;
descriptor->CopyTo(&normDescProto);
// Recurse for all messages DEFINED within this message
for (int j = 0; j < descriptor->nested_type_count(); j++) {
*normDescProto.mutable_nested_type(j) = this->ParseAndNormalizeDescriptor(descriptor->nested_type(j));
}
// Now the actual Descriptor normalization
for (int k = 0; k < descriptor->field_count(); k++) {
const FieldDescriptor * field = descriptor->field(k);
// NULL if not a message
const Descriptor * nestedMessage = field->message_type();
// If this field is effectively a message,
// and that Message is keyable...
if(nestedMessage != nullptr && this->IsKeyable(nestedMessage)) {
// ... make the field in the normalized descriptor a Key object
// TODO Maybe we could use an option here as well
normDescProto.mutable_field(k)->set_type(
FieldDescriptorProto_Type::FieldDescriptorProto_Type_TYPE_MESSAGE); // Redundant, is message already
normDescProto.mutable_field(k)->set_type_name(Key::descriptor()->full_name()); // TODO Should include Key in normalizedPool
}
}
return normDescProto;
}
bool profanedb::format::protobuf::Loader::IsKeyable(const Descriptor * descriptor) const
{
for (int i = 0; i < descriptor->field_count(); i++) {
// If any field in message has profanedb::protobuf::options::key set
if (descriptor->field(i)->options().GetExtension(profanedb::protobuf::options).key())
return true;
}
return false;
}
const google::protobuf::DescriptorPool & profanedb::format::protobuf::Loader::GetSchemaPool() const
{
return this->schemaPool;
}
const google::protobuf::DescriptorPool & profanedb::format::protobuf::Loader::GetNormalizedPool() const
{
return this->normalizedPool;
}
...@@ -17,56 +17,72 @@ ...@@ -17,56 +17,72 @@
* *
*/ */
#ifndef PROFANEDB_STORAGE_LOADER_H #ifndef PROFANEDB_FORMAT_PROTOBUF_LOADER_H
#define PROFANEDB_STORAGE_LOADER_H #define PROFANEDB_FORMAT_PROTOBUF_LOADER_H
#include <memory> #include <profanedb/protobuf/options.pb.h>
#include <profanedb/protobuf/storage.pb.h>
#include <boost/filesystem.hpp> #include <google/protobuf/descriptor.h>
#include <google/protobuf/compiler/importer.h>
#include "parser.h" #include <boost/filesystem.hpp>
namespace profanedb { namespace profanedb {
namespace storage { namespace format {
namespace protobuf {
using boost::filesystem::path;
using boost::filesystem::recursive_directory_iterator;
using boost::filesystem::symlink_option;
using google::protobuf::DescriptorDatabase;
using google::protobuf::compiler::SourceTreeDescriptorDatabase;
using google::protobuf::FileDescriptor;
using google::protobuf::FileDescriptorProto;
using google::protobuf::Descriptor;
// A Loader
class Loader class Loader
{ {
public: public:
Loader(std::shared_ptr<Parser> parser); // RootSourceTree is a utility class that creates a DiskSourceTree,
// mapping all the paths provided to the root ("/") path for easier import.
// Paths are available for Loader to populate its Pool;
class RootSourceTree : public google::protobuf::compiler::DiskSourceTree {
friend Loader;
RootSourceTree(std::initializer_list<boost::filesystem::path> paths);
private:
std::vector<boost::filesystem::path> paths;
};
void SetIncludePaths(std::initializer_list<path> paths); // include has the import directories
void LoadSchema(path path); // ("/usr/include/[google/protobuf/*.proto]", "src/[profanedb/protobuf/*.proto]")
// schema the user proto files, with ProfaneDB options set
Loader(
std::unique_ptr<RootSourceTree> include,
std::unique_ptr<RootSourceTree> schema);
std::unique_ptr<DescriptorPool> GetSchemaPool(); const google::protobuf::DescriptorPool & GetSchemaPool() const;
const google::protobuf::DescriptorPool & GetNormalizedPool() const;
private: private:
std::shared_ptr<Parser> parser; // Given a Protobuf FileDescriptor from the pool, parse all of its messages,
// find the keyable messages, and return a FileDescriptorProto,
// ready to be put in the normalizedDescriptorDb
google::protobuf::FileDescriptorProto ParseFile(
const google::protobuf::FileDescriptor * fileDescriptor);
// DescriptorDatabases must outlive their DescriptorPool, // Parse a Descriptor and its nested messages
// so other classes can access schemaPool, google::protobuf::DescriptorProto ParseAndNormalizeDescriptor(
// but make sure Loader doesn't go out of scope const google::protobuf::Descriptor * descriptor);
std::unique_ptr<DescriptorDatabase> includeDescDb;
std::unique_ptr<DescriptorDatabase> schemaDescDb;
std::unique_ptr<DescriptorPool> schemaPool;
// A DiskSourceTree where paths are automatically mapped to root ("") // Check whether a Descriptor has a field with key option set
class RootSourceTree : public google::protobuf::compiler::DiskSourceTree { bool IsKeyable(const google::protobuf::Descriptor * descriptor) const;
public:
RootSourceTree(std::initializer_list<path> mappings); std::unique_ptr<RootSourceTree> includeSourceTree;
}; std::unique_ptr<RootSourceTree> schemaSourceTree;
google::protobuf::compiler::SourceTreeDescriptorDatabase includeDb;
google::protobuf::compiler::SourceTreeDescriptorDatabase schemaDb;
google::protobuf::SimpleDescriptorDatabase normalizedDescriptorDb;
google::protobuf::DescriptorPool schemaPool;
google::protobuf::DescriptorPool normalizedPool;
}; };
} }
} }
}
#endif // PROFANEDB_STORAGE_LOADER_H #endif // PROFANEDB_FORMAT_PROTOBUF_LOADER_H
/* /*
* <one line to give the program's name and a brief idea of what it does.> * ProfaneDB - A Protocol Buffers database.
* Copyright (C) 2017 <copyright holder> <email> * Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
...@@ -32,8 +32,8 @@ using profanedb::protobuf::StorableMessage; ...@@ -32,8 +32,8 @@ using profanedb::protobuf::StorableMessage;
using profanedb::protobuf::Key; using profanedb::protobuf::Key;
profanedb::format::protobuf::Marshaller::Marshaller( profanedb::format::protobuf::Marshaller::Marshaller(
DescriptorPool & schemaPool, const DescriptorPool & schemaPool,
DescriptorPool & normalizedPool, const DescriptorPool & normalizedPool,
const Storage & storage) const Storage & storage)
: schemaPool(schemaPool) : schemaPool(schemaPool)
, normalizedPool(normalizedPool) , normalizedPool(normalizedPool)
......
/* /*
* <one line to give the program's name and a brief idea of what it does.> * ProfaneDB - A Protocol Buffers database.
* Copyright (C) 2017 <copyright holder> <email> * Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
...@@ -37,24 +37,24 @@ class Marshaller : profanedb::format::Marshaller<google::protobuf::Message> ...@@ -37,24 +37,24 @@ class Marshaller : profanedb::format::Marshaller<google::protobuf::Message>
{ {
public: public:
Marshaller( Marshaller(
google::protobuf::DescriptorPool & schemaPool, const google::protobuf::DescriptorPool & schemaPool,
google::protobuf::DescriptorPool & normalizedPool, const google::protobuf::DescriptorPool & normalizedPool,
const profanedb::vault::Storage & storage const profanedb::vault::Storage & storage
); );
~Marshaller(); ~Marshaller();
virtual MessageTreeNode Marshal(const google::protobuf::Message & message) override; virtual profanedb::protobuf::MessageTreeNode Marshal(const google::protobuf::Message & message) override;
virtual const google::protobuf::Message & Unmarshal(const StorableMessage & storable) override; virtual const google::protobuf::Message & Unmarshal(const profanedb::protobuf::StorableMessage & storable) override;
private: private:
// TODO schemaPool, normalizedPool and CopyField are strictly related, should have their class // TODO schemaPool, normalizedPool and CopyField are strictly related, should have their class
// schemaPool keeps track of the original messages // schemaPool keeps track of the original messages
google::protobuf::DescriptorPool & schemaPool; const google::protobuf::DescriptorPool & schemaPool;