Commit 90855b5c authored by Giorgio Azzinnaro's avatar Giorgio Azzinnaro

added Loader

parent b2b7aef5
add_subdirectory(protobuf) # Generated Protobuf (& gRPC if neeeded) code
add_subdirectory(boot)
add_subdirectory(format)
add_subdirectory(vault)
add_library(profanedb db.cpp)
......
add_library(profanedb_boot protobuf/schema.cpp)
target_link_libraries(profanedb_boot profanedb_protobuf)
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "schema.h"
using google::protobuf::Message;
using google::protobuf::Reflection;
using google::protobuf::Descriptor;
using google::protobuf::FieldDescriptor;
using profanedb::protobuf::Key;
profanedb::boot::protobuf::Schema::Schema()
{
}
profanedb::boot::protobuf::Schema::~Schema()
{
}
bool profanedb::boot::protobuf::Schema::IsKeyable(const Message & message) const
{
const Descriptor * descriptor = message.GetDescriptor();
for (int i = 0; i < descriptor->field_count(); i++) {
if (descriptor->field(i)->options().GetExtension(profanedb::protobuf::options).key())
return true;
}
return false;
}
Key profanedb::boot::protobuf::Schema::GetKey(const Message & message) const
{
const Descriptor * descriptor = message.GetDescriptor();
for (int i = 0; i < descriptor->field_count(); i++) {
if (descriptor->field(i)->options().GetExtension(profanedb::protobuf::options).key()) {
return FieldToKey(message, descriptor->field(i));
}
}
throw std::runtime_error(message.GetTypeName() + " is not keyable");
}
std::vector<const google::protobuf::Message *> profanedb::boot::protobuf::Schema::GetNestedMessages(
const Message & message) const
{
std::vector<const Message *> nested;
const Reflection * reflection = message.GetReflection();
const Descriptor * descriptor = message.GetDescriptor();
for (int i = 0; i < descriptor->field_count(); i++) {
if (descriptor->field(i)->type() == FieldDescriptor::TYPE_MESSAGE) {
nested.push_back(&reflection->GetMessage(message,descriptor->field(i)));
}
}
return nested;
}
Key profanedb::boot::protobuf::Schema::FieldToKey(
const Message & message,
const FieldDescriptor * fd)
{
Key key;
*key.mutable_message_type() = message.GetTypeName();
*key.mutable_field() = fd->name();
const Reflection * reflection = message.GetReflection();
if (fd->is_repeated()) {
for (int y = 0; y < reflection->FieldSize(message, fd); y++) {
switch (fd->cpp_type()) {
#define HANDLE_TYPE(CPPTYPE, METHOD) \
case FieldDescriptor::CPPTYPE_##CPPTYPE: \
*key.mutable_value() += "$" + std::to_string(reflection->GetRepeated##METHOD(message, fd, y)); \
break;
HANDLE_TYPE(INT32 , Int32 );
HANDLE_TYPE(INT64 , Int64 );
HANDLE_TYPE(UINT32, UInt32);
HANDLE_TYPE(UINT64, UInt64);
HANDLE_TYPE(DOUBLE, Double);
HANDLE_TYPE(FLOAT , Float );
HANDLE_TYPE(BOOL , Bool );
#undef HANDLE_TYPE
case FieldDescriptor::CPPTYPE_ENUM:
*key.mutable_value() += "$" + std::to_string(reflection->GetRepeatedEnum(message, fd, y)->index());
break;
case FieldDescriptor::CPPTYPE_STRING:
*key.mutable_value() += "$" + reflection->GetRepeatedString(message, fd, y);
break;
case FieldDescriptor::CPPTYPE_MESSAGE:
*key.mutable_value() += "$" + reflection->GetRepeatedMessage(message, fd, y).SerializeAsString();
break;
}
}
} else {
switch (fd->cpp_type()) {
#define HANDLE_TYPE(CPPTYPE, METHOD) \
case FieldDescriptor::CPPTYPE_##CPPTYPE: \
*key.mutable_value() = std::to_string(reflection->Get##METHOD(message, fd)); \
break;
HANDLE_TYPE(INT32 , Int32 );
HANDLE_TYPE(INT64 , Int64 );
HANDLE_TYPE(UINT32, UInt32);
HANDLE_TYPE(UINT64, UInt64);
HANDLE_TYPE(DOUBLE, Double);
HANDLE_TYPE(FLOAT , Float );
HANDLE_TYPE(BOOL , Bool );
#undef HANDLE_TYPE
case FieldDescriptor::CPPTYPE_ENUM:
*key.mutable_value() = std::to_string(reflection->GetEnum(message, fd)->index());
break;
case FieldDescriptor::CPPTYPE_STRING:
*key.mutable_value() = reflection->GetString(message, fd);
break;
case FieldDescriptor::CPPTYPE_MESSAGE:
*key.mutable_value() = reflection->GetMessage(message, fd).SerializeAsString();
break;
}
}
return key;
}
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef PROFANEDB_BOOT_PROTOBUF_SCHEMA_H
#define PROFANEDB_BOOT_PROTOBUF_SCHEMA_H
#include <google/protobuf/message.h>
#include <profanedb/protobuf/storage.pb.h>
#include <profanedb/protobuf/options.pb.h>
#include <profanedb/boot/schema.h>
namespace profanedb {
namespace boot {
namespace protobuf {
// Redundant here, might be relevant for other kind of "Message" classes,
// to copy and paste from profanedb::boot::Schema interface
// Can be changed if collisions occur
using Message = google::protobuf::Message;
// Protobuf effectively stores all info regarding a Message in its Descriptor,
// so considering the Message was created before, we don't require any dependency here
class Schema : public profanedb::boot::Schema <Message>
{
public:
Schema();
~Schema();
// Check whether a Message has a key, therefore can be stored
virtual bool IsKeyable(const Message & message) const override;
// The Key is defined as an option of the given message
virtual profanedb::protobuf::Key GetKey(const Message & message) const override;
// Retrieve nested messages from a message
virtual std::vector<const Message *> GetNestedMessages(const Message & message) const override;
private:
static profanedb::protobuf::Key FieldToKey(const Message & message, const google::protobuf::FieldDescriptor * fd);
};
}
}
}
#endif // PROFANEDB_BOOT_PROTOBUFSCHEMA_H
/*
* ProfaneDB - A Protocol Buffers database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef PROFANEDB_BOOT_SCHEMA_H
#define PROFANEDB_BOOT_SCHEMA_H
#include <profanedb/protobuf/storage.pb.h>
namespace profanedb {
namespace boot {
// A Schema defines a Key for each message,
// a Key is a unique identifier for that given message.
// This data might be retrieved directly from the message,
// or the Schema might need to parse some definition to know what to do
template <typename Message>
class Schema {
public:
virtual ~Schema() = 0;
// Check whether a Message has a key, therefore can be stored
virtual bool IsKeyable(const Message & message) const = 0;
// Extract a Key from a Message
virtual profanedb::protobuf::Key GetKey(const Message & message) const = 0;
// Retrieve nested messages from a message
virtual std::vector<const Message *> GetNestedMessages(const Message & message) const = 0;
};
}
}
#endif // PROFANEDB_BOOT_SCHEMA_H
......@@ -21,9 +21,9 @@
template<typename Message>
profanedb::Db<Message>::Db(
std::shared_ptr< profanedb::boot::Schema<Message> > schema,
std::shared_ptr< format::Marshaller<Message> > marshaller,
std::shared_ptr<profanedb::vault::Storage> storage)
: schema(schema)
: marshaller(marshaller)
, storage(storage)
{
}
......@@ -36,21 +36,20 @@ profanedb::Db<Message>::~Db()
template<typename Message>
const Message & profanedb::Db<Message>::Get(const protobuf::Key & key) const
{
this->storage->Retrieve(key);
// Unmarshal message
// For each nested key retrieve and set message
// TODO Storable to Message and return
return this->marshaller->Unmarshal(this->storage->Retrieve(key));
}
template<typename Message>
bool profanedb::Db<Message>::Put(const Message & message)
{
// TODO Message to Storable
// storage->Store()
this->storage->Store(this->marshaller->Marshal(message));
// TODO Check exceptions
return true;
}
template<typename Message>
bool profanedb::Db<Message>::Delete(const protobuf::Key & key)
{
// TODO in Storage
}
......@@ -17,16 +17,16 @@
*
*/
#ifndef PROFANEDB_DB_H
#define PROFANEDB_DB_H
#include <memory>
#include <profanedb/boot/schema.h>
#include <profanedb/format/marshaller.h>
#include <profanedb/vault/storage.h>
#include <profanedb/protobuf/storage.pb.h>
#ifndef PROFANEDB_DB_H
#define PROFANEDB_DB_H
namespace profanedb {
// Db should be the main interface when embedding ProfaneDB
......@@ -35,7 +35,7 @@ class Db
{
public:
Db(
std::shared_ptr< boot::Schema<Message> > schema,
std::shared_ptr< format::Marshaller<Message> > marshaller,
std::shared_ptr<vault::Storage> storage);
~Db();
......@@ -44,7 +44,7 @@ public:
virtual bool Delete(const protobuf::Key & key);
private:
std::shared_ptr< boot::Schema<Message> > schema;
std::shared_ptr< format::Marshaller<Message> > marshaller;
std::shared_ptr<vault::Storage> storage;
};
}
......
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "loader.h"
using boost::filesystem::path;
using boost::filesystem::recursive_directory_iterator;
using boost::filesystem::symlink_option;
using google::protobuf::io::ZeroCopyInputStream;
using google::protobuf::MergedDescriptorDatabase;
using google::protobuf::compiler::SourceTreeDescriptorDatabase;
using google::protobuf::FileDescriptor;
using google::protobuf::FileDescriptorProto;
using google::protobuf::Descriptor;
using google::protobuf::DescriptorProto;
using google::protobuf::FieldDescriptor;
using google::protobuf::FieldDescriptorProto_Type;
using profanedb::protobuf::Key;
profanedb::format::protobuf::Loader::RootSourceTree::RootSourceTree(std::initializer_list<path> paths)
{
for (const auto & path: paths)
this->MapPath("", path.string());
ZeroCopyInputStream * inputStream = this->Open("");
if (inputStream == nullptr)
throw std::runtime_error(this->GetLastErrorMessage());
}
profanedb::format::protobuf::Loader::Loader(
std::unique_ptr<RootSourceTree> include,
std::unique_ptr<RootSourceTree> schema)
: includeSourceTree(std::move(include))
, schemaSourceTree(std::move(schema))
, includeDb(includeSourceTree.get())
, schemaDb(schemaSourceTree.get())
, schemaPool(new MergedDescriptorDatabase(&includeDb, &schemaDb))
, normalizedPool(&normalizedDescriptorDb)
{
// profanedb.protobuf.options.key is defined in here
// It is used to mark the primary key on Messages
schemaPool.FindFileByName("profanedb/protobuf/options.proto");
// Just in case schema is defined in more than one place
for (const auto & path: schemaSourceTree->paths) {
// Iterate through all files in that mapped path
for (const auto & file: recursive_directory_iterator(path, symlink_option::recurse)) {
// Only consider files ending in .proto
// TODO This might be configured differently
if (file.path().extension() == ".proto") {
// The file is now retrieved, and its path for Protobuf must be relative to the mapping
this->ParseFile(schemaPool.FindFileByName(file.path().lexically_relative(path).string()));
}
}
}
}
FileDescriptorProto profanedb::format::protobuf::Loader::ParseFile(
const FileDescriptor * fileDescriptor)
{
// A FileDescriptorProto is needed to edit messages and populate the normalized descriptor database
FileDescriptorProto normFileDescProto;
fileDescriptor->CopyTo(&normFileDescProto);
// For each message in the file...
for (int i = 0; i < fileDescriptor->message_type_count(); i++) {
// ... parse it, make nested messages Key objects
*normFileDescProto.mutable_message_type(i) = this->ParseAndNormalizeDescriptor(fileDescriptor->message_type(i));
}
return normFileDescProto;
}
DescriptorProto profanedb::format::protobuf::Loader::ParseAndNormalizeDescriptor(
const Descriptor * descriptor)
{
DescriptorProto normDescProto;
descriptor->CopyTo(&normDescProto);
// Recurse for all messages DEFINED within this message
for (int j = 0; j < descriptor->nested_type_count(); j++) {
*normDescProto.mutable_nested_type(j) = this->ParseAndNormalizeDescriptor(descriptor->nested_type(j));
}
// Now the actual Descriptor normalization
for (int k = 0; k < descriptor->field_count(); k++) {
const FieldDescriptor * field = descriptor->field(k);
// NULL if not a message
const Descriptor * nestedMessage = field->message_type();
// If this field is effectively a message,
// and that Message is keyable...
if(nestedMessage != nullptr && this->IsKeyable(nestedMessage)) {
// ... make the field in the normalized descriptor a Key object
// TODO Maybe we could use an option here as well
normDescProto.mutable_field(k)->set_type(
FieldDescriptorProto_Type::FieldDescriptorProto_Type_TYPE_MESSAGE); // Redundant, is message already
normDescProto.mutable_field(k)->set_type_name(Key::descriptor()->full_name()); // TODO Should include Key in normalizedPool
}
}
return normDescProto;
}
bool profanedb::format::protobuf::Loader::IsKeyable(const Descriptor * descriptor) const
{
for (int i = 0; i < descriptor->field_count(); i++) {
// If any field in message has profanedb::protobuf::options::key set
if (descriptor->field(i)->options().GetExtension(profanedb::protobuf::options).key())
return true;
}
return false;
}
const google::protobuf::DescriptorPool & profanedb::format::protobuf::Loader::GetSchemaPool() const
{
return this->schemaPool;
}
const google::protobuf::DescriptorPool & profanedb::format::protobuf::Loader::GetNormalizedPool() const
{
return this->normalizedPool;
}
......@@ -17,56 +17,72 @@
*
*/
#ifndef PROFANEDB_STORAGE_LOADER_H
#define PROFANEDB_STORAGE_LOADER_H
#ifndef PROFANEDB_FORMAT_PROTOBUF_LOADER_H
#define PROFANEDB_FORMAT_PROTOBUF_LOADER_H
#include <memory>
#include <profanedb/protobuf/options.pb.h>
#include <profanedb/protobuf/storage.pb.h>
#include <boost/filesystem.hpp>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/compiler/importer.h>
#include "parser.h"
#include <boost/filesystem.hpp>
namespace profanedb {
namespace storage {
using boost::filesystem::path;
using boost::filesystem::recursive_directory_iterator;
using boost::filesystem::symlink_option;
using google::protobuf::DescriptorDatabase;
using google::protobuf::compiler::SourceTreeDescriptorDatabase;
using google::protobuf::FileDescriptor;
using google::protobuf::FileDescriptorProto;
using google::protobuf::Descriptor;
namespace format {
namespace protobuf {
// A Loader
class Loader
{
public:
Loader(std::shared_ptr<Parser> parser);
// RootSourceTree is a utility class that creates a DiskSourceTree,
// mapping all the paths provided to the root ("/") path for easier import.
// Paths are available for Loader to populate its Pool;
class RootSourceTree : public google::protobuf::compiler::DiskSourceTree {
friend Loader;
RootSourceTree(std::initializer_list<boost::filesystem::path> paths);
private:
std::vector<boost::filesystem::path> paths;
};
void SetIncludePaths(std::initializer_list<path> paths);
void LoadSchema(path path);
std::unique_ptr<DescriptorPool> GetSchemaPool();
// include has the import directories
// ("/usr/include/[google/protobuf/*.proto]", "src/[profanedb/protobuf/*.proto]")
// schema the user proto files, with ProfaneDB options set
Loader(
std::unique_ptr<RootSourceTree> include,
std::unique_ptr<RootSourceTree> schema);
const google::protobuf::DescriptorPool & GetSchemaPool() const;
const google::protobuf::DescriptorPool & GetNormalizedPool() const;
private:
std::shared_ptr<Parser> parser;
// Given a Protobuf FileDescriptor from the pool, parse all of its messages,
// find the keyable messages, and return a FileDescriptorProto,
// ready to be put in the normalizedDescriptorDb
google::protobuf::FileDescriptorProto ParseFile(
const google::protobuf::FileDescriptor * fileDescriptor);
// DescriptorDatabases must outlive their DescriptorPool,
// so other classes can access schemaPool,
// but make sure Loader doesn't go out of scope
std::unique_ptr<DescriptorDatabase> includeDescDb;
std::unique_ptr<DescriptorDatabase> schemaDescDb;
std::unique_ptr<DescriptorPool> schemaPool;
// Parse a Descriptor and its nested messages
google::protobuf::DescriptorProto ParseAndNormalizeDescriptor(
const google::protobuf::Descriptor * descriptor);
// A DiskSourceTree where paths are automatically mapped to root ("")
class RootSourceTree : public google::protobuf::compiler::DiskSourceTree {
public:
RootSourceTree(std::initializer_list<path> mappings);
};
// Check whether a Descriptor has a field with key option set
bool IsKeyable(const google::protobuf::Descriptor * descriptor) const;
std::unique_ptr<RootSourceTree> includeSourceTree;
std::unique_ptr<RootSourceTree> schemaSourceTree;
google::protobuf::compiler::SourceTreeDescriptorDatabase includeDb;
google::protobuf::compiler::SourceTreeDescriptorDatabase schemaDb;
google::protobuf::SimpleDescriptorDatabase normalizedDescriptorDb;
google::protobuf::DescriptorPool schemaPool;
google::protobuf::DescriptorPool normalizedPool;
};
}
}
}
#endif // PROFANEDB_STORAGE_LOADER_H
#endif // PROFANEDB_FORMAT_PROTOBUF_LOADER_H
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
* ProfaneDB - A Protocol Buffers database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
......@@ -32,8 +32,8 @@ using profanedb::protobuf::StorableMessage;
using profanedb::protobuf::Key;
profanedb::format::protobuf::Marshaller::Marshaller(
DescriptorPool & schemaPool,
DescriptorPool & normalizedPool,
const DescriptorPool & schemaPool,
const DescriptorPool & normalizedPool,
const Storage & storage)
: schemaPool(schemaPool)
, normalizedPool(normalizedPool)
......
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
* ProfaneDB - A Protocol Buffers database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
......@@ -37,24 +37,24 @@ class Marshaller : profanedb::format::Marshaller<google::protobuf::Message>
{
public:
Marshaller(
google::protobuf::DescriptorPool & schemaPool,
google::protobuf::DescriptorPool & normalizedPool,
const google::protobuf::DescriptorPool & schemaPool,
const google::protobuf::DescriptorPool & normalizedPool,
const profanedb::vault::Storage & storage
);
~Marshaller();
virtual MessageTreeNode Marshal(const google::protobuf::Message & message) override;
virtual const google::protobuf::Message & Unmarshal(const StorableMessage & storable) override;
virtual profanedb::protobuf::MessageTreeNode Marshal(const google::protobuf::Message & message) override;
virtual const google::protobuf::Message & Unmarshal(const profanedb::protobuf::StorableMessage & storable) override;
private:
// TODO schemaPool, normalizedPool and CopyField are strictly related, should have their class
// schemaPool keeps track of the original messages
google::protobuf::DescriptorPool & schemaPool;
const google::protobuf::DescriptorPool & schemaPool;
// For each keyable message in schema, there is a normalized version
// which has Key in place of nested keyable messages
google::protobuf::DescriptorPool & normalizedPool;
const google::protobuf::DescriptorPool & normalizedPool;
// Because a StorableMessage only holds references to its children objects,
// Storage is used to recursively retrieve them.
......
add_library(profanedb_storage db.cpp parser.cpp loader.cpp normalizer.cpp storage.cpp rocks.cpp)
target_link_libraries(profanedb_storage profanedb_protobuf ${ROCKSDB_LIBRARIES} ${Boost_LIBRARIES})
/*
* ProfaneDB - A Protocol Buffers database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "loader.h"
profanedb::storage::Loader::Loader(profanedb::storage::Parser & parser)
: parser(parser)
{
}
profanedb::storage::Loader::RootSourceTree::RootSourceTree(std::initializer_list<path> mappings)
: google::protobuf::compiler::DiskSourceTree()
{
for (const auto & path: mappings)
this->MapPath("", path.string());
google::protobuf::io::ZeroCopyInputStream * inputStream = this->Open("");
if (inputStream == nullptr)
throw std::runtime_error(this->GetLastErrorMessage());
}
void profanedb::storage::Loader::SetIncludePaths(std::initializer_list<path> paths)
{
this->includeDescDb = std::unique_ptr<SourceTreeDescriptorDatabase>(
new SourceTreeDescriptorDatabase(new RootSourceTree(paths)));
}
void profanedb::storage::Loader::LoadSchema(path path)
{
this->schemaDescDb = std::unique_ptr<SourceTreeDescriptorDatabase>(
new SourceTreeDescriptorDatabase(new RootSourceTree{path}));