Commit 05b4c9e4 authored by Giorgio Azzinnaro's avatar Giorgio Azzinnaro

added boot::Schema and vault::Storage

parent 57448bb4
......@@ -21,10 +21,4 @@ endif()
include_directories(${CMAKE_CURRENT_BINARY_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/src)
add_subdirectory(src/profanedb/protobuf) # Generated Protobuf (& gRPC if neeeded) code
add_subdirectory(src/profanedb/storage) # libprofanedb shared lib, and common interface
# Nothing should depend on code in here, so it can be skipped entirely
if(BUILD_PROFANEDB_SERVER)
add_subdirectory(src/profanedb/server)
endif()
add_subdirectory(src/profanedb)
add_subdirectory(protobuf) # Generated Protobuf (& gRPC if neeeded) code
add_subdirectory(boot)
add_subdirectory(vault)
add_library(profanedb db.cpp)
target_link_libraries(profanedb profanedb_protobuf)
# Nothing should depend on code in here, so it can be skipped entirely
if(BUILD_PROFANEDB_SERVER)
add_subdirectory(server)
endif()
add_library(profanedb_boot protobuf/schema.cpp)
target_link_libraries(profanedb_boot profanedb_protobuf)
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "schema.h"
profanedb::boot::protobuf::Schema::Schema()
{
}
profanedb::boot::protobuf::Schema::~Schema()
{
}
bool profanedb::boot::protobuf::Schema::IsKeyable(const google::protobuf::Message & message) const
{
const Descriptor * descriptor = message.GetDescriptor();
for (int i = 0; i < descriptor->field_count(); i++) {
if (descriptor->field(i)->options().GetExtension(profanedb::protobuf::options).key())
return true;
}
return false;
}
profanedb::protobuf::Key profanedb::boot::protobuf::Schema::GetKey(const google::protobuf::Message & message) const
{
const Descriptor * descriptor = message.GetDescriptor();
for (int i = 0; i < descriptor->field_count(); i++) {
if (descriptor->field(i)->options().GetExtension(profanedb::protobuf::options).key()) {
return FieldToKey(message, descriptor->field(i));
}
}
throw std::runtime_error(message.GetTypeName() + " is not keyable");
}
std::vector<const google::protobuf::Message *> profanedb::boot::protobuf::Schema::GetNestedMessages(
const google::protobuf::Message & message) const
{
std::vector<const google::protobuf::Message *> nested;
const Reflection * reflection = message.GetReflection();
const Descriptor * descriptor = message.GetDescriptor();
for (int i = 0; i < descriptor->field_count(); i++) {
if (descriptor->field(i)->type() == FieldDescriptor::TYPE_MESSAGE) {
nested.push_back(&reflection->GetMessage(message,descriptor->field(i)));
}
}
return nested;
}
profanedb::protobuf::Key profanedb::boot::protobuf::Schema::FieldToKey(
const Message & message,
const google::protobuf::FieldDescriptor * fd)
{
profanedb::protobuf::Key key;
*key.mutable_message_type() = message.GetTypeName();
*key.mutable_field() = fd->name();
const google::protobuf::Reflection * reflection = message.GetReflection();
if (fd->is_repeated()) {
for (int y = 0; y < reflection->FieldSize(message, fd); y++) {
switch (fd->cpp_type()) {
#define HANDLE_TYPE(CPPTYPE, METHOD) \
case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \
*key.mutable_value() += "$" + std::to_string(reflection->GetRepeated##METHOD(message, fd, y)); \
break;
HANDLE_TYPE(INT32 , Int32 );
HANDLE_TYPE(INT64 , Int64 );
HANDLE_TYPE(UINT32, UInt32);
HANDLE_TYPE(UINT64, UInt64);
HANDLE_TYPE(DOUBLE, Double);
HANDLE_TYPE(FLOAT , Float );
HANDLE_TYPE(BOOL , Bool );
#undef HANDLE_TYPE
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM:
*key.mutable_value() += "$" + std::to_string(reflection->GetRepeatedEnum(message, fd, y)->index());
break;
case google::protobuf::FieldDescriptor::CPPTYPE_STRING:
*key.mutable_value() += "$" + reflection->GetRepeatedString(message, fd, y);
break;
case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE:
*key.mutable_value() += "$" + reflection->GetRepeatedMessage(message, fd, y).SerializeAsString();
break;
}
}
} else {
switch (fd->cpp_type()) {
#define HANDLE_TYPE(CPPTYPE, METHOD) \
case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \
*key.mutable_value() = std::to_string(reflection->Get##METHOD(message, fd)); \
break;
HANDLE_TYPE(INT32 , Int32 );
HANDLE_TYPE(INT64 , Int64 );
HANDLE_TYPE(UINT32, UInt32);
HANDLE_TYPE(UINT64, UInt64);
HANDLE_TYPE(DOUBLE, Double);
HANDLE_TYPE(FLOAT , Float );
HANDLE_TYPE(BOOL , Bool );
#undef HANDLE_TYPE
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM:
*key.mutable_value() = std::to_string(reflection->GetEnum(message, fd)->index());
break;
case google::protobuf::FieldDescriptor::CPPTYPE_STRING:
*key.mutable_value() = reflection->GetString(message, fd);
break;
case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE:
*key.mutable_value() = reflection->GetMessage(message, fd).SerializeAsString();
break;
}
}
return key;
}
/*
* <one line to give the program's name and a brief idea of what it does.>
* Copyright (C) 2017 <copyright holder> <email>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef PROFANEDB_BOOT_PROTOBUF_SCHEMA_H
#define PROFANEDB_BOOT_PROTOBUF_SCHEMA_H
#include <google/protobuf/message.h>
#include <profanedb/protobuf/storage.pb.h>
#include <profanedb/protobuf/options.pb.h>
#include <profanedb/boot/schema.h>
namespace profanedb {
namespace boot {
namespace protobuf {
using google::protobuf::Message;
using google::protobuf::Reflection;
using google::protobuf::Descriptor;
using google::protobuf::FieldDescriptor;
// Redundant here, might be relevant for other kind of "Message" classes,
// to copy and paste from profanedb::boot::Schema interface
// Can be changed if collisions occur
using Message = google::protobuf::Message;
// Protobuf effectively stores all info regarding a Message in its Descriptor,
// so considering the Message was created before, we don't require any dependency here
class Schema : public profanedb::boot::Schema <Message>
{
public:
Schema();
~Schema();
// Check whether a Message has a key, therefore can be stored
virtual bool IsKeyable(const Message & message) const override;
// The Key is defined as an option of the given message
virtual profanedb::protobuf::Key GetKey(const Message & message) const override;
// Retrieve nested messages from a message
virtual std::vector<const Message *> GetNestedMessages(const Message & message) const override;
private:
static profanedb::protobuf::Key FieldToKey(const Message & message, const FieldDescriptor * fd);
};
}
}
}
#endif // PROFANEDB_BOOT_PROTOBUFSCHEMA_H
......@@ -17,30 +17,33 @@
*
*/
#ifndef PROFANEDB_STORAGE_STORAGE_H
#define PROFANEDB_STORAGE_STORAGE_H
#include <google/protobuf/message.h>
#ifndef PROFANEDB_BOOT_SCHEMA_H
#define PROFANEDB_BOOT_SCHEMA_H
#include <profanedb/protobuf/storage.pb.h>
namespace profanedb {
namespace storage {
namespace boot {
// Handles storage and retrieval of objects,
// subclass to implement a storage layer
class Storage {
// A Schema defines a Key for each message,
// a Key is a unique identifier for that given message.
// This data might be retrieved directly from the message,
// or the Schema might need to parse some definition to know what to do
template <typename Message>
class Schema {
public:
virtual ~Storage() = 0;
virtual ~Schema() = 0;
// Check whether a Message has a key, therefore can be stored
virtual bool IsKeyable(const Message & message) const = 0;
void Put(const profanedb::protobuf::MessageTreeNode & messageTree);
const profanedb::protobuf::MessageTreeNode Get(const profanedb::protobuf::Key & key) const;
// Extract a Key from a Message
virtual protobuf::Key GetKey(const Message & message) const = 0;
protected:
virtual void Store(const profanedb::protobuf::Key & key, const std::string & payload) = 0;
virtual const std::string Retrieve(const profanedb::protobuf::Key & key) const = 0;
// Retrieve nested messages from a message
virtual std::vector<const Message *> GetNestedMessages(const Message & message) const = 0;
};
}
}
#endif /* PROFANEDB_STORAGE_STORAGE_H */
#endif // PROFANEDB_BOOT_SCHEMA_H
......@@ -17,18 +17,14 @@
*
*/
#include "rootsourcetree.h"
#include "db.h"
profanedb::storage::RootSourceTree::RootSourceTree(
std::initializer_list<boost::filesystem::path> mappings)
: google::protobuf::compiler::DiskSourceTree()
template<typename Message>
profanedb::storage::Db<Message>::Db()
{
for (const auto & path: mappings)
this->MapPath("", path.string());
google::protobuf::io::ZeroCopyInputStream * inputStream = this->Open("");
if (inputStream == nullptr)
throw std::runtime_error(this->GetLastErrorMessage());
}
template<typename Message>
profanedb::storage::Db<Message>::~Db()
{
}
......@@ -17,42 +17,32 @@
*
*/
#ifndef PROFANEDB_STORAGE_DB_H
#define PROFANEDB_STORAGE_DB_H
#include <profanedb/boot/schema.h>
#include <profanedb/vault/storage.h>
#include <iostream>
#include <string>
#include <profanedb/protobuf/storage.pb.h>
#include <rocksdb/db.h>
#include <profanedb/protobuf/db.pb.h>
#include "config.h"
#include "parser.h"
#include "normalizer.h"
#include "rocks.h"
#ifndef PROFANEDB_DB_H
#define PROFANEDB_DB_H
namespace profanedb {
namespace storage {
// Db should be the main interface when embedding ProfaneDB
template<typename Message>
class Db
{
public:
Db(std::shared_ptr<Storage> storage);
Db();
~Db();
protobuf::GetResp Get(const protobuf::GetReq & request);
protobuf::PutResp Put(const protobuf::PutReq & request);
protobuf::DelResp Delete(const protobuf::DelReq & request);
virtual const Message & Get(const protobuf::Key & key) const = 0;
virtual bool Put(const Message & message) = 0;
virtual bool Delete(const protobuf::Key & key) = 0;
private:
Parser parser;
Normalizer normalizer;
std::shared_ptr<Storage> storage;
std::shared_ptr< boot::Schema<Message> > schema;
std::shared_ptr<vault::Storage> storage;
};
}
}
#endif // PROFANEDB_STORAGE_DB_H
#endif // PROFANEDB_DB_H
......@@ -28,8 +28,9 @@ option objc_class_prefix = "PDB";
// A Key uniquely identifies a message stored in the database
message Key {
string type = 1;
bytes value = 2;
string message_type = 1; // A identifier for the message
string field = 2; // The field within the message
bytes value = 3; // The value serialized in such way that it doesn't create collisions
}
// A StorableMessage has a unique Key and a serialized representation of the object
......
......@@ -20,28 +20,9 @@
#include "server.h"
profanedb::server::Server::Server()
: config(
profanedb::storage::Config::ProfaneDB(
boost::filesystem::path("/home/giorgio/Documents/ProfaneDB/test"),
boost::filesystem::path("/home/giorgio/Documents/ProfaneDB/src")
),
profanedb::storage::Config::RocksDB(
Server::RocksDBOptions(),
"/tmp/profanedb"
)
)
, service(config)
{
}
rocksdb::Options profanedb::server::Server::RocksDBOptions()
{
rocksdb::Options options;
options.create_if_missing = true;
return options;
}
profanedb::server::Server::~Server()
{
server->Shutdown();
......@@ -68,28 +49,28 @@ void profanedb::server::Server::HandleRpcs()
server->Wait();
}
profanedb::server::Server::DbServiceImpl::DbServiceImpl(const profanedb::storage::Config & config)
: db(config)
profanedb::server::Server::DbServiceImpl::DbServiceImpl(profanedb::Db<google::protobuf::Message> & profanedb)
: profanedb(profanedb)
{
}
grpc::Status profanedb::server::Server::DbServiceImpl::Get(grpc::ServerContext * context, const profanedb::protobuf::GetReq * request, profanedb::protobuf::GetResp * response)
{
db.Get(*request);
response->mutable_message()->PackFrom(this->profanedb.Get(request->key()));
return grpc::Status::OK;
}
grpc::Status profanedb::server::Server::DbServiceImpl::Put(grpc::ServerContext * context, const profanedb::protobuf::PutReq * request, profanedb::protobuf::PutResp * response)
{
db.Put(*request);
// TODO Unpack
this->profanedb.Put(request->serializable());
return grpc::Status::OK;
}
grpc::Status profanedb::server::Server::DbServiceImpl::Delete(grpc::ServerContext * context, const profanedb::protobuf::DelReq * request, profanedb::protobuf::DelResp * response)
{
db.Delete(*request);
return grpc::Status::OK;
}
......@@ -23,10 +23,7 @@
#include <grpc++/grpc++.h>
#include <grpc/support/log.h>
#include <rocksdb/db.h>
#include <profanedb/storage/db.h>
#include <profanedb/storage/config.h>
#include <profanedb/db.h>
#include <profanedb/protobuf/db.pb.h>
#include <profanedb/protobuf/db.grpc.pb.h>
......@@ -44,16 +41,13 @@ public:
void Run();
private:
static rocksdb::Options RocksDBOptions();
profanedb::storage::Config config;
void HandleRpcs();
std::unique_ptr<grpc::Server> server;
class DbServiceImpl : public profanedb::protobuf::Db::Service {
public:
DbServiceImpl(const profanedb::storage::Config & config);
DbServiceImpl(profanedb::Db<google::protobuf::Message> & profanedb);
grpc::Status Get(grpc::ServerContext * context, const profanedb::protobuf::GetReq * request, profanedb::protobuf::GetResp* response) override;
......@@ -62,7 +56,7 @@ private:
grpc::Status Delete(grpc::ServerContext * context, const profanedb::protobuf::DelReq * request, profanedb::protobuf::DelResp * response) override;
private:
profanedb::storage::Db db;
profanedb::Db<google::protobuf::Message> & profanedb;
};
DbServiceImpl service;
};
......
add_library(profanedb_storage db.cpp parser.cpp normalizer.cpp rootsourcetree.cpp storage.cpp rocks.cpp)
add_library(profanedb_storage db.cpp parser.cpp loader.cpp normalizer.cpp storage.cpp rocks.cpp)
target_link_libraries(profanedb_storage profanedb_protobuf ${ROCKSDB_LIBRARIES} ${Boost_LIBRARIES})
......@@ -17,40 +17,54 @@
*
*/
#include "db.h"
#include "loader.h"
profanedb::storage::Db::Db(Config config)
: config(config)
, parser(config.GetProfaneConfig())
, normalizer(parser)
profanedb::storage::Loader::Loader(profanedb::storage::Parser & parser)
: parser(parser)
{
}
profanedb::storage::Db::~Db()
profanedb::storage::Loader::RootSourceTree::RootSourceTree(std::initializer_list<path> mappings)
: google::protobuf::compiler::DiskSourceTree()
{
delete this->db;
for (const auto & path: mappings)
this->MapPath("", path.string());
google::protobuf::io::ZeroCopyInputStream * inputStream = this->Open("");
if (inputStream == nullptr)
throw std::runtime_error(this->GetLastErrorMessage());
}
profanedb::protobuf::GetResp profanedb::storage::Db::Get(const protobuf::GetReq & request)
void profanedb::storage::Loader::SetIncludePaths(std::initializer_list<path> paths)
{
std::string * serialized = new std::string();
db->Get(rocksdb::ReadOptions(), request.key().SerializeAsString(), serialized);
return *protobuf::GetResp().New();
this->includeDescDb = std::unique_ptr<SourceTreeDescriptorDatabase>(
new SourceTreeDescriptorDatabase(new RootSourceTree(paths)));
}
profanedb::protobuf::PutResp profanedb::storage::Db::Put(const protobuf::PutReq & request)
void profanedb::storage::Loader::LoadSchema(path path)
{
auto map = normalizer.NormalizeMessage(request.serializable());
for (auto const & obj: map) {
std::cout << obj.first << std::endl;
db->Put(rocksdb::WriteOptions(), obj.first, obj.second.SerializeAsString());
}
this->schemaDescDb = std::unique_ptr<SourceTreeDescriptorDatabase>(
new SourceTreeDescriptorDatabase(new RootSourceTree{path}));
return *protobuf::PutResp().New();
}
google::protobuf::MergedDescriptorDatabase schemaDb(this->includeDescDb.get(), this->schemaDescDb.get());
google::protobuf::DescriptorPool schemaPool(&schemaDb);
profanedb::protobuf::DelResp profanedb::storage::Db::Delete(const protobuf::DelReq & request)
{
// Load ProfaneDB options, which schema definitions refer to
schemaPool.FindFileByName("profanedb/protobuf/options.proto");
for (auto const & file: recursive_directory_iterator(path, symlink_option::recurse)) {
if (file.path().extension() == ".proto") {
const FileDescriptor * fileDesc = schemaPool.FindFileByName(
file.path().lexically_relative(path).string());
FileDescriptorProto * fileProto = new FileDescriptorProto;
fileDesc->CopyTo(fileProto);
for (int i = 0; i < fileDesc->message_type_count(); i++) {
const Descriptor * message = fileDesc->message_type(i);
this->parser->ParseMessageDescriptor();
}
}
}
}
/*
* ProfaneDB - A Protocol Buffers database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef PROFANEDB_STORAGE_LOADER_H
#define PROFANEDB_STORAGE_LOADER_H
#include <memory>
#include <boost/filesystem.hpp>
#include "parser.h"
namespace profanedb {
namespace storage {
using boost::filesystem::path;
using boost::filesystem::recursive_directory_iterator;
using boost::filesystem::symlink_option;
using google::protobuf::DescriptorDatabase;
using google::protobuf::compiler::SourceTreeDescriptorDatabase;
using google::protobuf::FileDescriptor;
using google::protobuf::FileDescriptorProto;
using google::protobuf::Descriptor;
// A Loader
class Loader
{
public:
Loader(std::shared_ptr<Parser> parser);
void SetIncludePaths(std::initializer_list<path> paths);
void LoadSchema(path path);
std::unique_ptr<DescriptorPool> GetSchemaPool();
private:
std::shared_ptr<Parser> parser;
// DescriptorDatabases must outlive their DescriptorPool,
// so other classes can access schemaPool,
// but make sure Loader doesn't go out of scope
std::unique_ptr<DescriptorDatabase> includeDescDb;
std::unique_ptr<DescriptorDatabase> schemaDescDb;
std::unique_ptr<DescriptorPool> schemaPool;
// A DiskSourceTree where paths are automatically mapped to root ("")
class RootSourceTree : public google::protobuf::compiler::DiskSourceTree {
public:
RootSourceTree(std::initializer_list<path> mappings);
};
};
}
}
#endif // PROFANEDB_STORAGE_LOADER_H
......@@ -34,7 +34,7 @@ std::map<std::string, const google::protobuf::Message &> profanedb::storage::Nor
// Any messages have a type url beginning with `type.googleapis.com/`, this is stripped
std::string type = serializable.type_url();
const google::protobuf::Descriptor * definition = schemaPool->FindMessageTypeByName(type.substr(type.rfind('/')+1, std::string::npos));
// Having the definition our message factory can simply generate a container,
auto container = std::shared_ptr<google::protobuf::Message>(messageFactory.GetPrototype(definition)->New());
......
......@@ -33,9 +33,6 @@
#include <profanedb/protobuf/db.pb.h>
#include <profanedb/protobuf/options.pb.h>
#include "config.h"
#include "rootsourcetree.h"
using namespace google::protobuf;
namespace profanedb {
......@@ -67,6 +64,16 @@ private:
google::protobuf::DescriptorProto & proto
);
// A simple ErrorCollector for debug, write to stderr
class ErrorCollector : public compiler::MultiFileErrorCollector {
public:
ErrorCollector();
void AddError(const string & filename, int line, int column, const string & message) override;
void AddWarning(const string & filename, int line, int column, const string & message) override;
};
ErrorCollector errCollector;
class NormalizedDescriptor {
public:
NormalizedDescriptor(
......@@ -85,16 +92,6 @@ private:
};
std::map<std::string, NormalizedDescriptor> normalizedDescriptors;
// A simple ErrorCollector for debug, write to stderr
class ErrorCollector : public compiler::MultiFileErrorCollector {
public:
ErrorCollector();
void AddError(const string & filename, int line, int column, const string & message) override;
void AddWarning(const string & filename, int line, int column, const string & message) override;
};
ErrorCollector errCollector;
};
}
}
......
/*
* ProfaneDB - A Protocol Buffers database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*