Commit 2a4e546b authored by Giorgio Azzinnaro's avatar Giorgio Azzinnaro

adde rootsourcetree

parent a959f0dc
......@@ -5,7 +5,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g")
set(PROTOBUF_IMPORT_DIRS "${CMAKE_SOURCE_DIR}/src")
option(BUILD_LIBPROFANEDB "Build libprofanedb.so, to embed ProfaneDB" ON)
option(BUILD_SHARED_LIBS "Build libprofanedb.so, to embed ProfaneDB" ON)
option(BUILD_PROFANEDB_SERVER "Build ProfaneDB gRPC server" OFF)
find_package(Threads REQUIRED)
......
......@@ -2,18 +2,18 @@
file(GLOB Proto "${CMAKE_CURRENT_SOURCE_DIR}/*.proto")
PROTOBUF_GENERATE_CPP(PROTO_CC PROTO_H ${Proto})
add_library(profanedb_protobuf ${PROTO_CC} ${PROTO_H} )
add_library(profanedb_protobuf STATIC ${PROTO_CC} ${PROTO_H} )
target_link_libraries(profanedb_protobuf ${CMAKE_THREAD_LIBS_INIT} ${PROTOBUF_LIBRARIES})
# If building the shared library, libprofanedb_protobuf.a needs to be PIC to be included
if(BUILD_LIBPROFANEDB)
if(BUILD_SHARED_LIBS)
set_target_properties(profanedb_protobuf PROPERTIES POSITION_INDEPENDENT_CODE ON)
endif()
# libprofanedb_grpc.a has the service definitions, it's not required by the shared lib
if(BUILD_PROFANEDB_SERVER)
PROTOBUF_GENERATE_GRPC_CPP(GRPC_CC GRPC_H ${Proto})
add_library(profanedb_grpc ${GRPC_CC} ${GRPC_H})
add_library(profanedb_grpc STATIC ${GRPC_CC} ${GRPC_H})
target_link_libraries(profanedb_grpc profanedb_protobuf ${GRPC_LIBRARIES})
endif()
add_executable(profanedb_server main.cpp server.cpp)
if(BUILD_LIBPROFANEDB)
target_link_libraries(profanedb_server profanedb profanedb_grpc)
elseif()
# profanedb_grpc contains profanedb_protobuf
# profanedb_storage is linked statically for this executable
target_link_libraries(profanedb_server profanedb_grpc profanedb_storage)
endif()
target_link_libraries(profanedb_server profanedb_grpc profanedb_storage)
add_library(profanedb_storage db.cpp parser.cpp config.cpp)
add_library(profanedb_storage db.cpp parser.cpp config.cpp normalizer.cpp rootsourcetree.cpp)
target_link_libraries(profanedb_storage profanedb_protobuf ${ROCKSDB_LIBRARIES} ${Boost_LIBRARIES})
# libprofanedb_storage is used statically by the server,
# built as shared lib here to allow other software to embed it
if(BUILD_LIBPROFANEDB)
add_library(profanedb SHARED db.cpp parser.cpp config.cpp)
target_link_libraries(profanedb profanedb_protobuf ${ROCKSDB_LIBRARIES} ${Boost_LIBRARIES})
endif()
......@@ -44,25 +44,9 @@ profanedb::storage::Config::ProfaneDB::ProfaneDB(
boost::filesystem::path include)
: schemaDefinition(schema)
, sourceTree(new google::protobuf::compiler::DiskSourceTree)
, profaneDbOptions(options)
, includePath(include)
{
sourceTree->MapPath("", include.string());
sourceTree->MapPath("", options.string());
sourceTree->MapPath("", schema.string());
google::protobuf::io::ZeroCopyInputStream * inputStream = sourceTree->Open("");
if (inputStream == NULL)
throw std::runtime_error(sourceTree->GetLastErrorMessage());
}
std::shared_ptr<google::protobuf::compiler::SourceTree> profanedb::storage::Config::ProfaneDB::GetSourceTree() const
{
return std::dynamic_pointer_cast<google::protobuf::compiler::SourceTree>(this->sourceTree);
}
const boost::filesystem::path & profanedb::storage::Config::ProfaneDB::GetSchemaDefinitionPath() const
{
return this->schemaDefinition;
}
profanedb::storage::Config::RocksDB::RocksDB(rocksdb::Options options, std::string name)
......
......@@ -43,14 +43,9 @@ public:
boost::filesystem::path options,
boost::filesystem::path include = boost::filesystem::path("/usr/include"));
const boost::filesystem::path & GetSchemaDefinitionPath() const;
std::shared_ptr<google::protobuf::compiler::SourceTree> GetSourceTree() const;
private:
boost::filesystem::path schemaDefinition;
// Setting the path updates the source tree
std::shared_ptr<google::protobuf::compiler::DiskSourceTree> sourceTree;
const boost::filesystem::path schemaDefinition;
const boost::filesystem::path profaneDbOptions;
const boost::filesystem::path includePath;
};
class RocksDB {
......
......@@ -22,13 +22,13 @@
profanedb::storage::Db::Db(profanedb::storage::Config config)
: config(config)
, parser(config.GetProfaneConfig())
, normalizer(parser)
{
// rocksdb::DB::Open(options, name, &db);
}
profanedb::storage::Db::~Db()
{
delete db;
}
profanedb::protobuf::GetResp profanedb::storage::Db::Get(const profanedb::protobuf::GetReq & request)
......
......@@ -27,7 +27,9 @@
#include <profanedb/protobuf/db.pb.h>
#include "config.h"
#include "parser.h"
#include "normalizer.h"
namespace profanedb {
namespace storage {
......@@ -43,10 +45,11 @@ public:
protobuf::PutResp Put(const protobuf::PutReq & request);
protobuf::DelResp Delete(const protobuf::DelReq & request);
private:
rocksdb::DB * db;
Config config;
Parser parser;
Normalizer normalizer;
std::unique_ptr<rocksdb::DB> db;
};
}
}
......
/*
* ProfaneDB - A Protocol Buffer database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "normalizer.h"
profanedb::storage::Normalizer::Normalizer(Parser & parser)
: schemaPool(parser.schemaPool)
, normalizedMergedDescriptorDb(
new google::protobuf::MergedDescriptorDatabase(parser.rootDescriptorDb.get(), parser.normalizedDescriptorDb.get()))
, normalizedPool(new google::protobuf::DescriptorPool(normalizedMergedDescriptorDb.get()))
{
std::cout << normalizedPool->FindFileByName("test.proto")->DebugString() << std::endl;
}
/*
* ProfaneDB - A Protocol Buffer database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef PROFANEDB_STORAGE_NORMALIZER_H
#define PROFANEDB_STORAGE_NORMALIZER_H
#include <map>
#include <google/protobuf/descriptor_database.h>
#include "parser.h"
namespace profanedb {
namespace storage {
class Normalizer
{
public:
Normalizer(Parser & parser);
private:
std::shared_ptr<google::protobuf::DescriptorPool> schemaPool;
std::unique_ptr<google::protobuf::MergedDescriptorDatabase> normalizedMergedDescriptorDb;
std::shared_ptr<google::protobuf::DescriptorPool> normalizedPool;
};
}
}
#endif // PROFANEDB_STORAGE_NORMALIZER_H
......@@ -20,86 +20,71 @@
#include "parser.h"
profanedb::storage::Parser::Parser(const Config::ProfaneDB & profaneConfig)
: descriptorDb(new compiler::SourceTreeDescriptorDatabase(profaneConfig.GetSourceTree().get()))
, pool(new DescriptorPool(descriptorDb.get()))
: rootDescriptorDb(
new google::protobuf::compiler::SourceTreeDescriptorDatabase(
new RootSourceTree {profaneConfig.includePath, profaneConfig.profaneDbOptions}
)
)
, schemaDescriptorDb(
new google::protobuf::compiler::SourceTreeDescriptorDatabase(
new RootSourceTree {profaneConfig.schemaDefinition}
)
)
, mergedSchemaDescriptorDb(
new google::protobuf::MergedDescriptorDatabase(rootDescriptorDb.get(), schemaDescriptorDb.get()))
, schemaPool(new DescriptorPool(mergedSchemaDescriptorDb.get()))
, normalizedDescriptorDb(new SimpleDescriptorDatabase)
{
descriptorDb->RecordErrorsTo(&errCollector);
schemaDescriptorDb->RecordErrorsTo(&errCollector);
// Load ProfaneDB options to be used during file import
pool->FindFileByName("profanedb/protobuf/options.proto");
schemaPool->FindFileByName("profanedb/protobuf/options.proto");
// Import all `.proto` files in kDbSchema into the pool,
// TODO Move the whole loading to functions
// Import all `.proto` files from the schema definition into the pool,
// so that FindMessageTypeByName can then be used
for (auto const & file:
boost::filesystem::recursive_directory_iterator(
profaneConfig.GetSchemaDefinitionPath(),
profaneConfig.schemaDefinition,
boost::filesystem::symlink_option::recurse)) {
// Files are loaded by extension, so care should be taken to name them appropriately
if (file.path().extension() == ".proto") {
// For the pool every file is relative to the mapping provided before (kDbSchema)
const FileDescriptor * fileD = pool->FindFileByName(
file.path().lexically_relative(profaneConfig.GetSchemaDefinitionPath()).string());
// For the pool every file is relative to the mapping provided before in config
const FileDescriptor * fileD = schemaPool->FindFileByName(
file.path().lexically_relative(profaneConfig.schemaDefinition).string());
FileDescriptorProto * fileProto = new FileDescriptorProto;
fileD->CopyTo(fileProto);
for (int i = 0; i < fileD->message_type_count(); i++) {
const Descriptor * message = fileD->message_type(i);
DescriptorProto * descriptorProto;
if (ParseMessageDescriptor(*message)) {
descriptorProto = fileDescProto.add_message_type();
message->CopyTo(descriptorProto);
for (int k = 0; k < message->field_count(); k++) {
if (message->field(k)->options().GetExtension(profanedb::protobuf::options).key()) {
fileProto->mutable_message_type(i)->mutable_field(k)->set_type(
FieldDescriptorProto_Type_TYPE_STRING
);
break;
}
}
}
normalizedDescriptorDb->AddAndOwn(fileProto);
}
}
// Now build a file with all generated messages in an empty pool
DescriptorPool newPool{};
*fileDescProto.mutable_name() = "profanedb_generated_schema";
std::cout << fileDescProto.DebugString() << std::endl;
newPool.BuildFile(fileDescProto);
}
profanedb::storage::Parser::~Parser()
{
}
bool profanedb::storage::Parser::ParseMessageDescriptor(const google::protobuf::Descriptor & descriptor)
{
// A DescriptorProto is needed to generate the message the way it will be serialized,
// with keys replacing
google::protobuf::DescriptorProto * descriptorProto;
bool hasKey = false;
for (int k = 0; k < descriptor.field_count(); k++) {
const FieldDescriptor * field = descriptor.field(k);
// Recursively call this function to add all nested messages
const google::protobuf::Descriptor * nested = field->message_type();
if (nested != NULL) {
if (ParseMessageDescriptor(*nested)) {
// If the nested message has a primary key, set the field to string to hold a reference
descriptorProto->mutable_field(k)->set_type(FieldDescriptorProto_Type_TYPE_STRING);
}
}
profanedb::protobuf::FieldOptions options = field->options().GetExtension(profanedb::protobuf::options);
if (options.key()) {
hasKey = true;
}
}
return hasKey;
}
map< std::string, const google::protobuf::Message & > profanedb::storage::Parser::NormalizeMessage(const google::protobuf::Any & serializable)
{
// The Descriptor is manually extracted from the pool,
// removing the prepending `type.googleapis.com/` in the Any message
string type = serializable.type_url();
const Descriptor * definition = pool->FindMessageTypeByName(type.substr(type.rfind('/')+1, string::npos));
const Descriptor * definition = schemaPool->FindMessageTypeByName(type.substr(type.rfind('/')+1, string::npos));
Message * container = messageFactory.GetPrototype(definition)->New();
serializable.UnpackTo(container);
......@@ -145,11 +130,11 @@ map< std::string, const google::protobuf::Message & > profanedb::storage::Parser
return *dependencies;
}
string profanedb::storage::Parser::FieldToKey(const google::protobuf::Message & container, const google::protobuf::FieldDescriptor & fd)
std::string profanedb::storage::Parser::FieldToKey(const google::protobuf::Message & container, const google::protobuf::FieldDescriptor & fd)
{
const Reflection * reflection = container.GetReflection();
string key_value;
std::string key_value;
switch (fd.cpp_type()) {
case FieldDescriptor::CPPTYPE_INT32:
......
......@@ -34,16 +34,19 @@
#include <profanedb/protobuf/options.pb.h>
#include "config.h"
#include "rootsourcetree.h"
using namespace google::protobuf;
namespace profanedb {
namespace storage {
// Given a Any message, Parser looks for the corresponding definition in .proto files,
// and generates a map with keys of nested messages
class Parser
{
friend class Normalizer;
public:
Parser(const Config::ProfaneDB & profaneConfig);
~Parser();
......@@ -53,6 +56,19 @@ public:
map<std::string, const Message &> NormalizeMessage(const Message & message);
private:
std::unique_ptr<compiler::SourceTreeDescriptorDatabase> rootDescriptorDb;
std::unique_ptr<compiler::SourceTreeDescriptorDatabase> schemaDescriptorDb;
std::unique_ptr<MergedDescriptorDatabase> mergedSchemaDescriptorDb;
std::shared_ptr<DescriptorPool> schemaPool;
std::unique_ptr<SimpleDescriptorDatabase> normalizedDescriptorDb;
DynamicMessageFactory messageFactory;
// Given a Field
string FieldToKey(const Message & container, const FieldDescriptor & fd);
// A simple ErrorCollector for debug, write to stderr
class ErrorCollector : public compiler::MultiFileErrorCollector {
public:
......@@ -62,18 +78,6 @@ private:
};
ErrorCollector errCollector;
std::shared_ptr<compiler::SourceTreeDescriptorDatabase> descriptorDb;
std::shared_ptr<DescriptorPool> pool;
DynamicMessageFactory messageFactory;
FileDescriptorProto fileDescProto;
// Given a Field
string FieldToKey(const Message & container, const FieldDescriptor & fd);
// Given a descriptor, find information about the key and nested objects,
// return true if the message has a key
bool ParseMessageDescriptor(const Descriptor & descriptor);
};
}
}
......
/*
* ProfaneDB - A Protocol Buffer database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "rootsourcetree.h"
profanedb::storage::RootSourceTree::RootSourceTree(
std::initializer_list<boost::filesystem::path> mappings)
: google::protobuf::compiler::DiskSourceTree()
{
for (const auto & path: mappings)
this->MapPath("", path.string());
google::protobuf::io::ZeroCopyInputStream * inputStream = this->Open("");
if (inputStream == nullptr)
throw std::runtime_error(this->GetLastErrorMessage());
}
/*
* ProfaneDB - A Protocol Buffer database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef PROFANEDB_STORAGE_ROOTSOURCETREE_H
#define PROFANEDB_STORAGE_ROOTSOURCETREE_H
#include <exception>
#include <boost/filesystem.hpp>
#include <google/protobuf/compiler/importer.h>
namespace profanedb {
namespace storage {
class RootSourceTree : public google::protobuf::compiler::DiskSourceTree
{
public:
RootSourceTree(std::initializer_list<boost::filesystem::path> mappings);
// private:
// std::unique_ptr<google::protobuf::io::ZeroCopyInputStream> inputStream;
};
}
}
#endif // PROFANEDB_STORAGE_ROOTSOURCETREE_H
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment