Commit a959f0dc authored by Giorgio Azzinnaro's avatar Giorgio Azzinnaro

moving stuff around

parent 39f7c8f7
......@@ -10,9 +10,9 @@ message Test {
int32 field_one_int = 1 [ (profanedb.protobuf.options).key = true ];
string field_two_str = 2;
bool field_three_bool = 3;
bytes field_four_bytes = 4;
Nested field_five_nested = 5;
bytes field_four_bytes = 4;
ExternalNested field_six_externalnested = 6;
}
......
add_executable(profanedb_server main.cpp server.cpp)
# profanedb_grpc contains profanedb_protobuf
# profanedb_storage is linked statically for this executable
target_link_libraries(profanedb_server profanedb_grpc profanedb_storage)
if(BUILD_LIBPROFANEDB)
target_link_libraries(profanedb_server profanedb profanedb_grpc)
elseif()
# profanedb_grpc contains profanedb_protobuf
# profanedb_storage is linked statically for this executable
target_link_libraries(profanedb_server profanedb_grpc profanedb_storage)
endif()
......@@ -20,6 +20,17 @@
#include "server.h"
profanedb::server::Server::Server()
: config(
profanedb::storage::Config::ProfaneDB(
boost::filesystem::path("/home/giorgio/Documents/ProfaneDB/test"),
boost::filesystem::path("/home/giorgio/Documents/ProfaneDB/src")
),
profanedb::storage::Config::RocksDB(
rocksdb::Options(),
"/tmp/profanedb"
)
)
, service(config)
{
}
......@@ -49,6 +60,11 @@ void profanedb::server::Server::HandleRpcs()
server->Wait();
}
profanedb::server::Server::DbServiceImpl::DbServiceImpl(const profanedb::storage::Config & config)
: db(config)
{
}
grpc::Status profanedb::server::Server::DbServiceImpl::Get(grpc::ServerContext * context, const profanedb::protobuf::GetReq * request, profanedb::protobuf::GetResp * response)
{
db.Get(*request);
......
......@@ -17,13 +17,14 @@
*
*/
#ifndef SERVER_H
#define SERVER_H
#ifndef PROFANEDB_STORAGE_SERVER_H
#define PROFANEDB_STORAGE_SERVER_H
#include <grpc++/grpc++.h>
#include <grpc/support/log.h>
#include <profanedb/storage/db.h>
#include <profanedb/storage/config.h>
#include <profanedb/protobuf/db.pb.h>
#include <profanedb/protobuf/db.grpc.pb.h>
......@@ -41,12 +42,16 @@ public:
void Run();
private:
profanedb::storage::Config config;
void HandleRpcs();
std::unique_ptr<grpc::Server> server;
class DbServiceImpl : public profanedb::protobuf::Db::Service {
public:
DbServiceImpl(const profanedb::storage::Config & config);
grpc::Status Get(grpc::ServerContext * context, const profanedb::protobuf::GetReq * request, profanedb::protobuf::GetResp* response) override;
grpc::Status Put(grpc::ServerContext * context, const profanedb::protobuf::PutReq * request, profanedb::protobuf::PutResp * response) override;
......@@ -62,4 +67,4 @@ private:
}
}
#endif // SERVER_H
#endif // PROFANEDB_STORAGE_SERVER_H
add_library(profanedb_storage db.cpp parser.cpp)
add_library(profanedb_storage db.cpp parser.cpp config.cpp)
target_link_libraries(profanedb_storage profanedb_protobuf ${ROCKSDB_LIBRARIES} ${Boost_LIBRARIES})
# libprofanedb_storage is used statically by the server,
# built as shared lib here to allow other software to embed it
if(BUILD_LIBPROFANEDB)
add_library(profanedb SHARED db.cpp parser.cpp)
add_library(profanedb SHARED db.cpp parser.cpp config.cpp)
target_link_libraries(profanedb profanedb_protobuf ${ROCKSDB_LIBRARIES} ${Boost_LIBRARIES})
endif()
/*
* ProfaneDB - A Protocol Buffer database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "config.h"
profanedb::storage::Config::Config(
profanedb::storage::Config::ProfaneDB profane,
profanedb::storage::Config::RocksDB rocks)
: profaneConfig(profane)
, rocksConfig(rocks)
{
}
profanedb::storage::Config::ProfaneDB & profanedb::storage::Config::GetProfaneConfig()
{
return this->profaneConfig;
}
profanedb::storage::Config::RocksDB & profanedb::storage::Config::GetRocksConfig()
{
return this->rocksConfig;
}
profanedb::storage::Config::ProfaneDB::ProfaneDB(
boost::filesystem::path schema,
boost::filesystem::path options,
boost::filesystem::path include)
: schemaDefinition(schema)
, sourceTree(new google::protobuf::compiler::DiskSourceTree)
{
sourceTree->MapPath("", include.string());
sourceTree->MapPath("", options.string());
sourceTree->MapPath("", schema.string());
google::protobuf::io::ZeroCopyInputStream * inputStream = sourceTree->Open("");
if (inputStream == NULL)
throw std::runtime_error(sourceTree->GetLastErrorMessage());
}
std::shared_ptr<google::protobuf::compiler::SourceTree> profanedb::storage::Config::ProfaneDB::GetSourceTree() const
{
return std::dynamic_pointer_cast<google::protobuf::compiler::SourceTree>(this->sourceTree);
}
const boost::filesystem::path & profanedb::storage::Config::ProfaneDB::GetSchemaDefinitionPath() const
{
return this->schemaDefinition;
}
profanedb::storage::Config::RocksDB::RocksDB(rocksdb::Options options, std::string name)
: options(options)
, name(name)
{
}
/*
* ProfaneDB - A Protocol Buffer database.
* Copyright (C) 2017 "Giorgio Azzinnaro" <giorgio.azzinnaro@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef PROFANEDB_STORAGE_CONFIG_H
#define PROFANEDB_STORAGE_CONFIG_H
#include <iostream>
#include <string>
#include <exception>
#include <google/protobuf/compiler/importer.h>
#include <rocksdb/options.h>
#include <boost/filesystem.hpp>
namespace profanedb {
namespace storage {
class Config
{
public:
class ProfaneDB {
public:
ProfaneDB(boost::filesystem::path schema,
boost::filesystem::path options,
boost::filesystem::path include = boost::filesystem::path("/usr/include"));
const boost::filesystem::path & GetSchemaDefinitionPath() const;
std::shared_ptr<google::protobuf::compiler::SourceTree> GetSourceTree() const;
private:
boost::filesystem::path schemaDefinition;
// Setting the path updates the source tree
std::shared_ptr<google::protobuf::compiler::DiskSourceTree> sourceTree;
};
class RocksDB {
public:
RocksDB(rocksdb::Options options, std::string name);
private:
rocksdb::Options options;
std::string name;
};
Config(ProfaneDB profane, RocksDB rocks);
ProfaneDB & GetProfaneConfig();
RocksDB & GetRocksConfig();
private:
ProfaneDB profaneConfig;
RocksDB rocksConfig;
};
}
}
#endif // PROFANEDB_STORAGE_CONFIG_H
......@@ -19,15 +19,16 @@
#include "db.h"
profanedb::storage::Db::Db()
profanedb::storage::Db::Db(profanedb::storage::Config config)
: config(config)
, parser(config.GetProfaneConfig())
{
options.create_if_missing = true;
rocksdb::DB::Open(options, "/tmp/profanedb", &db);
// rocksdb::DB::Open(options, name, &db);
}
profanedb::storage::Db::~Db()
{
delete db;
}
profanedb::protobuf::GetResp profanedb::storage::Db::Get(const profanedb::protobuf::GetReq & request)
......
......@@ -17,17 +17,18 @@
*
*/
#ifndef DB_H
#define DB_H
#ifndef PROFANEDB_STORAGE_DB_H
#define PROFANEDB_STORAGE_DB_H
#include <iostream>
#include "parser.h"
#include <string>
#include <rocksdb/db.h>
#include <profanedb/protobuf/db.pb.h>
#include "parser.h"
namespace profanedb {
namespace storage {
......@@ -35,7 +36,7 @@ namespace storage {
class Db
{
public:
Db();
Db(Config config);
~Db();
protobuf::GetResp Get(const protobuf::GetReq & request);
......@@ -43,11 +44,11 @@ public:
protobuf::DelResp Delete(const protobuf::DelReq & request);
private:
rocksdb::DB * db;
rocksdb::Options options;
Config config;
Parser parser;
};
}
}
#endif // DB_H
#endif // PROFANEDB_STORAGE_DB_H
......@@ -19,47 +19,45 @@
#include "parser.h"
profanedb::storage::Parser::Parser()
profanedb::storage::Parser::Parser(const Config::ProfaneDB & profaneConfig)
: descriptorDb(new compiler::SourceTreeDescriptorDatabase(profaneConfig.GetSourceTree().get()))
, pool(new DescriptorPool(descriptorDb.get()))
{
// HACK Should be in config
const auto kOptions = boost::filesystem::path("/home/giorgio/Documents/ProfaneDB/src");
const auto kDbSchema = boost::filesystem::path("/home/giorgio/Documents/ProfaneDB/test");
// Everything is mapped to "" to preserve directory structure
sourceTree.MapPath("", "/usr/include"); // google/protobuf/... should be here
sourceTree.MapPath("", kOptions.string());
sourceTree.MapPath("", kDbSchema.string());
inputStream = sourceTree.Open("");
if (inputStream == NULL) {
std::cerr << "Couldn't open .proto source tree: " << sourceTree.GetLastErrorMessage() << std::endl;
}
descriptorDb = new compiler::SourceTreeDescriptorDatabase(&sourceTree);
descriptorDb->RecordErrorsTo(errCollector);
pool = new DescriptorPool(descriptorDb);
descriptorDb->RecordErrorsTo(&errCollector);
// Load ProfaneDB options to be used during file import
pool->FindFileByName("profanedb/protobuf/options.proto");
// Import all `.proto` files in kDbSchema into the pool,
// so that FindMessageTypeByName can then be used
boost::filesystem::path path(kDbSchema);
for (auto const & file: boost::filesystem::recursive_directory_iterator(path, boost::filesystem::symlink_option::recurse)) {
for (auto const & file:
boost::filesystem::recursive_directory_iterator(
profaneConfig.GetSchemaDefinitionPath(),
boost::filesystem::symlink_option::recurse)) {
if (file.path().extension() == ".proto") {
// For the pool every file is relative to the mapping provided before (kDbSchema)
const FileDescriptor * fileD = pool->FindFileByName(file.path().lexically_relative(kDbSchema).native());
const FileDescriptor * fileD = pool->FindFileByName(
file.path().lexically_relative(profaneConfig.GetSchemaDefinitionPath()).string());
for (int i = 0; i < fileD->message_type_count(); i++) {
const Descriptor * message = fileD->message_type(i);
DescriptorProto * descriptorProto;
ParseMessageDescriptor(*message);
if (ParseMessageDescriptor(*message)) {
descriptorProto = fileDescProto.add_message_type();
message->CopyTo(descriptorProto);
}
}
// TODO Here should load all the messages, find the keys and nested messages, and generate Descriptors with references to save in DB
}
}
// Now build a file with all generated messages in an empty pool
DescriptorPool newPool{};
*fileDescProto.mutable_name() = "profanedb_generated_schema";
std::cout << fileDescProto.DebugString() << std::endl;
newPool.BuildFile(fileDescProto);
}
profanedb::storage::Parser::~Parser()
......@@ -70,8 +68,7 @@ bool profanedb::storage::Parser::ParseMessageDescriptor(const google::protobuf::
{
// A DescriptorProto is needed to generate the message the way it will be serialized,
// with keys replacing
auto * descriptorProto = new google::protobuf::DescriptorProto();
descriptor.CopyTo(descriptorProto);
google::protobuf::DescriptorProto * descriptorProto;
bool hasKey = false;
......@@ -83,7 +80,6 @@ bool profanedb::storage::Parser::ParseMessageDescriptor(const google::protobuf::
if (nested != NULL) {
if (ParseMessageDescriptor(*nested)) {
// If the nested message has a primary key, set the field to string to hold a reference
std::cout << descriptorProto->mutable_field(k)->DebugString() << std::endl;
descriptorProto->mutable_field(k)->set_type(FieldDescriptorProto_Type_TYPE_STRING);
}
}
......@@ -102,7 +98,7 @@ map< std::string, const google::protobuf::Message & > profanedb::storage::Parser
{
// The Descriptor is manually extracted from the pool,
// removing the prepending `type.googleapis.com/` in the Any message
std::string type = serializable.type_url();
string type = serializable.type_url();
const Descriptor * definition = pool->FindMessageTypeByName(type.substr(type.rfind('/')+1, string::npos));
Message * container = messageFactory.GetPrototype(definition)->New();
......@@ -113,7 +109,7 @@ map< std::string, const google::protobuf::Message & > profanedb::storage::Parser
map< std::string, const google::protobuf::Message & > profanedb::storage::Parser::NormalizeMessage(const google::protobuf::Message & message)
{
auto dependencies = new map<std::string, const google::protobuf::Message &>();
auto dependencies = new map< std::string, const google::protobuf::Message & >();
// TODO This only takes set fields into account. Maybe using Descriptor::field(0 <= i < field_count()) is better
auto fields = new std::vector< const FieldDescriptor * >();
......@@ -122,7 +118,7 @@ map< std::string, const google::protobuf::Message & > profanedb::storage::Parser
DescriptorProto * descProto = new DescriptorProto();
message.GetDescriptor()->CopyTo(descProto);
std::string key;
string key;
for (auto const & fd: *fields) {
if (fd->message_type() != NULL) {
......@@ -139,56 +135,56 @@ map< std::string, const google::protobuf::Message & > profanedb::storage::Parser
// TODO This only uses a single key, one could set multiple keys
if (options.key()) {
key = FieldToKey(&message, fd);
key = FieldToKey(message, *fd);
}
}
}
dependencies->insert( std::pair< std::string, const google::protobuf::Message & >(key, message) );
dependencies->insert( std::pair< string, const google::protobuf::Message & >(key, message) );
return *dependencies;
}
std::string profanedb::storage::Parser::FieldToKey(const google::protobuf::Message * container, const google::protobuf::FieldDescriptor * fd)
string profanedb::storage::Parser::FieldToKey(const google::protobuf::Message & container, const google::protobuf::FieldDescriptor & fd)
{
const Reflection * reflection = container->GetReflection();
const Reflection * reflection = container.GetReflection();
std::string key_value;
string key_value;
switch (fd->cpp_type()) {
switch (fd.cpp_type()) {
case FieldDescriptor::CPPTYPE_INT32:
key_value = std::to_string(reflection->GetInt32(*container, fd));
key_value = std::to_string(reflection->GetInt32(container, &fd));
break;
case FieldDescriptor::CPPTYPE_INT64:
key_value = std::to_string(reflection->GetInt64(*container, fd));
key_value = std::to_string(reflection->GetInt64(container, &fd));
break;
case FieldDescriptor::CPPTYPE_UINT32:
key_value = std::to_string(reflection->GetUInt32(*container, fd));
key_value = std::to_string(reflection->GetUInt32(container, &fd));
break;
case FieldDescriptor::CPPTYPE_UINT64:
key_value = std::to_string(reflection->GetUInt64(*container, fd));
key_value = std::to_string(reflection->GetUInt64(container, &fd));
break;
case FieldDescriptor::CPPTYPE_DOUBLE:
key_value = std::to_string(reflection->GetDouble(*container, fd));
key_value = std::to_string(reflection->GetDouble(container, &fd));
break;
case FieldDescriptor::CPPTYPE_FLOAT:
key_value = std::to_string(reflection->GetFloat(*container, fd));
key_value = std::to_string(reflection->GetFloat(container, &fd));
break;
case FieldDescriptor::CPPTYPE_BOOL:
key_value = std::to_string(reflection->GetBool(*container, fd));
key_value = std::to_string(reflection->GetBool(container, &fd));
break;
case FieldDescriptor::CPPTYPE_ENUM:
key_value = std::to_string(reflection->GetEnum(*container, fd)->index());
key_value = std::to_string(reflection->GetEnum(container, &fd)->index());
break;
case FieldDescriptor::CPPTYPE_STRING:
key_value = reflection->GetString(*container, fd);
key_value = reflection->GetString(container, &fd);
break;
case FieldDescriptor::CPPTYPE_MESSAGE:
key_value = reflection->GetMessage(*container, fd, &messageFactory).SerializeAsString();
key_value = reflection->GetMessage(container, &fd, &messageFactory).SerializeAsString();
break;
}
return fd->full_name() + '$' + key_value;
return fd.full_name() + '$' + key_value;
}
profanedb::storage::Parser::ErrorCollector::ErrorCollector()
......
......@@ -17,11 +17,12 @@
*
*/
#ifndef PARSER_H
#define PARSER_H
#ifndef PROFANEDB_STORAGE_PARSER_H
#define PROFANEDB_STORAGE_PARSER_H
#include <iostream>
#include <map>
#include <string>
#include <google/protobuf/compiler/importer.h>
#include <google/protobuf/io/zero_copy_stream.h>
......@@ -32,6 +33,8 @@
#include <profanedb/protobuf/db.pb.h>
#include <profanedb/protobuf/options.pb.h>
#include "config.h"
using namespace google::protobuf;
namespace profanedb {
......@@ -42,7 +45,7 @@ namespace storage {
class Parser
{
public:
Parser();
Parser(const Config::ProfaneDB & profaneConfig);
~Parser();
// Using the provided schema, get a map with all nested messages and their unique key
......@@ -50,30 +53,29 @@ public:
map<std::string, const Message &> NormalizeMessage(const Message & message);
private:
io::ZeroCopyInputStream * inputStream;
compiler::DiskSourceTree sourceTree;
compiler::MultiFileErrorCollector * errCollector = new ErrorCollector();
compiler::SourceTreeDescriptorDatabase * descriptorDb;
DescriptorPool * pool;
// A simple ErrorCollector for debug, write to stderr
class ErrorCollector : public compiler::MultiFileErrorCollector {
public:
ErrorCollector();
void AddError(const string & filename, int line, int column, const string & message) override;
void AddWarning(const string & filename, int line, int column, const string & message) override;
};
ErrorCollector errCollector;
std::shared_ptr<compiler::SourceTreeDescriptorDatabase> descriptorDb;
std::shared_ptr<DescriptorPool> pool;
DynamicMessageFactory messageFactory;
FileDescriptorProto fileDescProto;
// Given a Field
std::string FieldToKey(const Message * container, const FieldDescriptor * fd);
string FieldToKey(const Message & container, const FieldDescriptor & fd);
// Given a descriptor, find information about the key and nested objects,
// return true if the message has a key
bool ParseMessageDescriptor(const Descriptor & descriptor);
// A simple ErrorCollector for debug, write to stderr
class ErrorCollector : public compiler::MultiFileErrorCollector {
public:
ErrorCollector();
void AddError(const string & filename, int line, int column, const string & message) override;
void AddWarning(const string & filename, int line, int column, const string & message) override;
};
};
}
}
#endif // PARSER_H
#endif // PROFANEDB_STORAGE_PARSER_H
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment