Commit 66245ffa authored by Giorgio Azzinnaro's avatar Giorgio Azzinnaro

test boot filesystem glob loading

parent ebf92b5a
syntax = "proto3";
package test;
import "profanedb/protobuf/options.proto";
message ExternalNested {
double field_one_double = 1 [ (profanedb.protobuf.options).key = true ];
}
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: nested.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from profanedb.protobuf import options_pb2 as profanedb_dot_protobuf_dot_options__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='nested.proto',
package='test',
syntax='proto3',
serialized_pb=_b('\n\x0cnested.proto\x12\x04test\x1a profanedb/protobuf/options.proto\"2\n\x0e\x45xternalNested\x12 \n\x10\x66ield_one_double\x18\x01 \x01(\x01\x42\x06\xfa\xe9\x30\x02\x08\x01\x62\x06proto3')
,
dependencies=[profanedb_dot_protobuf_dot_options__pb2.DESCRIPTOR,])
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_EXTERNALNESTED = _descriptor.Descriptor(
name='ExternalNested',
full_name='test.ExternalNested',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='field_one_double', full_name='test.ExternalNested.field_one_double', index=0,
number=1, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), _b('\372\3510\002\010\001'))),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=56,
serialized_end=106,
)
DESCRIPTOR.message_types_by_name['ExternalNested'] = _EXTERNALNESTED
ExternalNested = _reflection.GeneratedProtocolMessageType('ExternalNested', (_message.Message,), dict(
DESCRIPTOR = _EXTERNALNESTED,
__module__ = 'nested_pb2'
# @@protoc_insertion_point(class_scope:test.ExternalNested)
))
_sym_db.RegisterMessage(ExternalNested)
_EXTERNALNESTED.fields_by_name['field_one_double'].has_options = True
_EXTERNALNESTED.fields_by_name['field_one_double']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), _b('\372\3510\002\010\001'))
try:
# THESE ELEMENTS WILL BE DEPRECATED.
# Please use the generated *_pb2_grpc.py files instead.
import grpc
from grpc.beta import implementations as beta_implementations
from grpc.beta import interfaces as beta_interfaces
from grpc.framework.common import cardinality
from grpc.framework.interfaces.face import utilities as face_utilities
except ImportError:
pass
# @@protoc_insertion_point(module_scope)
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
import grpc
......@@ -4,6 +4,8 @@ package test;
import "profanedb/protobuf/options.proto";
import "nested.proto";
message Test {
int32 field_one_int = 1 [ (profanedb.protobuf.options).key = true ];
string field_two_str = 2;
......@@ -11,6 +13,8 @@ message Test {
bytes field_four_bytes = 4;
Nested field_five_nested = 5;
ExternalNested field_six_externalnested = 6;
}
message Nested {
......
......@@ -14,15 +14,16 @@ _sym_db = _symbol_database.Default()
from profanedb.protobuf import options_pb2 as profanedb_dot_protobuf_dot_options__pb2
import nested_pb2 as nested__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='test.proto',
package='test',
syntax='proto3',
serialized_pb=_b('\n\ntest.proto\x12\x04test\x1a profanedb/protobuf/options.proto\"\x99\x01\n\x04Test\x12\x1d\n\rfield_one_int\x18\x01 \x01(\x05\x42\x06\xfa\xe9\x30\x02\x08\x01\x12\x15\n\rfield_two_str\x18\x02 \x01(\t\x12\x18\n\x10\x66ield_three_bool\x18\x03 \x01(\x08\x12\x18\n\x10\x66ield_four_bytes\x18\x04 \x01(\x0c\x12\'\n\x11\x66ield_five_nested\x18\x05 \x01(\x0b\x32\x0c.test.Nested\"o\n\x06Nested\x12$\n\x14nested_field_one_str\x18\x01 \x01(\tB\x06\xfa\xe9\x30\x02\x08\x01\x12\x1c\n\x14nested_field_two_int\x18\x02 \x01(\x03\x12!\n\x19nested_field_three_double\x18\x03 \x01(\x01\x62\x06proto3')
serialized_pb=_b('\n\ntest.proto\x12\x04test\x1a profanedb/protobuf/options.proto\x1a\x0cnested.proto\"\xd1\x01\n\x04Test\x12\x1d\n\rfield_one_int\x18\x01 \x01(\x05\x42\x06\xfa\xe9\x30\x02\x08\x01\x12\x15\n\rfield_two_str\x18\x02 \x01(\t\x12\x18\n\x10\x66ield_three_bool\x18\x03 \x01(\x08\x12\x18\n\x10\x66ield_four_bytes\x18\x04 \x01(\x0c\x12\'\n\x11\x66ield_five_nested\x18\x05 \x01(\x0b\x32\x0c.test.Nested\x12\x36\n\x18\x66ield_six_externalnested\x18\x06 \x01(\x0b\x32\x14.test.ExternalNested\"o\n\x06Nested\x12$\n\x14nested_field_one_str\x18\x01 \x01(\tB\x06\xfa\xe9\x30\x02\x08\x01\x12\x1c\n\x14nested_field_two_int\x18\x02 \x01(\x03\x12!\n\x19nested_field_three_double\x18\x03 \x01(\x01\x62\x06proto3')
,
dependencies=[profanedb_dot_protobuf_dot_options__pb2.DESCRIPTOR,])
dependencies=[profanedb_dot_protobuf_dot_options__pb2.DESCRIPTOR,nested__pb2.DESCRIPTOR,])
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
......@@ -70,6 +71,13 @@ _TEST = _descriptor.Descriptor(
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='field_six_externalnested', full_name='test.Test.field_six_externalnested', index=5,
number=6, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
......@@ -82,8 +90,8 @@ _TEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=55,
serialized_end=208,
serialized_start=69,
serialized_end=278,
)
......@@ -127,11 +135,12 @@ _NESTED = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=210,
serialized_end=321,
serialized_start=280,
serialized_end=391,
)
_TEST.fields_by_name['field_five_nested'].message_type = _NESTED
_TEST.fields_by_name['field_six_externalnested'].message_type = nested__pb2._EXTERNALNESTED
DESCRIPTOR.message_types_by_name['Test'] = _TEST
DESCRIPTOR.message_types_by_name['Nested'] = _NESTED
......
......@@ -20,7 +20,7 @@ profanedb::protobuf::PutResp profanedb::storage::Db::Put(const profanedb::protob
auto map = parser.ParseMessage(request.serializable());
for (auto const & obj: map) {
std::cout << obj.first << ":" << std::endl << obj.second.DebugString() << std::endl;
std::cout << obj.first << ":" << std::endl << obj.second.SerializeAsString() << std::endl;
}
}
......
......@@ -8,8 +8,8 @@ profanedb::storage::Parser::Parser()
// Everything is mapped to "" to preserve directory structure
sourceTree.MapPath("", "/usr/include"); // google/protobuf/... should be here
sourceTree.MapPath("", kOptions.native());
sourceTree.MapPath("", kDbSchema.native());
sourceTree.MapPath("", kOptions.string());
sourceTree.MapPath("", kDbSchema.string());
inputStream = sourceTree.Open("");
if (inputStream == NULL) {
......@@ -21,17 +21,18 @@ profanedb::storage::Parser::Parser()
pool = new DescriptorPool(descriptorDb);
// Load options to be used during file import
// Load ProfaneDB options to be used during file import
pool->FindFileByName("profanedb/protobuf/options.proto");
// Import all `.proto` files in kDbSchema into the pool,
// so that FindMessageTypeByName can then be used
boost::filesystem::path path(kDbSchema);
for (auto const & file: boost::filesystem::recursive_directory_iterator(path, boost::filesystem::symlink_option::recurse)) {
if (file.path().extension() == ".proto") {
// For the pool every file is relative to the mapping provided before (kDbSchema)
pool->FindFileByName(file.path().lexically_relative(kDbSchema).native());
// TODO Here should load all the messages, find the keys and nested messages, and generate Descriptors with references to save in DB
}
}
}
......@@ -40,7 +41,7 @@ profanedb::storage::Parser::~Parser()
{
}
map< std::string, const google::protobuf::Message & > profanedb::storage::Parser::ParseMessage(const google::protobuf::Any& serializable)
map< std::string, const google::protobuf::Message & > profanedb::storage::Parser::ParseMessage(const google::protobuf::Any & serializable)
{
// The Descriptor is manually extracted from the pool,
// removing the prepending `type.googleapis.com/` in the Any message
......@@ -49,29 +50,39 @@ map< std::string, const google::protobuf::Message & > profanedb::storage::Parser
Message * container = messageFactory.GetPrototype(definition)->New();
serializable.UnpackTo(container);
return ParseMessage(*container);
}
map< std::string, const google::protobuf::Message & > profanedb::storage::Parser::ParseMessage(const google::protobuf::Message & message)
{
auto dependencies = new map<std::string, const google::protobuf::Message &>();
auto fields = new std::vector< const FieldDescriptor * >();
// TODO This only takes set fields into account. Maybe using Descriptor::field(0 <= i < field_count()) is better
auto fields = new std::vector< const FieldDescriptor * >();
message.GetReflection()->ListFields(message, fields);
DescriptorProto * descProto = new DescriptorProto();
message.GetDescriptor()->CopyTo(descProto);
std::string key;
for (auto const & fd: *fields) {
if (fd->message_type() != NULL) {
auto nested = ParseMessage(message.GetReflection()->GetMessage(message, fd, &messageFactory));
dependencies->insert(nested.begin(), nested.end());
// TODO If nested has primary key set a reference
if (nested.size() > 0) {
// The nested message might contain other messages, all of them are stored in the dependency map
dependencies->insert(nested.begin(), nested.end());
}
} else {
auto options = fd->options().GetExtension(profanedb::protobuf::options);
// TODO This only uses a single key, one could set multiple keys
if (options.key()) {
key = fd->full_name() + "$" + FieldToString(&message, fd);
key = fd->full_name() + '$' + FieldToString(&message, fd);
}
}
}
......@@ -84,41 +95,39 @@ map< std::string, const google::protobuf::Message & > profanedb::storage::Parser
std::string profanedb::storage::Parser::FieldToString(const google::protobuf::Message * container, const google::protobuf::FieldDescriptor * fd)
{
const Reflection * reflection = container->GetReflection();
std::string field_bytes;
switch (fd->cpp_type()) {
case FieldDescriptor::CPPTYPE_INT32:
field_bytes = std::to_string(reflection->GetInt32(*container, fd));
return std::to_string(reflection->GetInt32(*container, fd));
break;
case FieldDescriptor::CPPTYPE_INT64:
field_bytes = std::to_string(reflection->GetInt64(*container, fd));
return std::to_string(reflection->GetInt64(*container, fd));
break;
case FieldDescriptor::CPPTYPE_UINT32:
field_bytes = std::to_string(reflection->GetUInt32(*container, fd));
return std::to_string(reflection->GetUInt32(*container, fd));
break;
case FieldDescriptor::CPPTYPE_UINT64:
field_bytes = std::to_string(reflection->GetUInt64(*container, fd));
return std::to_string(reflection->GetUInt64(*container, fd));
break;
case FieldDescriptor::CPPTYPE_DOUBLE:
field_bytes = std::to_string(reflection->GetDouble(*container, fd));
return std::to_string(reflection->GetDouble(*container, fd));
break;
case FieldDescriptor::CPPTYPE_FLOAT:
field_bytes = std::to_string(reflection->GetFloat(*container, fd));
return std::to_string(reflection->GetFloat(*container, fd));
break;
case FieldDescriptor::CPPTYPE_BOOL:
field_bytes = std::to_string(reflection->GetBool(*container, fd));
return std::to_string(reflection->GetBool(*container, fd));
break;
case FieldDescriptor::CPPTYPE_ENUM:
field_bytes = std::to_string(reflection->GetEnum(*container, fd)->index());
return std::to_string(reflection->GetEnum(*container, fd)->index());
break;
case FieldDescriptor::CPPTYPE_STRING:
field_bytes = reflection->GetString(*container, fd);
return reflection->GetString(*container, fd);
break;
case FieldDescriptor::CPPTYPE_MESSAGE:
// TODO Normalize data here
return reflection->GetMessage(*container, fd, &messageFactory).SerializeAsString();
break;
}
return field_bytes;
}
profanedb::storage::Parser::ErrorCollector::ErrorCollector()
......
../client_test/nested.proto
\ No newline at end of file
../client_test/test.proto
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment