Verified Commit 6443651c authored by tglman's avatar tglman

replaced unsigned-varint with interger-encoding add some internal varint encoding implementations

parent 3040c857
Pipeline #168790675 passed with stages
in 17 minutes and 54 seconds
......@@ -21,7 +21,7 @@ rand = "0.7"
fs2 = "0.4"
data-encoding = "2.1"
crc= "1.0"
unsigned-varint = "0.4"
integer-encoding = "1.1"
[dev-dependencies]
serde = "1.0"
......
......@@ -288,8 +288,6 @@ impl Address {
if seg_page.segment_delete_entry(delete.segment, delete.recref.pos)? {
// Avoid to remove last pages, for avoid concurrent operations with page
// creation
// TODO: this is not enough because allocations can go over the size of a page,
// so is needed to track allocations in a different way than deletes
if seg_page.get_next()? != 0 {
pages_to_remove.push((delete.segment, page));
}
......
......@@ -2,14 +2,12 @@ pub use std::fs::OpenOptions;
use std::{borrow::Cow, error, fmt, io, str, sync};
use crate::id::PersyId;
use unsigned_varint::decode::Error as VarintError;
/// Enum of possible errors from Persy
#[derive(Debug)]
pub enum PersyError {
Io(io::Error),
DecodingUtf8(str::Utf8Error),
DecodingVarint(VarintError),
DecodingDataEncoding(data_encoding::DecodeError),
Custom(Box<dyn error::Error + Send + Sync + 'static>),
VersionNotLastest,
......@@ -23,6 +21,7 @@ pub enum PersyError {
IndexTypeMismatch(Cow<'static, str>),
IndexDuplicateKey(String, String),
TransactionTimeout,
InvalidId(String),
#[doc(hidden)]
__NonExhausive,
......@@ -54,12 +53,6 @@ impl From<data_encoding::DecodeError> for PersyError {
}
}
impl From<VarintError> for PersyError {
fn from(err: VarintError) -> PersyError {
PersyError::DecodingVarint(err)
}
}
impl From<Box<dyn error::Error + Send + Sync + 'static>> for PersyError {
fn from(err: Box<dyn error::Error + Send + Sync + 'static>) -> PersyError {
PersyError::Custom(err)
......@@ -72,7 +65,6 @@ impl error::Error for PersyError {
match self {
Io(ref e) => Some(e),
DecodingUtf8(ref e) => Some(e),
DecodingVarint(ref e) => Some(e),
DecodingDataEncoding(ref e) => Some(e),
Custom(ref e) => Some(e.as_ref()),
_ => None,
......@@ -87,7 +79,6 @@ impl fmt::Display for PersyError {
Io(m) => write!(f, "IO Error: {}", m),
DecodingUtf8(e) => write!(f, "String decoding error: {}", e),
DecodingDataEncoding(e) => write!(f, "Data Encoding Decoding error: {}", e),
DecodingVarint(e) => write!(f, "Varint decoding error: {}", e),
Custom(e) => write!(f, "{}", e),
VersionNotLastest => write!(f, "The record version is not latest"),
RecordNotFound(r) => write!(f, "Record not found: {}", r),
......@@ -112,6 +103,7 @@ impl fmt::Display for PersyError {
IndexDuplicateKey(i, k) => write!(f, "Found duplicate key:{} for index: {}", k, i),
TransactionTimeout => write!(f, "Timeout acquiring the data locks for the transaction"),
InvalidId(id) => write!(f, "The id '{}' has no valid format", id),
__NonExhausive => write!(f, "__non-Exhausive"),
}
......
......@@ -7,25 +7,19 @@ use crate::{
transaction::{Transaction, TxSegCheck},
};
use data_encoding::BASE32_DNSSEC;
use integer_encoding::{VarInt, VarIntReader};
pub use std::fs::OpenOptions;
use std::{fmt, io::Write, str};
use unsigned_varint::{
decode::{u32 as u32_vdec, u64 as u64_vdec},
encode::{u32 as u32_venc, u32_buffer, u64 as u64_venc, u64_buffer},
};
use std::{fmt, io::Cursor, str};
fn write_id(f: &mut fmt::Formatter, id: u32) -> fmt::Result {
let mut buffer = Vec::new();
buffer
.write_all(u32_venc(id, &mut u32_buffer()))
.expect("no failure expected only memory allocation");
let mut buffer = id.encode_var_vec();
buffer.push(0b0101_0101);
write!(f, "{}", BASE32_DNSSEC.encode(&buffer))
}
fn read_id(s: &str) -> PRes<u32> {
let bytes = BASE32_DNSSEC.decode(s.as_bytes())?;
Ok(u32_vdec(&bytes)?.0)
VarIntReader::read_varint(&mut Cursor::new(bytes)).map_err(|_| PersyError::InvalidId(String::from(s)))
}
#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Debug)]
......@@ -43,14 +37,9 @@ impl RecRef {
impl fmt::Display for RecRef {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let mut buffer = Vec::new();
buffer
.write_all(u64_venc(self.page, &mut u64_buffer()))
.expect("no failure expected only memory allocation");
let mut buffer = self.page.encode_var_vec();
buffer.push(0b0101_0101);
buffer
.write_all(u32_venc(self.pos, &mut u32_buffer()))
.expect("no failure expected only memory allocation");
buffer.extend_from_slice(&self.pos.encode_var_vec());
write!(f, "{}", BASE32_DNSSEC.encode(&buffer))
}
}
......@@ -60,8 +49,14 @@ impl std::str::FromStr for RecRef {
fn from_str(s: &str) -> Result<Self, Self::Err> {
let bytes = BASE32_DNSSEC.decode(s.as_bytes())?;
let (page, rest) = u64_vdec(&bytes)?;
let (pos, _) = u32_vdec(&rest[1..])?;
let mut cursor = Cursor::new(bytes);
let page = cursor
.read_varint()
.map_err(|_| PersyError::InvalidId(String::from(s)))?;
cursor.set_position(cursor.position() + 1);
let pos = cursor
.read_varint()
.map_err(|_| PersyError::InvalidId(String::from(s)))?;
Ok(RecRef::new(page, pos))
}
}
......@@ -302,13 +297,8 @@ impl IndexId {
impl fmt::Display for IndexId {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let mut buffer = Vec::new();
buffer
.write_all(u32_venc(self.meta, &mut u32_buffer()))
.expect("no failure expected only memory allocation");
buffer
.write_all(u32_venc(self.data, &mut u32_buffer()))
.expect("no failure expected only memory allocation");
let mut buffer = self.meta.encode_var_vec();
buffer.extend_from_slice(&self.data.encode_var_vec());
buffer.push(0b0101_0101);
write!(f, "{}", BASE32_DNSSEC.encode(&buffer))
}
......@@ -319,10 +309,14 @@ impl std::str::FromStr for IndexId {
fn from_str(s: &str) -> Result<Self, Self::Err> {
let bytes = BASE32_DNSSEC.decode(s.as_bytes())?;
let (meta, bytes) = u32_vdec(&bytes)?;
let data = if bytes.len() > 1 {
let (d, _) = u32_vdec(&bytes)?;
d
let mut cursor = Cursor::new(bytes);
let meta = cursor
.read_varint()
.map_err(|_| PersyError::InvalidId(String::from(s)))?;
let data = if cursor.get_ref().len() - cursor.position() as usize > 1 {
cursor
.read_varint()
.map_err(|_| PersyError::InvalidId(String::from(s)))?
} else {
0
};
......
use crate::PRes;
use byteorder::{BigEndian, ByteOrder, ReadBytesExt, WriteBytesExt};
use integer_encoding::VarInt;
use std::io::{Cursor, Read, Write};
pub trait InfallibleRead {
......@@ -10,6 +11,123 @@ pub trait InfallibleWrite {
fn write_all(&mut self, buf: &[u8]);
}
#[allow(dead_code)]
pub(crate) fn write_varint<V: VarInt, W: InfallibleWrite + ?Sized>(w: &mut W, v: V) {
let mut buf = [0 as u8; 10];
let b = v.encode_var(&mut buf);
w.write_all(&buf[0..b])
}
// Start of snippet taken from integer-ecoding specialized for InfallibleRead
pub const MSB: u8 = 0b1000_0000;
#[allow(dead_code)]
pub(crate) fn read_varint<V: VarInt, R: InfallibleRead + ?Sized>(r: &mut R) -> V {
let mut int_buf = [0 as u8; 10];
let mut int_cursor = 0;
let mut buf = [0 as u8; 1];
while int_cursor == 0 || (int_buf[int_cursor - 1] & MSB != 0) {
r.read_exact(&mut buf);
if int_cursor >= 10 {
panic!("Unterminated varint")
}
int_buf[int_cursor] = buf[0];
int_cursor = 0;
}
V::decode_var(&int_buf[0..int_cursor]).0
}
// End of snippet taken from integer-ecoding specialized for InfallibleRead
pub(crate) trait InfallibleReadVarInt: InfallibleRead {
#[inline]
fn read_u8_varint(&mut self) -> u8 {
read_varint(self)
}
#[inline]
fn read_u16_varint(&mut self) -> u16 {
read_varint(self)
}
#[inline]
fn read_u32_varint(&mut self) -> u32 {
read_varint(self)
}
#[inline]
fn read_u64_varint(&mut self) -> u64 {
read_varint(self)
}
#[inline]
fn read_i8_varint(&mut self) -> i8 {
read_varint(self)
}
#[inline]
fn read_i16_varint(&mut self) -> i16 {
read_varint(self)
}
#[inline]
fn read_i32_varint(&mut self) -> i32 {
read_varint(self)
}
#[inline]
fn read_i64_varint(&mut self) -> i64 {
read_varint(self)
}
}
impl<R: InfallibleRead + ?Sized> InfallibleReadVarInt for R {}
pub(crate) trait InfallibleWriteVarInt: InfallibleWrite {
#[inline]
fn write_varint_u8(&mut self, value: u8) {
write_varint(self, value)
}
#[inline]
fn write_varint_u16(&mut self, value: u16) {
write_varint(self, value)
}
#[inline]
fn write_varint_u32(&mut self, value: u32) {
write_varint(self, value)
}
#[inline]
fn write_varint_u64(&mut self, value: u64) {
write_varint(self, value)
}
#[inline]
fn write_varint_i8(&mut self, value: i8) {
write_varint(self, value)
}
#[inline]
fn write_varint_i16(&mut self, value: i16) {
write_varint(self, value)
}
#[inline]
fn write_varint_i32(&mut self, value: i32) {
write_varint(self, value)
}
#[inline]
fn write_varint_i64(&mut self, value: i64) {
write_varint(self, value)
}
}
impl<W: InfallibleWrite + ?Sized> InfallibleWriteVarInt for W {}
pub(crate) trait InfallibleWriteFormat: InfallibleWrite {
#[inline]
fn write_u8(&mut self, value: u8) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment