Verified Commit 9eb83cd5 authored by Phil Booth's avatar Phil Booth

refactor(tokens): make arena reference-counted member instead of static

The initial motivation for this was the possibility of multiple arenas
per file, which obviously precludes having a single static arena.

But happily, in making the change I also fixed two other problems:

1. The tests no longer need to be run in a single thread.

2. The "mystery" of missing odd-numbered token ids was fixed by using
   mem::replace instead of Vec::insert inside TokenArena::set.
parent f460e3c6
Pipeline #33036249 passed with stage
in 3 minutes and 34 seconds
......@@ -19,4 +19,4 @@ test:cargo:
- rustc --version && cargo --version
- cargo fmt -- --check
- cargo clippy
- cargo t -- --test-threads=1
- cargo t
......@@ -18,12 +18,16 @@
#[cfg(test)]
mod test;
use tokens::{arena, Token, TokenId};
use std::sync::Arc;
use parking_lot::{RwLock, RwLockReadGuard};
use tokens::{Token, TokenArena, TokenId};
macro_rules! simple_move {
($self:ident, $delta:ident, $id:expr, $index:ident, $move_absolute:ident) => {
if let Some(id) = $id {
let ids = arena().read(id, &|token| {
let ids = $self.arena().read(id, &|token| {
if $delta >= 0 {
token.following_siblings()
} else {
......@@ -40,21 +44,28 @@ macro_rules! simple_move {
};
}
#[derive(Debug, Default)]
#[derive(Debug)]
pub struct Cursor {
ideal_grapheme_index: usize,
arena: Arc<RwLock<TokenArena>>,
grapheme_id: Option<TokenId>,
grapheme_index: usize,
ideal_grapheme_index: usize,
line_index: usize,
stick_to_line_end: bool,
}
impl Cursor {
pub fn new(grapheme_id: Option<TokenId>, grapheme_index: usize, line_index: usize) -> Self {
pub fn new(
arena: Arc<RwLock<TokenArena>>,
grapheme_id: Option<TokenId>,
grapheme_index: usize,
line_index: usize,
) -> Self {
Self {
ideal_grapheme_index: grapheme_index,
arena,
grapheme_id,
grapheme_index,
ideal_grapheme_index: grapheme_index,
line_index,
stick_to_line_end: false,
}
......@@ -93,7 +104,10 @@ impl Cursor {
break;
}
if arena().read(grapheme_id, &|grapheme| grapheme.is_newline_grapheme()) {
if self
.arena()
.read(grapheme_id, &|grapheme| grapheme.is_newline_grapheme())
{
break;
}
......@@ -104,11 +118,15 @@ impl Cursor {
count
}
fn arena(&self) -> RwLockReadGuard<TokenArena> {
self.arena.read()
}
pub fn move_word(&mut self, delta: isize) {
self.stick_to_line_end = false;
if let Some(word_id) = self.word_id() {
let word_ids = arena().read(word_id, &|word| {
let word_ids = self.arena().read(word_id, &|word| {
if delta >= 0 {
word.following_siblings()
} else {
......@@ -136,7 +154,8 @@ impl Cursor {
fn word_id(&self) -> Option<TokenId> {
if let Some(grapheme_id) = self.grapheme_id {
arena().read(grapheme_id, &|grapheme| grapheme.parent())
self.arena()
.read(grapheme_id, &|grapheme| grapheme.parent())
} else {
None
}
......@@ -160,8 +179,9 @@ impl Cursor {
break;
}
let (is_newline, is_space) =
arena().read(word_id, &|word| (word.is_newline_word(), word.is_space()));
let (is_newline, is_space) = self
.arena()
.read(word_id, &|word| (word.is_newline_word(), word.is_space()));
if is_newline {
line_count += 1;
......@@ -173,8 +193,9 @@ impl Cursor {
}
if let Some(new_word_id) = new_word_id {
let (grapheme_id, line_id) =
arena().read(new_word_id, &|word| (position(&word), word.parent()));
let (grapheme_id, line_id) = self
.arena()
.read(new_word_id, &|word| (position(&word), word.parent()));
self.move_to_grapheme_on_line(grapheme_id, line_id);
}
......@@ -187,21 +208,25 @@ impl Cursor {
if let Some(grapheme_id) = grapheme_id {
if let Some(line_id) = line_id {
let first_word_id = arena().read(line_id, &|line| line.first_child().unwrap());
let first_grapheme_id =
arena().read(first_word_id, &|word| word.first_child().unwrap());
let first_word_id = self
.arena()
.read(line_id, &|line| line.first_child().unwrap());
let first_grapheme_id = self
.arena()
.read(first_word_id, &|word| word.first_child().unwrap());
if grapheme_id != first_grapheme_id {
let second_grapheme_id =
arena().read(first_grapheme_id, &|grapheme| grapheme.next().unwrap());
let second_grapheme_id = self
.arena()
.read(first_grapheme_id, &|grapheme| grapheme.next().unwrap());
if grapheme_id == second_grapheme_id {
self.grapheme_index = 1;
} else {
let grapheme_ids = arena().read(second_grapheme_id, &|grapheme| {
let grapheme_ids = self.arena().read(second_grapheme_id, &|grapheme| {
grapheme.iter(None, Some(grapheme_id))
});
let line_length = arena().read(line_id, &|line| line.len());
let line_length = self.arena().read(line_id, &|line| line.len());
self.grapheme_index =
self.move_grapheme_absolute(line_length, grapheme_ids);
}
......@@ -214,7 +239,7 @@ impl Cursor {
self.stick_to_line_end = false;
if let Some(word_id) = self.word_id() {
let word_ids = arena().read(word_id, &|word| {
let word_ids = self.arena().read(word_id, &|word| {
if delta >= 0 {
if self.grapheme_id.unwrap() == word.last_child().unwrap() {
word.following_siblings()
......@@ -250,7 +275,7 @@ impl Cursor {
fn line_id(&self) -> Option<TokenId> {
if let Some(word_id) = self.word_id() {
arena().read(word_id, &|word| word.parent())
self.arena().read(word_id, &|word| word.parent())
} else {
None
}
......@@ -273,15 +298,17 @@ impl Cursor {
}
if let Some(new_line_id) = new_line_id {
let word_id = arena().read(new_line_id, &|line| line.first_child());
let word_id = self.arena().read(new_line_id, &|line| line.first_child());
if let Some(word_id) = word_id {
self.grapheme_id = arena().read(word_id, &|word| word.first_child());
let arena = self.arena.clone();
self.grapheme_id = arena.read().read(word_id, &|word| word.first_child());
if let Some(grapheme_id) = self.grapheme_id {
let grapheme_index = self.ideal_grapheme_index;
let grapheme_ids =
arena().read(grapheme_id, &|grapheme| grapheme.following_siblings());
let grapheme_ids = arena
.read()
.read(grapheme_id, &|grapheme| grapheme.following_siblings());
self.grapheme_index = self.move_grapheme_absolute(grapheme_index, grapheme_ids);
}
}
......@@ -292,7 +319,7 @@ impl Cursor {
pub fn move_to_line_end(&mut self) {
if let Some(line_id) = self.line_id() {
let line_length = arena().read(line_id, &|line| line.len());
let line_length = self.arena().read(line_id, &|line| line.len());
if line_length > self.grapheme_index + 1 {
let delta = line_length - self.grapheme_index - 1;
self.move_grapheme(delta as isize);
......@@ -308,7 +335,7 @@ impl Cursor {
}
if let Some(line_id) = self.line_id() {
let line_ids = arena().read(line_id, &|line| {
let line_ids = self.arena().read(line_id, &|line| {
if delta > 0 {
line.following_siblings()
} else {
......@@ -336,10 +363,9 @@ impl Cursor {
let mut count = 0;
let mut new_line_id = None;
let mut started = true;
let mut finished = false;
if let Some(line_id) = self.line_id() {
if arena().read(line_id, &|line| line.is_newline_line()) {
if self.arena().read(line_id, &|line| line.is_newline_line()) {
// Seek past current blank lines before looking for the next one
started = false;
}
......@@ -349,9 +375,8 @@ impl Cursor {
count += 1;
new_line_id = Some(line_id);
if arena().read(line_id, &|line| line.is_newline_line()) {
if self.arena().read(line_id, &|line| line.is_newline_line()) {
if started {
finished = true;
break;
}
} else if !started {
......@@ -360,8 +385,11 @@ impl Cursor {
}
if let Some(new_line_id) = new_line_id {
let word_id = arena().read(new_line_id, &|line| line.first_child().unwrap());
self.grapheme_id = arena().read(word_id, &|word| word.first_child());
let word_id = self
.arena()
.read(new_line_id, &|line| line.first_child().unwrap());
let arena = self.arena.clone();
self.grapheme_id = arena.read().read(word_id, &|word| word.first_child());
self.grapheme_index = 0;
}
......
......@@ -15,43 +15,49 @@
// You should have received a copy of the GNU General Public License along with
// pbvi. If not, see <https://www.gnu.org/licenses/>.
use super::Cursor;
use tokens::{arena, parse};
use super::*;
use tokens::parse;
#[test]
fn grapheme_index() {
let cursor = Cursor::new(None, 0, 0);
let arena = Arc::new(RwLock::new(TokenArena::new()));
let cursor = Cursor::new(arena.clone(), None, 0, 0);
assert_eq!(cursor.grapheme_index(), 0);
let cursor = Cursor::new(None, 1, 0);
let cursor = Cursor::new(arena.clone(), None, 1, 0);
assert_eq!(cursor.grapheme_index(), 1);
let cursor = Cursor::new(None, 2, 0);
let cursor = Cursor::new(arena.clone(), None, 2, 0);
assert_eq!(cursor.grapheme_index(), 2);
let cursor = Cursor::new(None, 0, 1);
let cursor = Cursor::new(arena.clone(), None, 0, 1);
assert_eq!(cursor.grapheme_index(), 0);
}
#[test]
fn line_index() {
let cursor = Cursor::new(None, 0, 0);
let arena = Arc::new(RwLock::new(TokenArena::new()));
let cursor = Cursor::new(arena.clone(), None, 0, 0);
assert_eq!(cursor.line_index(), 0);
let cursor = Cursor::new(None, 0, 1);
let cursor = Cursor::new(arena.clone(), None, 0, 1);
assert_eq!(cursor.line_index(), 1);
let cursor = Cursor::new(None, 0, 2);
let cursor = Cursor::new(arena.clone(), None, 0, 2);
assert_eq!(cursor.line_index(), 2);
let cursor = Cursor::new(None, 1, 0);
let cursor = Cursor::new(arena.clone(), None, 1, 0);
assert_eq!(cursor.line_index(), 0);
}
#[test]
fn move_grapheme() {
let grapheme_ids = parse::graphemes("012").unwrap();
let mut cursor = Cursor::new(Some(grapheme_ids[0]), 0, 0);
let arena = Arc::new(RwLock::new(TokenArena::new()));
let grapheme_ids = parse::graphemes(arena.clone(), "012").unwrap();
let mut cursor = Cursor::new(arena.clone(), Some(grapheme_ids[0]), 0, 0);
cursor.move_grapheme(0);
assert_eq!(cursor.grapheme_index(), 0);
......@@ -100,10 +106,14 @@ fn move_grapheme() {
#[test]
fn move_grapheme_with_line_ending() {
let line_ids = parse::lines("01\n01").unwrap();
let word_id = arena().read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(grapheme_id, 0, 0);
let arena = Arc::new(RwLock::new(TokenArena::new()));
let line_ids = parse::lines(arena.clone(), "01\n01").unwrap();
let word_id = arena
.read()
.read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena.read().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(arena.clone(), grapheme_id, 0, 0);
cursor.move_grapheme(2);
assert_eq!(cursor.grapheme_index(), 1);
......@@ -132,13 +142,17 @@ fn move_grapheme_with_line_ending() {
#[test]
fn move_grapheme_with_paragraph_ending() {
let paragraph_ids = parse::paragraphs("01\r\n\r\n01").unwrap();
let line_id = arena().read(paragraph_ids[0], &|paragraph| {
let arena = Arc::new(RwLock::new(TokenArena::new()));
let paragraph_ids = parse::paragraphs(arena.clone(), "01\r\n\r\n01").unwrap();
let line_id = arena.read().read(paragraph_ids[0], &|paragraph| {
paragraph.first_child().unwrap()
});
let word_id = arena().read(line_id, &|line| line.first_child().unwrap());
let grapheme_id = arena().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(grapheme_id, 0, 0);
let word_id = arena
.read()
.read(line_id, &|line| line.first_child().unwrap());
let grapheme_id = arena.read().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(arena.clone(), grapheme_id, 0, 0);
cursor.move_grapheme(1);
assert_eq!(cursor.grapheme_index(), 1);
......@@ -171,10 +185,14 @@ fn move_grapheme_with_paragraph_ending() {
#[test]
fn move_word() {
let line_ids = parse::lines("0 11, 222\n3333").unwrap();
let word_id = arena().read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(grapheme_id, 0, 0);
let arena = Arc::new(RwLock::new(TokenArena::new()));
let line_ids = parse::lines(arena.clone(), "0 11, 222\n3333").unwrap();
let word_id = arena
.read()
.read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena.read().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(arena.clone(), grapheme_id, 0, 0);
cursor.move_word(0);
assert_eq!(cursor.grapheme_index(), 0);
......@@ -251,10 +269,14 @@ fn move_word() {
#[test]
fn move_to_word_end() {
let line_ids = parse::lines("00 111, 2222\n33333").unwrap();
let word_id = arena().read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(grapheme_id, 0, 0);
let arena = Arc::new(RwLock::new(TokenArena::new()));
let line_ids = parse::lines(arena.clone(), "00 111, 2222\n33333").unwrap();
let word_id = arena
.read()
.read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena.read().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(arena.clone(), grapheme_id, 0, 0);
cursor.move_to_word_end(1);
assert_eq!(cursor.grapheme_index(), 1);
......@@ -323,10 +345,14 @@ fn move_to_word_end() {
#[test]
fn move_line() {
let line_ids = parse::lines("000\n11\n2\n33333").unwrap();
let word_id = arena().read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(grapheme_id, 2, 0);
let arena = Arc::new(RwLock::new(TokenArena::new()));
let line_ids = parse::lines(arena.clone(), "000\n11\n2\n33333").unwrap();
let word_id = arena
.read()
.read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena.read().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(arena.clone(), grapheme_id, 2, 0);
cursor.move_line(0);
assert_eq!(cursor.grapheme_index(), 2);
......@@ -375,10 +401,14 @@ fn move_line() {
#[test]
fn move_to_line_end() {
let line_ids = parse::lines("00\n111\n2222").unwrap();
let word_id = arena().read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(grapheme_id, 0, 0);
let arena = Arc::new(RwLock::new(TokenArena::new()));
let line_ids = parse::lines(arena.clone(), "00\n111\n2222").unwrap();
let word_id = arena
.read()
.read(line_ids[0], &|line| line.first_child().unwrap());
let grapheme_id = arena.read().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(arena.clone(), grapheme_id, 0, 0);
cursor.move_to_line_end();
assert_eq!(cursor.grapheme_index(), 1);
......@@ -403,13 +433,18 @@ fn move_to_line_end() {
#[test]
fn move_paragraph() {
let paragraph_ids = parse::paragraphs("000\n111\n2222\n\n333\n444\n\n\n555").unwrap();
let line_id = arena().read(paragraph_ids[0], &|paragraph| {
let arena = Arc::new(RwLock::new(TokenArena::new()));
let paragraph_ids =
parse::paragraphs(arena.clone(), "000\n111\n2222\n\n333\n444\n\n\n555").unwrap();
let line_id = arena.read().read(paragraph_ids[0], &|paragraph| {
paragraph.first_child().unwrap()
});
let word_id = arena().read(line_id, &|line| line.first_child().unwrap());
let grapheme_id = arena().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(grapheme_id, 0, 0);
let word_id = arena
.read()
.read(line_id, &|line| line.first_child().unwrap());
let grapheme_id = arena.read().read(word_id, &|word| word.first_child());
let mut cursor = Cursor::new(arena.clone(), grapheme_id, 0, 0);
cursor.move_paragraph(0);
assert_eq!(cursor.grapheme_index(), 0);
......
......@@ -37,6 +37,7 @@ macro_rules! metadata {
pub struct File {
path: PathBuf,
// TODO: give this struct its own token arena?
}
impl File {
......
......@@ -15,6 +15,8 @@
// You should have received a copy of the GNU General Public License along with
// pbvi. If not, see <https://www.gnu.org/licenses/>.
use std::{mem::replace, sync::Arc};
use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard};
use super::{Token, TokenError, TokenResult};
......@@ -25,12 +27,12 @@ pub struct TokenArena {
}
impl TokenArena {
pub fn new() -> TokenArena {
TokenArena { tokens: Vec::new() }
pub fn new() -> Self {
Self { tokens: Vec::new() }
}
pub fn reserve_id(&mut self) -> usize {
self.tokens.push(RwLock::new(Token::Null));
pub fn reserve_id(&mut self, arena: Arc<RwLock<Self>>) -> usize {
self.tokens.push(RwLock::new(Token::Null { arena }));
self.tokens.len() - 1
}
......@@ -38,7 +40,7 @@ impl TokenArena {
if id >= self.tokens.len() {
Err(TokenError::invalid_id("set"))
} else {
self.tokens.insert(id, RwLock::new(token));
replace(&mut self.tokens[id], RwLock::new(token));
Ok(())
}
}
......
......@@ -15,35 +15,57 @@
// You should have received a copy of the GNU General Public License along with
// pbvi. If not, see <https://www.gnu.org/licenses/>.
use std::fmt::{self, Display, Formatter};
use std::{
fmt::{self, Display, Formatter},
sync::Arc,
};
use super::{arena, TokenId, TokenIterator, TokenType};
use parking_lot::{RwLock, RwLockReadGuard};
use super::{TokenArena, TokenId, TokenIterator, TokenType};
#[derive(Debug)]
pub struct Children {
arena: Arc<RwLock<TokenArena>>,
first: Option<TokenId>,
last: Option<TokenId>,
}
impl Children {
pub fn new(children: &[TokenId], terminates: TokenType) -> Children {
pub fn new(
arena: Arc<RwLock<TokenArena>>,
children: &[TokenId],
terminates: TokenType,
) -> Self {
if let Some(last_id) = children.last() {
arena().mutate(*last_id, &|mut last| {
arena.read().mutate(*last_id, &|mut last| {
last.set_terminates(terminates);
});
}
Children {
Self {
arena,
first: children.first().cloned(),
last: children.last().cloned(),
}
}
pub fn len(&self) -> usize {
self.iter().fold(0, |sum, child_id| {
sum + self.arena().read(child_id, &|child| child.len())
})
}
fn arena(&self) -> RwLockReadGuard<TokenArena> {
self.arena.read()
}
pub fn iter(&self) -> TokenIterator {
if let Some(first_id) = self.first {
arena().read(first_id, &|first| first.iter(self.first, self.last))
self.arena()
.read(first_id, &|first| first.iter(self.first, self.last))
} else {
TokenIterator::default()
TokenIterator::null(self.arena.clone())
}
}
......@@ -57,7 +79,7 @@ impl Children {
pub fn set_terminates(&mut self, terminates: TokenType) {
if let Some(last_id) = self.last {
arena().mutate(last_id, &|mut last| {
self.arena().mutate(last_id, &|mut last| {
last.set_terminates(terminates);
});
}
......@@ -65,7 +87,7 @@ impl Children {
pub fn set_previous_and_cascade(&mut self, previous_id: TokenId) {
if let Some(first_id) = self.first {
arena().mutate(previous_id, &|mut previous| {
self.arena().mutate(previous_id, &|mut previous| {
previous.set_next_and_cascade(first_id);
});
}
......@@ -73,7 +95,7 @@ impl Children {
pub fn set_child_previous_and_cascade(&mut self, previous_id: TokenId) {
if let Some(first_id) = self.first {
arena().mutate(previous_id, &|mut previous| {
self.arena().mutate(previous_id, &|mut previous| {
previous.set_child_next_and_cascade(first_id);
});
}
......@@ -81,7 +103,7 @@ impl Children {
pub fn set_next_and_cascade(&mut self, next_id: TokenId) {
if let Some(last_id) = self.last {
arena().mutate(next_id, &|mut next| {
self.arena().mutate(next_id, &|mut next| {
next.set_previous_and_cascade(last_id);
});
}
......@@ -89,7 +111,7 @@ impl Children {
pub fn set_child_next_and_cascade(&mut self, next_id: TokenId) {
if let Some(last_id) = self.last {
arena().mutate(next_id, &|mut next| {
self.arena().mutate(next_id, &|mut next| {
next.set_child_previous_and_cascade(last_id);
});
}
......@@ -130,7 +152,7 @@ impl Children {
pub fn delete_all(&mut self) {
self.iter().for_each(|child_id| {
arena().mutate(child_id, &|mut child| {
self.arena().mutate(child_id, &|mut child| {
child.delete_self_and_descendants();
});
});
......@@ -176,7 +198,7 @@ impl Children {
pub fn reinstate_all(&mut self) {
self.iter().for_each(|child_id| {
arena().mutate(child_id, &|mut child| {
self.arena().mutate(child_id, &|mut child| {
child.reinstate_self_and_descendants();
});
});
......@@ -210,10 +232,14 @@ impl Children {
}
pub fn replace_all(&mut self, replacement_id: TokenId) {
let tokens = arena();
let replacement = tokens.get(replacement_id).unwrap();
self.first = replacement.first_child();
self.last = replacement.last_child();
let (first, last) = {
let lock = self.arena.clone();
let arena = lock.read();
let replacement = arena.get(replacement_id).unwrap();
(replacement.first_child(), replacement.last_child())
};
self.first = first;
self.last = last;
}
}
......@@ -224,7 +250,7 @@ impl Display for Children {
"{}",
&self
.iter()
.map(|child_id| arena().read(child_id, &|child| format!("{}", *child)))
.map(|child_id| self.arena().read(child_id, &|child| format!("{}", *child)))
.collect::<Vec<String>>()
.concat()
)
......
......@@ -15,10 +15,15 @@
// You should have received a copy of the GNU General Public License along with
// pbvi. If not, see <https://www.gnu.org/licenses/>.
use super::{arena, TokenId};
use std::sync::Arc;
#[derive(Debug, Default)]
use parking_lot::{RwLock, RwLockReadGuard};
use super::{TokenArena, TokenId};
#[derive(Debug)]
pub struct TokenIterator {
arena: Arc<RwLock<TokenArena>>,
current: Option<TokenId>,
first: Option<TokenId>,
last: Option<TokenId>,
......@@ -26,16 +31,31 @@ pub struct TokenIterator {