Skip to content

Add character index to Marker #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kinnison merged 3 commits into from
May 26, 2025
Merged

Add character index to Marker #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ea93576
Marker: Add .character() support
kinnison May 26, 2025
373c20c
marked-yaml: Test .character() some more
kinnison May 26, 2025
803ed68
chore: Update to cache@v4
kinnison May 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/basic.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,12 @@ jobs:
path: ~/.cargo/registry
key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
- name: Cache cargo git trees
uses: actions/cache@v1
uses: actions/cache@v4
with:
path: ~/.cargo/git
key: ${{ runner.os }}-cargo-gits-${{ hashFiles('**/Cargo.lock') }}
- name: Cache cargo build
uses: actions/cache@v1
uses: actions/cache@v4
with:
path: target
key: ${{ runner.os }}-cargo-build-${{ matrix.target }}-${{ steps.rustc.outputs.commit_hash }}-${{ hashFiles('**/Cargo.lock') }}
Expand Down
2 changes: 1 addition & 1 deletion marked-yaml/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ let YAML = "Daniel: Author\nUser: Not Author\n";
let roles: HashMap<Spanned<String>, Spanned<String>> = from_yaml(0, YAML).unwrap();

assert_eq!(roles["Daniel"], "Author");
assert_eq!(roles["User"].span().start().copied(), Some(Marker::new(0, 2, 7)));
assert_eq!(roles["User"].span().start().copied(), Some(Marker::new(0, 21, 2, 7)));
```

You do not have to have all values [`Spanned`], and you can deserialize from an already
Expand Down
32 changes: 19 additions & 13 deletions marked-yaml/src/loader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,11 @@
use LoadError::*;
#[allow(deprecated)]
match self {
TopLevelMustBeMapping(m) => write!(f, "{}: Top level must be a mapping", m),

Check warning on line 129 in marked-yaml/src/loader.rs

View workflow job for this annotation

GitHub Actions / Build (ubuntu-latest, beta) 

variables can be used directly in the `format!` string
TopLevelMustBeSequence(m) => write!(f, "{}: Top level must be a sequence", m),

Check warning on line 130 in marked-yaml/src/loader.rs

View workflow job for this annotation

GitHub Actions / Build (ubuntu-latest, beta) 

variables can be used directly in the `format!` string
UnexpectedAnchor(m) => write!(f, "{}: Unexpected definition of anchor", m),

Check warning on line 131 in marked-yaml/src/loader.rs

View workflow job for this annotation

GitHub Actions / Build (ubuntu-latest, beta) 

variables can be used directly in the `format!` string
MappingKeyMustBeScalar(m) => write!(f, "{}: Keys in mappings must be scalar", m),

Check warning on line 132 in marked-yaml/src/loader.rs

View workflow job for this annotation

GitHub Actions / Build (ubuntu-latest, beta) 

variables can be used directly in the `format!` string
UnexpectedTag(m) => write!(f, "{}: Unexpected use of YAML tag", m),

Check warning on line 133 in marked-yaml/src/loader.rs

View workflow job for this annotation

GitHub Actions / Build (ubuntu-latest, beta) 

variables can be used directly in the `format!` string
DuplicateKey(inner) => {
let DuplicateKeyInner { prev_key, key } = inner.as_ref();
write!(
Expand Down Expand Up @@ -409,7 +409,7 @@
}

fn marker(&self, mark: YamlMarker) -> Marker {
Marker::new(self.source, mark.line(), mark.col() + 1)
Marker::new(self.source, mark.index(), mark.line(), mark.col() + 1)
}

fn finish(mut self) -> Result<Node, LoadError> {
Expand Down Expand Up @@ -538,7 +538,7 @@
let err = parse_yaml(0, "foo");
assert_eq!(
err,
Err(LoadError::TopLevelMustBeMapping(Marker::new(0, 1, 1)))
Err(LoadError::TopLevelMustBeMapping(Marker::new(0, 0, 1, 1)))
);
assert!(format!("{}", err.err().unwrap()).contains("1:1: "));
}
Expand All @@ -547,7 +547,7 @@
fn toplevel_is_sequence() {
assert_eq!(
parse_yaml(0, "[]"),
Err(LoadError::TopLevelMustBeMapping(Marker::new(0, 1, 1)))
Err(LoadError::TopLevelMustBeMapping(Marker::new(0, 0, 1, 1)))
);
}

Expand All @@ -562,8 +562,8 @@
assert_eq!(
err,
Err(LoadError::DuplicateKey(Box::new(DuplicateKeyInner {
prev_key: MarkedScalarNode::new(Span::new_start(Marker::new(0, 1, 1)), "foo"),
key: MarkedScalarNode::new(Span::new_start(Marker::new(0, 1, 11)), "foo")
prev_key: MarkedScalarNode::new(Span::new_start(Marker::new(0, 0, 1, 1)), "foo"),
key: MarkedScalarNode::new(Span::new_start(Marker::new(0, 10, 1, 11)), "foo")
})))
);

Expand All @@ -581,23 +581,26 @@
#[test]
fn unexpected_anchor() {
let err = parse_yaml(0, "&foo {}");
assert_eq!(err, Err(LoadError::UnexpectedAnchor(Marker::new(0, 1, 6))));
assert_eq!(
err,
Err(LoadError::UnexpectedAnchor(Marker::new(0, 5, 1, 6)))
);
assert!(format!("{}", err.err().unwrap()).starts_with("1:6: "));
}

#[test]
fn unexpected_anchor2() {
assert_eq!(
parse_yaml(0, "{bar: &foo []}"),
Err(LoadError::UnexpectedAnchor(Marker::new(0, 1, 12)))
Err(LoadError::UnexpectedAnchor(Marker::new(0, 11, 1, 12)))
);
}

#[test]
fn unexpected_anchor3() {
assert_eq!(
parse_yaml(0, "{bar: &foo susan}"),
Err(LoadError::UnexpectedAnchor(Marker::new(0, 1, 12)))
Err(LoadError::UnexpectedAnchor(Marker::new(0, 11, 1, 12)))
);
}

Expand All @@ -606,7 +609,7 @@
let err = parse_yaml(0, "{? {} : {}}");
assert_eq!(
err,
Err(LoadError::MappingKeyMustBeScalar(Marker::new(0, 1, 4)))
Err(LoadError::MappingKeyMustBeScalar(Marker::new(0, 3, 1, 4)))
);
assert!(format!("{}", err.err().unwrap()).starts_with("1:4: "));
}
Expand All @@ -615,22 +618,25 @@
fn mapping_key_sequence() {
assert_eq!(
parse_yaml(0, "{? [] : {}}"),
Err(LoadError::MappingKeyMustBeScalar(Marker::new(0, 1, 4)))
Err(LoadError::MappingKeyMustBeScalar(Marker::new(0, 3, 1, 4)))
);
}

#[test]
fn unexpected_tag() {
let err = parse_yaml(0, "{foo: !!str bar}");
assert_eq!(err, Err(LoadError::UnexpectedTag(Marker::new(0, 1, 13))));
assert_eq!(
err,
Err(LoadError::UnexpectedTag(Marker::new(0, 12, 1, 13)))
);
assert!(format!("{}", err.err().unwrap()).starts_with("1:13: "));
}

#[test]
fn nested_mapping_key_mapping() {
assert_eq!(
parse_yaml(0, "{foo: {? [] : {}}}"),
Err(LoadError::MappingKeyMustBeScalar(Marker::new(0, 1, 10)))
Err(LoadError::MappingKeyMustBeScalar(Marker::new(0, 9, 1, 10)))
);
}

Expand All @@ -653,7 +659,7 @@
fn toplevel_sequence_wanted_got_mapping() {
assert_eq!(
parse_yaml_with_options(0, "{}", LoaderOptions::default().toplevel_sequence()),
Err(LoadError::TopLevelMustBeSequence(Marker::new(0, 1, 1)))
Err(LoadError::TopLevelMustBeSequence(Marker::new(0, 0, 1, 1)))
);
}

Expand Down
49 changes: 45 additions & 4 deletions marked-yaml/src/spanned_serde.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ impl<T> Spanned<T> {
///
/// ```
/// # use marked_yaml::{Spanned, Span, Marker};
/// # let span = Span::new_start(Marker::new(0,1,2));
/// # let span = Span::new_start(Marker::new(0, 1, 1, 2));
/// let spanned = Spanned::new(span, "Hello World");
/// assert_eq!(spanned.span(), &span);
/// ```
Expand Down Expand Up @@ -132,18 +132,22 @@ impl Borrow<str> for Spanned<&'_ str> {

const SPANNED_TYPE: &str = "$___::marked_data::serde::Spanned<T>";
const SPANNED_SPAN_START_SOURCE: &str = "$___::marked_data::serde::Spanned<T>::span_start_source";
const SPANNED_SPAN_START_CHARACTER: &str = "$___::marked_data::serde::Spanned<T>::span_start_char";
const SPANNED_SPAN_START_LINE: &str = "$___::marked_data::serde::Spanned<T>::span_start_line";
const SPANNED_SPAN_START_COLUMN: &str = "$___::marked_data::serde::Spanned<T>::span_start_column";
const SPANNED_SPAN_END_SOURCE: &str = "$___::marked_data::serde::Spanned<T>::span_end_source";
const SPANNED_SPAN_END_CHARACTER: &str = "$___::marked_data::serde::Spanned<T>::span_end_char";
const SPANNED_SPAN_END_LINE: &str = "$___::marked_data::serde::Spanned<T>::span_end_line";
const SPANNED_SPAN_END_COLUMN: &str = "$___::marked_data::serde::Spanned<T>::span_end_column";
const SPANNED_INNER: &str = "$___::marked_data::serde::Spanned<T>::inner";

const SPANNED_FIELDS: [&str; 7] = [
const SPANNED_FIELDS: [&str; 9] = [
SPANNED_SPAN_START_SOURCE,
SPANNED_SPAN_START_CHARACTER,
SPANNED_SPAN_START_LINE,
SPANNED_SPAN_START_COLUMN,
SPANNED_SPAN_END_SOURCE,
SPANNED_SPAN_END_CHARACTER,
SPANNED_SPAN_END_LINE,
SPANNED_SPAN_END_COLUMN,
SPANNED_INNER,
Expand Down Expand Up @@ -177,6 +181,12 @@ where

let span_start = if key == Some(SPANNED_SPAN_START_SOURCE) {
let source: usize = visitor.next_value()?;
if visitor.next_key()? != Some(SPANNED_SPAN_START_CHARACTER) {
return Err(serde::de::Error::custom(
"marked node span start character missing",
));
}
let character: usize = visitor.next_value()?;
if visitor.next_key()? != Some(SPANNED_SPAN_START_LINE) {
return Err(serde::de::Error::custom(
"marked node span start line missing",
Expand All @@ -190,13 +200,19 @@ where
}
let column: usize = visitor.next_value()?;
key = visitor.next_key()?;
Some(Marker::new(source, line, column))
Some(Marker::new(source, character, line, column))
} else {
None
};

let span_end = if key == Some(SPANNED_SPAN_END_SOURCE) {
let source: usize = visitor.next_value()?;
if visitor.next_key()? != Some(SPANNED_SPAN_END_CHARACTER) {
return Err(serde::de::Error::custom(
"marked node span end character missing",
));
}
let character: usize = visitor.next_value()?;
if visitor.next_key()? != Some(SPANNED_SPAN_END_LINE) {
return Err(serde::de::Error::custom(
"marked node span end line missing",
Expand All @@ -210,7 +226,7 @@ where
}
let column: usize = visitor.next_value()?;
key = visitor.next_key()?;
Some(Marker::new(source, line, column))
Some(Marker::new(source, character, line, column))
} else {
None
};
Expand Down Expand Up @@ -834,9 +850,11 @@ struct SpannedDeserializer<'de, T> {

enum SpannedDeserializerState {
SendStartSource,
SendStartCharacter,
SendStartLine,
SendStartColumn,
SendEndSource,
SendEndCharacter,
SendEndLine,
SendEndColumn,
SendValue,
Expand Down Expand Up @@ -872,9 +890,11 @@ where
{
let key = match self.state {
SpannedDeserializerState::SendStartSource => SPANNED_SPAN_START_SOURCE,
SpannedDeserializerState::SendStartCharacter => SPANNED_SPAN_START_CHARACTER,
SpannedDeserializerState::SendStartLine => SPANNED_SPAN_START_LINE,
SpannedDeserializerState::SendStartColumn => SPANNED_SPAN_START_COLUMN,
SpannedDeserializerState::SendEndSource => SPANNED_SPAN_END_SOURCE,
SpannedDeserializerState::SendEndCharacter => SPANNED_SPAN_END_CHARACTER,
SpannedDeserializerState::SendEndLine => SPANNED_SPAN_END_LINE,
SpannedDeserializerState::SendEndColumn => SPANNED_SPAN_END_COLUMN,
SpannedDeserializerState::SendValue => SPANNED_INNER,
Expand All @@ -896,6 +916,16 @@ where
.start()
.expect("Span missing start")
.source();
self.state = SpannedDeserializerState::SendStartCharacter;
seed.deserialize(v.into_deserializer())
}
SpannedDeserializerState::SendStartCharacter => {
let v = self
.node
.mark_span()
.start()
.expect("Span missing start")
.character();
self.state = SpannedDeserializerState::SendStartLine;
seed.deserialize(v.into_deserializer())
}
Expand Down Expand Up @@ -930,6 +960,16 @@ where
.end()
.expect("Span missing end")
.source();
self.state = SpannedDeserializerState::SendEndCharacter;
seed.deserialize(v.into_deserializer())
}
SpannedDeserializerState::SendEndCharacter => {
let v = self
.node
.mark_span()
.end()
.expect("Span missing end")
.character();
self.state = SpannedDeserializerState::SendEndLine;
seed.deserialize(v.into_deserializer())
}
Expand Down Expand Up @@ -1504,6 +1544,7 @@ shouting: TRUE
Error::IntegerParseFailure(_e, s) => {
let start = s.start().unwrap();
assert_eq!(start.source(), 0);
assert_eq!(start.character(), 93);
assert_eq!(start.line(), 4);
assert_eq!(start.column(), 21);
}
Expand Down
Loading
Loading