First Commit
This commit is contained in:
Fabian Stamm
461
crates/libjrpc/src/ir.rs
Normal file
461
crates/libjrpc/src/ir.rs
Normal file
@ -0,0 +1,461 @@
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
error::Error,
|
||||
fmt::Display,
|
||||
hash::{Hash, Hasher},
|
||||
};
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use crate::parser::{
|
||||
EnumStatement, Node, ParserPosition, RootNode, ServiceStatement, TypeStatement,
|
||||
};
|
||||
|
||||
static BUILT_INS: [&str; 4] = ["int", "float", "string", "boolean"];
|
||||
|
||||
pub trait Definition {
|
||||
fn get_position(&self) -> ParserPosition;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IR {
|
||||
options: HashMap<String, String>,
|
||||
steps: Vec<Step>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Step {
|
||||
Type(TypeDefinition),
|
||||
Enum(EnumDefinition),
|
||||
Service(ServiceDefinition),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum Type {
|
||||
Int,
|
||||
Float,
|
||||
String,
|
||||
Bool,
|
||||
Custom(String),
|
||||
}
|
||||
impl Hash for Type {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
match self {
|
||||
Type::Int => "int".hash(state),
|
||||
Type::Float => "float".hash(state),
|
||||
Type::String => "string".hash(state),
|
||||
Type::Bool => "bool".hash(state),
|
||||
Type::Custom(name) => name.hash(state),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TypeDefinition {
|
||||
name: String,
|
||||
depends: HashSet<Type>,
|
||||
fields: Vec<Field>,
|
||||
position: ParserPosition,
|
||||
}
|
||||
|
||||
impl Definition for TypeDefinition {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Field {
|
||||
name: String,
|
||||
typ: Type,
|
||||
array: bool,
|
||||
optional: bool,
|
||||
map: Option<Type>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EnumDefinition {
|
||||
name: String,
|
||||
values: Vec<EnumField>,
|
||||
position: ParserPosition,
|
||||
}
|
||||
|
||||
impl Definition for EnumDefinition {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EnumField {
|
||||
name: String,
|
||||
value: i32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ServiceDefinition {
|
||||
name: String,
|
||||
depends: HashSet<Type>,
|
||||
methods: Vec<Method>,
|
||||
position: ParserPosition,
|
||||
}
|
||||
|
||||
impl Definition for ServiceDefinition {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Method {
|
||||
name: String,
|
||||
inputs: Vec<MethodInput>,
|
||||
output: Option<MethodOutput>,
|
||||
decorators: MethodDecorators,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MethodInput {
|
||||
name: String,
|
||||
typ: Type,
|
||||
array: bool,
|
||||
optional: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MethodOutput {
|
||||
typ: Type,
|
||||
array: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MethodDecorators {
|
||||
description: Option<String>,
|
||||
parameter_descriptions: HashMap<String, String>,
|
||||
return_description: Option<String>,
|
||||
}
|
||||
|
||||
fn typename_to_type(name: &str) -> Type {
|
||||
match name {
|
||||
"int" => Type::Int,
|
||||
"float" => Type::Float,
|
||||
"string" => Type::String,
|
||||
"boolean" => Type::Bool,
|
||||
"bool" => Type::Bool,
|
||||
_ => Type::Custom(name.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_type(stmt: &TypeStatement) -> Result<TypeDefinition> {
|
||||
let mut typedef = TypeDefinition {
|
||||
position: stmt.position.clone(),
|
||||
name: stmt.name.clone(),
|
||||
depends: HashSet::new(),
|
||||
fields: Vec::new(),
|
||||
};
|
||||
|
||||
for field in &stmt.fields {
|
||||
let typ = typename_to_type(&field.fieldtype);
|
||||
typedef.depends.insert(typ.clone());
|
||||
|
||||
if let Some(maptype) = &field.map {
|
||||
if maptype != "string" && maptype != "int" {
|
||||
return Err(IRError::new("Map type must be string or int", field).into());
|
||||
}
|
||||
}
|
||||
|
||||
typedef.fields.push(Field {
|
||||
name: field.name.clone(),
|
||||
typ: typ.clone(),
|
||||
array: field.array,
|
||||
optional: field.optional,
|
||||
map: field.map.as_ref().map(|s| typename_to_type(s)),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(typedef)
|
||||
}
|
||||
|
||||
fn build_enum(stmt: &EnumStatement) -> Result<EnumDefinition> {
|
||||
let mut enumdef = EnumDefinition {
|
||||
position: stmt.position.clone(),
|
||||
name: stmt.name.clone(),
|
||||
values: Vec::new(),
|
||||
};
|
||||
|
||||
let mut last = -1 as i32;
|
||||
for field in &stmt.values {
|
||||
let value = if let Some(value) = field.value {
|
||||
if value > std::i32::MAX as i64 {
|
||||
return Err(IRError::new("Enum value too large", field).into());
|
||||
}
|
||||
let value = value as i32;
|
||||
if value <= last {
|
||||
return Err(IRError::new("Enum values must be increasing", field).into());
|
||||
}
|
||||
last = value;
|
||||
value
|
||||
} else {
|
||||
last = last + 1;
|
||||
last
|
||||
};
|
||||
enumdef.values.push(EnumField {
|
||||
name: field.name.clone(),
|
||||
value,
|
||||
});
|
||||
}
|
||||
Ok(enumdef)
|
||||
}
|
||||
|
||||
fn build_service(stmt: &ServiceStatement) -> Result<ServiceDefinition> {
|
||||
let mut servdef = ServiceDefinition {
|
||||
position: stmt.position.clone(),
|
||||
name: stmt.name.clone(),
|
||||
depends: HashSet::new(),
|
||||
methods: Vec::new(),
|
||||
};
|
||||
|
||||
for method in &stmt.methods {
|
||||
let mut methoddef = Method {
|
||||
name: method.name.clone(),
|
||||
inputs: Vec::new(),
|
||||
output: method.return_type.as_ref().map(|rt| {
|
||||
let typ = typename_to_type(&rt.fieldtype);
|
||||
servdef.depends.insert(typ.clone());
|
||||
MethodOutput {
|
||||
typ,
|
||||
array: rt.array,
|
||||
}
|
||||
}),
|
||||
decorators: MethodDecorators {
|
||||
description: None,
|
||||
parameter_descriptions: HashMap::new(),
|
||||
return_description: None,
|
||||
},
|
||||
};
|
||||
|
||||
let mut optional_starts = false;
|
||||
for inp in &method.inputs {
|
||||
let typ = typename_to_type(&inp.fieldtype);
|
||||
servdef.depends.insert(typ.clone());
|
||||
|
||||
if optional_starts && !inp.optional {
|
||||
return Err(
|
||||
IRError::new("Optional fields must come after required fields", inp).into(),
|
||||
);
|
||||
}
|
||||
|
||||
if inp.optional {
|
||||
optional_starts = true;
|
||||
}
|
||||
|
||||
methoddef.inputs.push(MethodInput {
|
||||
name: inp.name.clone(),
|
||||
typ,
|
||||
array: inp.array,
|
||||
optional: inp.optional,
|
||||
});
|
||||
}
|
||||
|
||||
for decorator in &method.decorators {
|
||||
match decorator.name.as_str() {
|
||||
"Description" => {
|
||||
if methoddef.decorators.description.is_some() {
|
||||
return Err(IRError::new("Duplicate description", decorator).into());
|
||||
}
|
||||
|
||||
if decorator.args.len() != 1 {
|
||||
return Err(IRError::new(
|
||||
"Description must have exactly one argument",
|
||||
decorator,
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
methoddef.decorators.description = Some(decorator.args[0].clone());
|
||||
}
|
||||
"Returns" => {
|
||||
if methoddef.decorators.return_description.is_some() {
|
||||
return Err(IRError::new("Duplicate return description", decorator).into());
|
||||
}
|
||||
|
||||
if decorator.args.len() != 1 {
|
||||
return Err(IRError::new(
|
||||
"Returns must have exactly one argument",
|
||||
decorator,
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
methoddef.decorators.return_description = Some(decorator.args[0].clone());
|
||||
}
|
||||
"Param" => {
|
||||
if decorator.args.len() != 2 {
|
||||
return Err(IRError::new(
|
||||
"Param must have exactly two arguments",
|
||||
decorator,
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
let name = decorator.args[0].clone();
|
||||
let description = decorator.args[1].clone();
|
||||
|
||||
if methoddef
|
||||
.decorators
|
||||
.parameter_descriptions
|
||||
.contains_key(&name)
|
||||
{
|
||||
return Err(
|
||||
IRError::new("Duplicate parameter description", decorator).into()
|
||||
);
|
||||
}
|
||||
|
||||
if methoddef.inputs.iter().find(|i| i.name == name).is_none() {
|
||||
return Err(IRError::new("Parameter not found", decorator).into());
|
||||
}
|
||||
|
||||
methoddef
|
||||
.decorators
|
||||
.parameter_descriptions
|
||||
.insert(name, description);
|
||||
}
|
||||
_ => {
|
||||
return Err(IRError::new("Unknown decorator", decorator).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
servdef.methods.push(methoddef);
|
||||
}
|
||||
|
||||
Ok(servdef)
|
||||
}
|
||||
|
||||
pub fn build_ir(root: &Vec<RootNode>) -> Result<IR> {
|
||||
let mut options = HashMap::<String, String>::new();
|
||||
let mut steps = Vec::new();
|
||||
|
||||
for node in root {
|
||||
match node {
|
||||
RootNode::Type(stmt) => steps.push(Step::Type(build_type(stmt)?)),
|
||||
RootNode::Enum(stmt) => steps.push(Step::Enum(build_enum(stmt)?)),
|
||||
RootNode::Service(stmt) => steps.push(Step::Service(build_service(stmt)?)),
|
||||
RootNode::Define(stmt) => {
|
||||
if options.contains_key(&stmt.key) {
|
||||
return Err(IRError::new("Duplicate define", stmt).into());
|
||||
}
|
||||
|
||||
if (stmt.key == "use_messagepack" || stmt.key == "allow_bytes")
|
||||
&& stmt.value == "true"
|
||||
{
|
||||
options.insert("allow_bytes".to_owned(), "true".to_owned());
|
||||
}
|
||||
options.insert(stmt.key.clone(), stmt.value.clone());
|
||||
}
|
||||
RootNode::Import(_) => {
|
||||
panic!("Import not supported at this stage!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut all_types = HashSet::<String>::new();
|
||||
let mut serv_types = HashSet::<String>::new();
|
||||
|
||||
for bi in &BUILT_INS {
|
||||
all_types.insert(bi.to_string());
|
||||
}
|
||||
|
||||
for step in &steps {
|
||||
match step {
|
||||
Step::Type(typedef) => {
|
||||
if all_types.contains(&typedef.name) {
|
||||
return Err(IRError::new_from_def("Duplicate type", typedef).into());
|
||||
}
|
||||
|
||||
all_types.insert(typedef.name.clone());
|
||||
}
|
||||
Step::Enum(enumdef) => {
|
||||
if all_types.contains(&enumdef.name) {
|
||||
return Err(IRError::new_from_def("Duplicate type", enumdef).into());
|
||||
}
|
||||
|
||||
all_types.insert(enumdef.name.clone());
|
||||
}
|
||||
Step::Service(servdef) => {
|
||||
if serv_types.contains(&servdef.name) {
|
||||
return Err(IRError::new_from_def("Duplicate type", servdef).into());
|
||||
}
|
||||
|
||||
serv_types.insert(servdef.name.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify dependencies
|
||||
|
||||
for step in &steps {
|
||||
match step {
|
||||
Step::Type(typedef) => {
|
||||
for dep in &typedef.depends {
|
||||
if let Type::Custom(dep) = dep {
|
||||
if !all_types.contains(dep) {
|
||||
return Err(IRError::new_from_def(
|
||||
&format!("Type {} depends on unknown type {}", typedef.name, dep),
|
||||
typedef,
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Step::Service(servdef) => {
|
||||
for dep in &servdef.depends {
|
||||
if let Type::Custom(dep) = dep {
|
||||
if !all_types.contains(dep) {
|
||||
return Err(IRError::new_from_def(
|
||||
&format!(
|
||||
"Service {} depends on unknown type {}",
|
||||
servdef.name, dep
|
||||
),
|
||||
servdef,
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(IR { options, steps })
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IRError {
|
||||
pub message: String,
|
||||
pub position: ParserPosition,
|
||||
}
|
||||
|
||||
impl IRError {
|
||||
fn new(msg: &str, node: &impl Node) -> IRError {
|
||||
IRError {
|
||||
message: format!("{}: {}", msg, node.get_name()),
|
||||
position: node.get_position(),
|
||||
}
|
||||
}
|
||||
|
||||
fn new_from_def(msg: &str, def: &impl Definition) -> IRError {
|
||||
IRError {
|
||||
message: msg.to_string(),
|
||||
position: def.get_position(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Error for IRError {}
|
||||
impl Display for IRError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "ParserError: {} at {:?}", self.message, self.position)
|
||||
}
|
||||
}
|
||||
39
crates/libjrpc/src/lib.rs
Normal file
39
crates/libjrpc/src/lib.rs
Normal file
@ -0,0 +1,39 @@
|
||||
mod ir;
|
||||
mod parser;
|
||||
mod process;
|
||||
mod shared;
|
||||
mod tokenizer;
|
||||
|
||||
pub use tokenizer::{tokenize, Token, TokenError, TokenPosition, TokenType};
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(test)]
|
||||
#[ctor::ctor]
|
||||
fn init() {
|
||||
env_logger::init();
|
||||
}
|
||||
|
||||
#[test]
|
||||
pub fn parse_jrpc() {
|
||||
let tokens = crate::tokenizer::tokenize(
|
||||
Arc::new("../test.jrpc".to_owned()),
|
||||
include_str!("../test.jrpc").to_owned(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let parsed = crate::parser::Parser::new(tokens).parse().unwrap();
|
||||
// println!("Parsed: {:?}", parsed);
|
||||
|
||||
let ir = crate::ir::build_ir(&parsed).unwrap();
|
||||
println!("IR: {:?}", ir);
|
||||
|
||||
// let result = crate::JRPCParser::parse(crate::Rule::root, );
|
||||
// match result {
|
||||
// Ok(result) => println!("{:?}", result),
|
||||
// Err(err) => println!("{:?}", err),
|
||||
// }
|
||||
}
|
||||
}
|
||||
683
crates/libjrpc/src/parser.rs
Normal file
683
crates/libjrpc/src/parser.rs
Normal file
@ -0,0 +1,683 @@
|
||||
use anyhow::Result;
|
||||
use log::{debug, trace};
|
||||
use std::{collections::HashMap, error::Error, fmt::Display, sync::Arc};
|
||||
|
||||
use crate::{Token, TokenPosition, TokenType};
|
||||
|
||||
pub type PResult<T> = Result<T, ParserError>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ParserPosition {
|
||||
path: Arc<String>,
|
||||
position: usize,
|
||||
token_positions: Vec<TokenPosition>,
|
||||
}
|
||||
|
||||
impl ParserPosition {
|
||||
pub fn from_token(token: &Token) -> Self {
|
||||
Self {
|
||||
path: token.2.path.clone(),
|
||||
position: token.2.start,
|
||||
token_positions: vec![token.2.clone()],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Node {
|
||||
fn get_position(&self) -> ParserPosition;
|
||||
fn get_name(&self) -> String {
|
||||
String::from("")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum RootNode {
|
||||
Define(DefineStatement),
|
||||
Import(ImportStatement),
|
||||
Service(ServiceStatement),
|
||||
Type(TypeStatement),
|
||||
Enum(EnumStatement),
|
||||
}
|
||||
|
||||
impl Node for RootNode {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
match self {
|
||||
RootNode::Define(stmt) => stmt.get_position(),
|
||||
RootNode::Import(stmt) => stmt.get_position(),
|
||||
RootNode::Service(stmt) => stmt.get_position(),
|
||||
RootNode::Type(stmt) => stmt.get_position(),
|
||||
RootNode::Enum(stmt) => stmt.get_position(),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
match self {
|
||||
RootNode::Define(_) => String::from("define"),
|
||||
RootNode::Import(_) => String::from("import"),
|
||||
RootNode::Service(_) => String::from("service"),
|
||||
RootNode::Type(_) => String::from("type"),
|
||||
RootNode::Enum(_) => String::from("enum"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DefineStatement {
|
||||
pub position: ParserPosition,
|
||||
pub key: String,
|
||||
pub value: String,
|
||||
}
|
||||
impl Node for DefineStatement {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
fn get_name(&self) -> String {
|
||||
format!("define {}", self.key)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ImportStatement {
|
||||
pub position: ParserPosition,
|
||||
pub path: String,
|
||||
}
|
||||
impl Node for ImportStatement {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
format!("import {}", self.path)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TypeStatement {
|
||||
pub position: ParserPosition,
|
||||
pub name: String,
|
||||
pub fields: Vec<TypeFieldNode>,
|
||||
}
|
||||
impl Node for TypeStatement {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
format!("type {}", self.name)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TypeFieldNode {
|
||||
pub position: ParserPosition,
|
||||
pub name: String,
|
||||
pub optional: bool,
|
||||
pub fieldtype: String,
|
||||
pub array: bool,
|
||||
pub map: Option<String>,
|
||||
}
|
||||
impl Node for TypeFieldNode {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
format!("type_field {}", self.name)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EnumStatement {
|
||||
pub position: ParserPosition,
|
||||
pub name: String,
|
||||
pub values: Vec<EnumValueNode>,
|
||||
}
|
||||
impl Node for EnumStatement {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
format!("enum {}", self.name)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EnumValueNode {
|
||||
pub position: ParserPosition,
|
||||
pub name: String,
|
||||
pub value: Option<i64>,
|
||||
}
|
||||
impl Node for EnumValueNode {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
format!("enum_field {}", self.name)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ServiceStatement {
|
||||
pub position: ParserPosition,
|
||||
pub name: String,
|
||||
pub methods: Vec<ServiceMethodNode>,
|
||||
}
|
||||
impl Node for ServiceStatement {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
format!("service {}", self.name)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ServiceMethodNode {
|
||||
pub position: ParserPosition,
|
||||
pub name: String,
|
||||
pub inputs: Vec<ServiceMethodInputNode>,
|
||||
pub return_type: Option<ServiceMethodReturnNode>,
|
||||
pub decorators: Vec<Decorator>,
|
||||
}
|
||||
impl Node for ServiceMethodNode {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
format!("method {}", self.name)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Decorator {
|
||||
pub position: ParserPosition,
|
||||
pub name: String,
|
||||
pub args: Vec<String>,
|
||||
}
|
||||
impl Node for Decorator {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
format!("decorator {}", self.name)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ServiceMethodInputNode {
|
||||
pub position: ParserPosition,
|
||||
pub name: String,
|
||||
pub fieldtype: String,
|
||||
pub optional: bool,
|
||||
pub array: bool,
|
||||
}
|
||||
impl Node for ServiceMethodInputNode {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
|
||||
fn get_name(&self) -> String {
|
||||
format!("method_arg {}", self.name)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ServiceMethodReturnNode {
|
||||
pub position: ParserPosition,
|
||||
pub fieldtype: String,
|
||||
pub array: bool,
|
||||
}
|
||||
impl Node for ServiceMethodReturnNode {
|
||||
fn get_position(&self) -> ParserPosition {
|
||||
self.position.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Parser {
|
||||
tokens: Vec<Token>,
|
||||
position: usize,
|
||||
}
|
||||
|
||||
impl Parser {
|
||||
pub fn new(tokens: Vec<Token>) -> Parser {
|
||||
// Remove all comments and spaces at this stage
|
||||
let tokens = tokens
|
||||
.into_iter()
|
||||
.filter(|e| e.0 != TokenType::Comment && e.0 != TokenType::Space)
|
||||
.collect();
|
||||
Parser {
|
||||
tokens,
|
||||
position: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn has_current_token(&self) -> bool {
|
||||
self.position < self.tokens.len()
|
||||
}
|
||||
|
||||
fn current_token(&mut self) -> &Token {
|
||||
&self.tokens[self.position]
|
||||
}
|
||||
|
||||
fn assert_types(&mut self, token_types: &[TokenType]) -> PResult<()> {
|
||||
if token_types.iter().any(|t| *t == self.current_token().0) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(ParserError::new(
|
||||
&format!("Unexpected token. Expected on of {:?}", token_types),
|
||||
self.current_token(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn eat_token(&mut self) -> PResult<(String, ParserPosition)> {
|
||||
debug!("Parser::eat_token()");
|
||||
let pos = ParserPosition::from_token(self.current_token());
|
||||
let value = self.current_token().1.clone();
|
||||
self.position += 1;
|
||||
Ok((value, pos))
|
||||
}
|
||||
|
||||
fn eat_token_value(&mut self, value: &str) -> PResult<(String, ParserPosition)> {
|
||||
debug!("Parser::eat_token_value({})", value);
|
||||
let (val, pos) = self.eat_token()?;
|
||||
|
||||
if val != value {
|
||||
return Err(ParserError::new(
|
||||
&format!("Expected token of value {}", value),
|
||||
self.current_token(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok((val, pos))
|
||||
}
|
||||
|
||||
fn eat_token_type(&mut self, token_type: TokenType) -> PResult<(String, ParserPosition)> {
|
||||
debug!("Parser::eat_token_type({:?})", token_type);
|
||||
if self.current_token().0 != token_type {
|
||||
return Err(ParserError::new(
|
||||
&format!("Expected token of type {:?}", token_type),
|
||||
self.current_token(),
|
||||
));
|
||||
}
|
||||
|
||||
self.eat_token()
|
||||
}
|
||||
|
||||
fn eat_text(&mut self) -> PResult<(String, ParserPosition)> {
|
||||
debug!("Parser::eat_text()");
|
||||
self.eat_token_type(TokenType::Text)
|
||||
}
|
||||
|
||||
fn eat_text_with_keywords(&mut self) -> PResult<(String, ParserPosition)> {
|
||||
debug!("Parser::eat_text_with_keywords()");
|
||||
self.assert_types(&[TokenType::Text, TokenType::Keyword])?;
|
||||
self.eat_token()
|
||||
}
|
||||
|
||||
fn eat_string_or_text_as_raw(&mut self) -> PResult<String> {
|
||||
debug!("Parser::eat_string_or_text_as_raw()");
|
||||
self.assert_types(&[TokenType::String, TokenType::Text])?;
|
||||
let token = self.current_token();
|
||||
if token.0 == TokenType::String {
|
||||
let (value, _) = self.eat_token()?;
|
||||
let count = value.chars().count();
|
||||
Ok(value.chars().skip(1).take(count - 2).collect())
|
||||
} else {
|
||||
let (value, _) = self.eat_token()?;
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn eat_number(&mut self) -> PResult<i64> {
|
||||
debug!("Parser::eat_number()");
|
||||
self.assert_types(&[TokenType::Number])?;
|
||||
let (value, _) = self.eat_token()?;
|
||||
value
|
||||
.parse()
|
||||
.map_err(|_| ParserError::new("Invalid number", self.current_token()))
|
||||
}
|
||||
|
||||
fn parse_define(&mut self) -> PResult<DefineStatement> {
|
||||
debug!("Parser::parse_define()");
|
||||
let (_, position) = self.eat_token_value("define")?;
|
||||
|
||||
let (key, _) = self.eat_token()?;
|
||||
trace!("Parser::parse_define()::key = {}", key);
|
||||
|
||||
let value = self.eat_string_or_text_as_raw()?;
|
||||
|
||||
self.eat_token_type(TokenType::Semicolon)?;
|
||||
|
||||
Ok(DefineStatement {
|
||||
position,
|
||||
key,
|
||||
value,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_import(&mut self) -> PResult<ImportStatement> {
|
||||
debug!("Parser::parse_import()");
|
||||
let (_, pos) = self.eat_token_value("import")?;
|
||||
let path = self.eat_string_or_text_as_raw()?;
|
||||
|
||||
self.eat_token_type(TokenType::Semicolon)?;
|
||||
|
||||
Ok(ImportStatement {
|
||||
position: pos,
|
||||
path,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_enum_value(&mut self) -> PResult<EnumValueNode> {
|
||||
debug!("Parser::parse_enum_value()");
|
||||
let (name, pos) = self.eat_token()?;
|
||||
let value = if self.current_token().0 == TokenType::Equals {
|
||||
self.eat_token()?;
|
||||
Some(self.eat_number()?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(EnumValueNode {
|
||||
position: pos,
|
||||
name,
|
||||
value,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_enum(&mut self) -> PResult<EnumStatement> {
|
||||
debug!("Parser::parse_enum()");
|
||||
let (_, pos) = self.eat_token_value("enum")?;
|
||||
let (name, _) = self.eat_token()?;
|
||||
trace!("Parser::parse_enum()::name = {}", name);
|
||||
|
||||
self.eat_token_type(TokenType::CurlyOpen)?;
|
||||
let mut values: Vec<EnumValueNode> = Vec::new();
|
||||
|
||||
while self.current_token().0 == TokenType::Text {
|
||||
values.push(self.parse_enum_value()?);
|
||||
if self.current_token().0 == TokenType::Comma {
|
||||
//TODO: Maybe use a next flag or break or something with the commas
|
||||
self.eat_token()?;
|
||||
}
|
||||
}
|
||||
|
||||
self.eat_token_type(TokenType::CurlyClose)?;
|
||||
|
||||
Ok(EnumStatement {
|
||||
position: pos,
|
||||
name,
|
||||
values,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_type_field(&mut self) -> PResult<TypeFieldNode> {
|
||||
debug!("Parser::parse_type_field()");
|
||||
let (name, pos) = self.eat_text_with_keywords()?;
|
||||
trace!("Parser::parse_type_field()::name = {}", name);
|
||||
|
||||
let mut optional = false;
|
||||
let mut array = false;
|
||||
let mut map = None;
|
||||
|
||||
if self.current_token().0 == TokenType::Questionmark {
|
||||
optional = true;
|
||||
self.eat_token()?;
|
||||
}
|
||||
|
||||
_ = self.eat_token_type(TokenType::Colon);
|
||||
let fieldtype = if self.current_token().0 == TokenType::CurlyOpen {
|
||||
self.eat_token()?;
|
||||
let (key_type, _) = self.eat_text()?;
|
||||
self.eat_token_type(TokenType::Comma)?;
|
||||
let (value_type, _) = self.eat_text()?;
|
||||
self.eat_token_type(TokenType::CurlyClose)?;
|
||||
map = Some(key_type);
|
||||
value_type
|
||||
} else {
|
||||
let (type_name, _) = self.eat_text()?;
|
||||
if self.current_token().0 == TokenType::Array {
|
||||
array = true;
|
||||
self.eat_token()?;
|
||||
}
|
||||
|
||||
type_name
|
||||
};
|
||||
|
||||
self.eat_token_type(TokenType::Semicolon)?;
|
||||
|
||||
Ok(TypeFieldNode {
|
||||
position: pos,
|
||||
name,
|
||||
optional,
|
||||
fieldtype,
|
||||
map,
|
||||
array,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_type(&mut self) -> PResult<TypeStatement> {
|
||||
debug!("Parser::parse_type()");
|
||||
let (_, pos) = self.eat_token_value("type")?;
|
||||
let (name, _) = self.eat_text()?;
|
||||
trace!("Parser::prase_type()::name = {}", name);
|
||||
self.eat_token_type(TokenType::CurlyOpen)?;
|
||||
|
||||
let mut fields = Vec::new();
|
||||
while self.current_token().0 == TokenType::Text
|
||||
|| self.current_token().0 == TokenType::Keyword
|
||||
{
|
||||
fields.push(self.parse_type_field()?);
|
||||
}
|
||||
|
||||
self.eat_token_type(TokenType::CurlyClose)?;
|
||||
|
||||
Ok(TypeStatement {
|
||||
position: pos,
|
||||
name,
|
||||
fields,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_decorator(&mut self) -> PResult<Decorator> {
|
||||
debug!("Parser::parse_decorator()");
|
||||
let (_, position) = self.eat_token_type(TokenType::At)?;
|
||||
let (decorator, _) = self.eat_text()?;
|
||||
trace!("Parser::parse_decorator()::name = {}", decorator);
|
||||
|
||||
self.eat_token_type(TokenType::BracketOpen)?;
|
||||
|
||||
let mut args = Vec::new();
|
||||
let mut first = true;
|
||||
while self.current_token().0 != TokenType::BracketClose {
|
||||
if first {
|
||||
first = false
|
||||
} else {
|
||||
self.eat_token_type(TokenType::Comma)?;
|
||||
}
|
||||
|
||||
args.push(self.eat_string_or_text_as_raw()?);
|
||||
}
|
||||
self.eat_token_type(TokenType::BracketClose)?;
|
||||
|
||||
Ok(Decorator {
|
||||
name: decorator,
|
||||
args,
|
||||
position,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_method(
|
||||
&mut self,
|
||||
decorators: Vec<Decorator>,
|
||||
notification: bool,
|
||||
) -> PResult<ServiceMethodNode> {
|
||||
debug!(
|
||||
"Parser::parse_method({}, {})",
|
||||
decorators.len(),
|
||||
notification
|
||||
);
|
||||
let (name, pos) = self.eat_text()?;
|
||||
trace!("Parser::parse_method()::name = {}", name);
|
||||
|
||||
self.eat_token_type(TokenType::BracketOpen)?;
|
||||
|
||||
let mut inputs = Vec::new();
|
||||
|
||||
while self.current_token().0 != TokenType::BracketClose {
|
||||
let (name, position) = self.eat_text_with_keywords()?;
|
||||
|
||||
let mut optional = false;
|
||||
if self.current_token().0 == TokenType::Questionmark {
|
||||
optional = true;
|
||||
self.eat_token()?;
|
||||
}
|
||||
|
||||
self.eat_token_type(TokenType::Colon)?;
|
||||
let (fieldtype, _) = self.eat_text()?;
|
||||
let mut array = false;
|
||||
if self.current_token().0 == TokenType::Array {
|
||||
array = true;
|
||||
self.eat_token()?;
|
||||
}
|
||||
inputs.push(ServiceMethodInputNode {
|
||||
name,
|
||||
fieldtype,
|
||||
array,
|
||||
optional,
|
||||
position,
|
||||
});
|
||||
|
||||
if self.current_token().0 == TokenType::Comma {
|
||||
self.eat_token()?;
|
||||
}
|
||||
|
||||
trace!(
|
||||
"Parser::parse_method()::params_next_token: {}",
|
||||
self.current_token().1
|
||||
);
|
||||
}
|
||||
|
||||
self.eat_token_type(TokenType::BracketClose)?;
|
||||
|
||||
let mut return_type = None;
|
||||
if !notification {
|
||||
self.eat_token_type(TokenType::Colon)?;
|
||||
|
||||
let (fieldtype, position) = self.eat_text()?;
|
||||
let mut array = false;
|
||||
if self.current_token().0 == TokenType::Array {
|
||||
array = true;
|
||||
self.eat_token()?;
|
||||
}
|
||||
|
||||
return_type = Some(ServiceMethodReturnNode {
|
||||
position,
|
||||
fieldtype,
|
||||
array,
|
||||
});
|
||||
}
|
||||
|
||||
self.eat_token_type(TokenType::Semicolon)?;
|
||||
|
||||
Ok(ServiceMethodNode {
|
||||
position: pos,
|
||||
name,
|
||||
inputs,
|
||||
return_type,
|
||||
decorators,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_service(&mut self) -> PResult<ServiceStatement> {
|
||||
debug!("Parser::parse_service()");
|
||||
let (_, pos) = self.eat_token_value("service")?;
|
||||
let (name, _) = self.eat_text()?;
|
||||
trace!("Parser::parse_service()::name = {}", name);
|
||||
|
||||
self.eat_token_type(TokenType::CurlyOpen)?;
|
||||
|
||||
let mut methods = Vec::new();
|
||||
while self.current_token().0 != TokenType::CurlyClose {
|
||||
let mut decorators = Vec::new();
|
||||
while self.current_token().0 == TokenType::At {
|
||||
decorators.push(self.parse_decorator()?);
|
||||
}
|
||||
|
||||
let mut notification = false;
|
||||
if self.current_token().1 == "notification" {
|
||||
self.eat_token()?;
|
||||
notification = true;
|
||||
}
|
||||
|
||||
methods.push(self.parse_method(decorators, notification)?);
|
||||
}
|
||||
|
||||
self.eat_token_type(TokenType::CurlyClose)?;
|
||||
|
||||
Ok(ServiceStatement {
|
||||
position: pos,
|
||||
name,
|
||||
methods,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_statement(&mut self) -> PResult<RootNode> {
|
||||
debug!("Parser::parse_statement()");
|
||||
let token = self.current_token();
|
||||
|
||||
if let TokenType::Keyword = token.0 {
|
||||
trace!("Parser::parse_statement()::type = {}", token.1);
|
||||
return match token.1.as_str() {
|
||||
"define" => Ok(RootNode::Define(self.parse_define()?)),
|
||||
"import" => Ok(RootNode::Import(self.parse_import()?)),
|
||||
"type" => Ok(RootNode::Type(self.parse_type()?)),
|
||||
"service" => Ok(RootNode::Service(self.parse_service()?)),
|
||||
"enum" => Ok(RootNode::Enum(self.parse_enum()?)),
|
||||
_ => Err(ParserError::new("Unknown keyword", token)),
|
||||
};
|
||||
} else {
|
||||
Err(ParserError::new("Expected keyword", token))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(&mut self) -> PResult<Vec<RootNode>> {
|
||||
debug!("Parser::parse()");
|
||||
let mut nodes = Vec::new();
|
||||
|
||||
while self.has_current_token() {
|
||||
nodes.push(self.parse_statement()?);
|
||||
}
|
||||
|
||||
Ok(nodes)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ParserError {
|
||||
pub message: String,
|
||||
pub token: Token,
|
||||
}
|
||||
|
||||
impl ParserError {
|
||||
fn new(msg: &str, token: &Token) -> ParserError {
|
||||
ParserError {
|
||||
message: format!("{}: {}", msg, token.1),
|
||||
token: token.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Error for ParserError {}
|
||||
impl Display for ParserError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "ParserError: {} at {:?}", self.message, self.token)
|
||||
}
|
||||
}
|
||||
99
crates/libjrpc/src/process.rs
Normal file
99
crates/libjrpc/src/process.rs
Normal file
@ -0,0 +1,99 @@
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use anyhow::Result;
|
||||
use url::Url;
|
||||
|
||||
use crate::ir::IR;
|
||||
|
||||
pub struct FileProcessor {
|
||||
file_cache: HashMap<String, String>,
|
||||
processed_files: HashSet<String>,
|
||||
}
|
||||
|
||||
impl FileProcessor {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
file_cache: HashMap::new(),
|
||||
processed_files: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_url(inp: Option<&str>) -> bool {
|
||||
#[cfg(feature = "http")]
|
||||
if let Some(inp) = inp {
|
||||
if inp.starts_with("http://") || inp.starts_with("https://") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn resolve_path(input: &str, context: Option<&str>) -> String {
|
||||
let mut input = input.to_string();
|
||||
#[cfg(feature = "http")]
|
||||
if cfg!(feature = "http") && (Self::is_url(Some(&input)) || Self::is_url(context)) {
|
||||
if Self::is_url(Some(&input)) {
|
||||
input.to_string()
|
||||
} else {
|
||||
let mut url = Url::parse(context.unwrap()).unwrap();
|
||||
if !input.ends_with(".jrpc") {
|
||||
input = format!("{}.jrpc", input);
|
||||
}
|
||||
url.join(&input).unwrap().to_string()
|
||||
}
|
||||
} else {
|
||||
if !input.ends_with(".jrpc") {
|
||||
input = format!("{}.jrpc", input);
|
||||
}
|
||||
if let Some(context) = context {
|
||||
let mut path = PathBuf::from(context);
|
||||
path = path.join(input);
|
||||
path.to_str().unwrap().to_string()
|
||||
} else {
|
||||
input
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_file_url(&mut self, url: &str) -> Result<String> {
|
||||
let resp = reqwest::blocking::get(url)?;
|
||||
let body = resp.text()?;
|
||||
self.file_cache.insert(url.to_string(), body.clone());
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn get_file_path(&mut self, path: &str) -> Result<String> {
|
||||
let body = std::fs::read_to_string(path)?;
|
||||
self.file_cache.insert(path.to_string(), body.clone());
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn get_file(&mut self, name: &str) -> Result<String> {
|
||||
if self.file_cache.contains_key(name) {
|
||||
return Ok(self.file_cache.get(name).unwrap().clone());
|
||||
} else {
|
||||
#[cfg(feature = "http")]
|
||||
{
|
||||
if name.starts_with("http://") || name.starts_with("https://") {
|
||||
return self.get_file_url(name);
|
||||
} else {
|
||||
return self.get_file_path(name);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "http"))]
|
||||
{
|
||||
return self.get_file_path(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn process_file(&mut self, file: &str, root: bool) {}
|
||||
|
||||
pub fn start_compile(&mut self, file: &str) -> Result<IR> {
|
||||
Err(anyhow::anyhow!("Not implemented"))
|
||||
}
|
||||
}
|
||||
7
crates/libjrpc/src/shared.rs
Normal file
7
crates/libjrpc/src/shared.rs
Normal file
@ -0,0 +1,7 @@
|
||||
pub enum Keywords {
|
||||
Type,
|
||||
Enum,
|
||||
Import,
|
||||
Service,
|
||||
Define,
|
||||
}
|
||||
197
crates/libjrpc/src/tokenizer.rs
Normal file
197
crates/libjrpc/src/tokenizer.rs
Normal file
@ -0,0 +1,197 @@
|
||||
use anyhow::Result;
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
use std::{error::Error, fmt::Display, sync::Arc};
|
||||
|
||||
fn match_regex(regex: &Regex, input: &str, index: usize) -> Option<String> {
|
||||
// println!("Matching regex {:?} at index {}", regex, index);
|
||||
let (_, b) = input.split_at(index);
|
||||
let Some(caps) = regex.captures(b) else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let capture = caps.get(0).unwrap();
|
||||
|
||||
if capture.start() != 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(capture.as_str().to_owned())
|
||||
}
|
||||
|
||||
struct TokenizerRegex(Regex, TokenType);
|
||||
|
||||
lazy_static! {
|
||||
static ref REGEXES: Vec<TokenizerRegex> = {
|
||||
let mut regexes = Vec::new();
|
||||
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\s+"#).unwrap(),
|
||||
TokenType::Space,
|
||||
));
|
||||
regexes.push(TokenizerRegex( //FIXME: This regex is not working
|
||||
Regex::new(r#"(?s)^(\/\*)(.|\s)*?(\*\/)/"#).unwrap(),
|
||||
TokenType::Comment,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\/\/.+"#).unwrap(),
|
||||
TokenType::Comment,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^#.+"#).unwrap(),
|
||||
TokenType::Comment,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^".*?""#).unwrap(),
|
||||
TokenType::String,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^(type|enum|import|service|define)\b"#).unwrap(),
|
||||
TokenType::Keyword,
|
||||
));
|
||||
regexes.push(TokenizerRegex(Regex::new(r#"^\@"#).unwrap(), TokenType::At));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\:"#).unwrap(),
|
||||
TokenType::Colon,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\;"#).unwrap(),
|
||||
TokenType::Semicolon,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\,"#).unwrap(),
|
||||
TokenType::Comma,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\="#).unwrap(),
|
||||
TokenType::Equals,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\{"#).unwrap(),
|
||||
TokenType::CurlyOpen,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\}"#).unwrap(),
|
||||
TokenType::CurlyClose,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\("#).unwrap(),
|
||||
TokenType::BracketOpen,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\)"#).unwrap(),
|
||||
TokenType::BracketClose,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\[\]"#).unwrap(),
|
||||
TokenType::Array,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^\?"#).unwrap(),
|
||||
TokenType::Questionmark,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^[\.0-9]+"#).unwrap(),
|
||||
TokenType::Number,
|
||||
));
|
||||
regexes.push(TokenizerRegex(
|
||||
Regex::new(r#"^[a-zA-Z_]([a-zA-Z0-9_]?)+"#).unwrap(),
|
||||
TokenType::Text,
|
||||
));
|
||||
|
||||
regexes
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TokenPosition {
|
||||
pub path: Arc<String>,
|
||||
pub start: usize,
|
||||
pub end: usize,
|
||||
}
|
||||
|
||||
impl TokenPosition {
|
||||
pub fn new(path: Arc<String>, start: usize, end: usize) -> Self {
|
||||
Self { path, start, end }
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for TokenPosition {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
//TODO: Map index to line and column
|
||||
write!(f, "{}:{}:{}", self.path, self.start, self.end)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum TokenType {
|
||||
Space,
|
||||
Comment,
|
||||
String,
|
||||
Keyword,
|
||||
At,
|
||||
Colon,
|
||||
Semicolon,
|
||||
Comma,
|
||||
Equals,
|
||||
CurlyOpen,
|
||||
CurlyClose,
|
||||
BracketOpen,
|
||||
BracketClose,
|
||||
Array,
|
||||
Questionmark,
|
||||
Number,
|
||||
Text,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TokenError {
|
||||
pub message: String,
|
||||
pub position: TokenPosition,
|
||||
}
|
||||
|
||||
impl Error for TokenError {}
|
||||
impl Display for TokenError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "TokenError: {} at {:?}", self.message, self.position)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Token(pub TokenType, pub String, pub TokenPosition);
|
||||
|
||||
pub fn tokenize(path: Arc<String>, input: String) -> Result<Vec<Token>> {
|
||||
let mut tokens = Vec::new();
|
||||
let mut index = 0;
|
||||
|
||||
while index < input.len() {
|
||||
let res = REGEXES
|
||||
.iter()
|
||||
.map(|regex| (match_regex(®ex.0, &input, index), regex.1))
|
||||
.find(|x| x.0.is_some());
|
||||
|
||||
if let Some(mat) = res {
|
||||
let value = mat.0.unwrap();
|
||||
let value_len = value.len();
|
||||
tokens.push(Token(
|
||||
mat.1,
|
||||
value,
|
||||
TokenPosition::new(path.clone(), index, index + value_len),
|
||||
));
|
||||
index += value_len;
|
||||
} else {
|
||||
return Err(TokenError {
|
||||
message: format!(
|
||||
"Unexpected character: {}",
|
||||
input.chars().skip(index).take(4).collect::<String>()
|
||||
),
|
||||
position: TokenPosition::new(path.clone(), index, index + 1),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
|
||||
// println!("{:?}", tokens);
|
||||
|
||||
Ok(tokens)
|
||||
}
|
||||
Reference in New Issue
Block a user