First Commit

.gitignore

@@ -0,0 +1 @@
+target/

Cargo.toml

@@ -0,0 +1,2 @@
+[workspace]
+members = ["crates/zed", "crates/libjrpc"]

crates/libjrpc/.gitignore

@@ -0,0 +1 @@
+/target

crates/libjrpc/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "libjrpc"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+default = ["http"]
+http = ["dep:reqwest", "dep:url"]
+
+[dependencies]
+anyhow = "1.0.89"
+lazy_static = "1.5.0"
+log = "0.4.22"
+regex = "1.10.6"
+reqwest = { version = "0.12.7", optional = true, features = ["blocking"] }
+url = { version = "2.5.2", optional = true }
+
+[dev-dependencies]
+env_logger = "0.11.5"
+ctor = "*"

crates/libjrpc/src/ir.rs

@@ -0,0 +1,461 @@
+use std::{
+    collections::{HashMap, HashSet},
+    error::Error,
+    fmt::Display,
+    hash::{Hash, Hasher},
+};
+
+use anyhow::Result;
+
+use crate::parser::{
+    EnumStatement, Node, ParserPosition, RootNode, ServiceStatement, TypeStatement,
+};
+
+static BUILT_INS: [&str; 4] = ["int", "float", "string", "boolean"];
+
+pub trait Definition {
+    fn get_position(&self) -> ParserPosition;
+}
+
+#[derive(Debug, Clone)]
+pub struct IR {
+    options: HashMap<String, String>,
+    steps: Vec<Step>,
+}
+
+#[derive(Debug, Clone)]
+pub enum Step {
+    Type(TypeDefinition),
+    Enum(EnumDefinition),
+    Service(ServiceDefinition),
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Type {
+    Int,
+    Float,
+    String,
+    Bool,
+    Custom(String),
+}
+impl Hash for Type {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        match self {
+            Type::Int => "int".hash(state),
+            Type::Float => "float".hash(state),
+            Type::String => "string".hash(state),
+            Type::Bool => "bool".hash(state),
+            Type::Custom(name) => name.hash(state),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TypeDefinition {
+    name: String,
+    depends: HashSet<Type>,
+    fields: Vec<Field>,
+    position: ParserPosition,
+}
+
+impl Definition for TypeDefinition {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Field {
+    name: String,
+    typ: Type,
+    array: bool,
+    optional: bool,
+    map: Option<Type>,
+}
+
+#[derive(Debug, Clone)]
+pub struct EnumDefinition {
+    name: String,
+    values: Vec<EnumField>,
+    position: ParserPosition,
+}
+
+impl Definition for EnumDefinition {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct EnumField {
+    name: String,
+    value: i32,
+}
+
+#[derive(Debug, Clone)]
+pub struct ServiceDefinition {
+    name: String,
+    depends: HashSet<Type>,
+    methods: Vec<Method>,
+    position: ParserPosition,
+}
+
+impl Definition for ServiceDefinition {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Method {
+    name: String,
+    inputs: Vec<MethodInput>,
+    output: Option<MethodOutput>,
+    decorators: MethodDecorators,
+}
+
+#[derive(Debug, Clone)]
+pub struct MethodInput {
+    name: String,
+    typ: Type,
+    array: bool,
+    optional: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct MethodOutput {
+    typ: Type,
+    array: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct MethodDecorators {
+    description: Option<String>,
+    parameter_descriptions: HashMap<String, String>,
+    return_description: Option<String>,
+}
+
+fn typename_to_type(name: &str) -> Type {
+    match name {
+        "int" => Type::Int,
+        "float" => Type::Float,
+        "string" => Type::String,
+        "boolean" => Type::Bool,
+        "bool" => Type::Bool,
+        _ => Type::Custom(name.to_string()),
+    }
+}
+
+fn build_type(stmt: &TypeStatement) -> Result<TypeDefinition> {
+    let mut typedef = TypeDefinition {
+        position: stmt.position.clone(),
+        name: stmt.name.clone(),
+        depends: HashSet::new(),
+        fields: Vec::new(),
+    };
+
+    for field in &stmt.fields {
+        let typ = typename_to_type(&field.fieldtype);
+        typedef.depends.insert(typ.clone());
+
+        if let Some(maptype) = &field.map {
+            if maptype != "string" && maptype != "int" {
+                return Err(IRError::new("Map type must be string or int", field).into());
+            }
+        }
+
+        typedef.fields.push(Field {
+            name: field.name.clone(),
+            typ: typ.clone(),
+            array: field.array,
+            optional: field.optional,
+            map: field.map.as_ref().map(|s| typename_to_type(s)),
+        });
+    }
+
+    Ok(typedef)
+}
+
+fn build_enum(stmt: &EnumStatement) -> Result<EnumDefinition> {
+    let mut enumdef = EnumDefinition {
+        position: stmt.position.clone(),
+        name: stmt.name.clone(),
+        values: Vec::new(),
+    };
+
+    let mut last = -1 as i32;
+    for field in &stmt.values {
+        let value = if let Some(value) = field.value {
+            if value > std::i32::MAX as i64 {
+                return Err(IRError::new("Enum value too large", field).into());
+            }
+            let value = value as i32;
+            if value <= last {
+                return Err(IRError::new("Enum values must be increasing", field).into());
+            }
+            last = value;
+            value
+        } else {
+            last = last + 1;
+            last
+        };
+        enumdef.values.push(EnumField {
+            name: field.name.clone(),
+            value,
+        });
+    }
+    Ok(enumdef)
+}
+
+fn build_service(stmt: &ServiceStatement) -> Result<ServiceDefinition> {
+    let mut servdef = ServiceDefinition {
+        position: stmt.position.clone(),
+        name: stmt.name.clone(),
+        depends: HashSet::new(),
+        methods: Vec::new(),
+    };
+
+    for method in &stmt.methods {
+        let mut methoddef = Method {
+            name: method.name.clone(),
+            inputs: Vec::new(),
+            output: method.return_type.as_ref().map(|rt| {
+                let typ = typename_to_type(&rt.fieldtype);
+                servdef.depends.insert(typ.clone());
+                MethodOutput {
+                    typ,
+                    array: rt.array,
+                }
+            }),
+            decorators: MethodDecorators {
+                description: None,
+                parameter_descriptions: HashMap::new(),
+                return_description: None,
+            },
+        };
+
+        let mut optional_starts = false;
+        for inp in &method.inputs {
+            let typ = typename_to_type(&inp.fieldtype);
+            servdef.depends.insert(typ.clone());
+
+            if optional_starts && !inp.optional {
+                return Err(
+                    IRError::new("Optional fields must come after required fields", inp).into(),
+                );
+            }
+
+            if inp.optional {
+                optional_starts = true;
+            }
+
+            methoddef.inputs.push(MethodInput {
+                name: inp.name.clone(),
+                typ,
+                array: inp.array,
+                optional: inp.optional,
+            });
+        }
+
+        for decorator in &method.decorators {
+            match decorator.name.as_str() {
+                "Description" => {
+                    if methoddef.decorators.description.is_some() {
+                        return Err(IRError::new("Duplicate description", decorator).into());
+                    }
+
+                    if decorator.args.len() != 1 {
+                        return Err(IRError::new(
+                            "Description must have exactly one argument",
+                            decorator,
+                        )
+                        .into());
+                    }
+
+                    methoddef.decorators.description = Some(decorator.args[0].clone());
+                }
+                "Returns" => {
+                    if methoddef.decorators.return_description.is_some() {
+                        return Err(IRError::new("Duplicate return description", decorator).into());
+                    }
+
+                    if decorator.args.len() != 1 {
+                        return Err(IRError::new(
+                            "Returns must have exactly one argument",
+                            decorator,
+                        )
+                        .into());
+                    }
+
+                    methoddef.decorators.return_description = Some(decorator.args[0].clone());
+                }
+                "Param" => {
+                    if decorator.args.len() != 2 {
+                        return Err(IRError::new(
+                            "Param must have exactly two arguments",
+                            decorator,
+                        )
+                        .into());
+                    }
+
+                    let name = decorator.args[0].clone();
+                    let description = decorator.args[1].clone();
+
+                    if methoddef
+                        .decorators
+                        .parameter_descriptions
+                        .contains_key(&name)
+                    {
+                        return Err(
+                            IRError::new("Duplicate parameter description", decorator).into()
+                        );
+                    }
+
+                    if methoddef.inputs.iter().find(|i| i.name == name).is_none() {
+                        return Err(IRError::new("Parameter not found", decorator).into());
+                    }
+
+                    methoddef
+                        .decorators
+                        .parameter_descriptions
+                        .insert(name, description);
+                }
+                _ => {
+                    return Err(IRError::new("Unknown decorator", decorator).into());
+                }
+            }
+        }
+        servdef.methods.push(methoddef);
+    }
+
+    Ok(servdef)
+}
+
+pub fn build_ir(root: &Vec<RootNode>) -> Result<IR> {
+    let mut options = HashMap::<String, String>::new();
+    let mut steps = Vec::new();
+
+    for node in root {
+        match node {
+            RootNode::Type(stmt) => steps.push(Step::Type(build_type(stmt)?)),
+            RootNode::Enum(stmt) => steps.push(Step::Enum(build_enum(stmt)?)),
+            RootNode::Service(stmt) => steps.push(Step::Service(build_service(stmt)?)),
+            RootNode::Define(stmt) => {
+                if options.contains_key(&stmt.key) {
+                    return Err(IRError::new("Duplicate define", stmt).into());
+                }
+
+                if (stmt.key == "use_messagepack" || stmt.key == "allow_bytes")
+                    && stmt.value == "true"
+                {
+                    options.insert("allow_bytes".to_owned(), "true".to_owned());
+                }
+                options.insert(stmt.key.clone(), stmt.value.clone());
+            }
+            RootNode::Import(_) => {
+                panic!("Import not supported at this stage!");
+            }
+        }
+    }
+
+    let mut all_types = HashSet::<String>::new();
+    let mut serv_types = HashSet::<String>::new();
+
+    for bi in &BUILT_INS {
+        all_types.insert(bi.to_string());
+    }
+
+    for step in &steps {
+        match step {
+            Step::Type(typedef) => {
+                if all_types.contains(&typedef.name) {
+                    return Err(IRError::new_from_def("Duplicate type", typedef).into());
+                }
+
+                all_types.insert(typedef.name.clone());
+            }
+            Step::Enum(enumdef) => {
+                if all_types.contains(&enumdef.name) {
+                    return Err(IRError::new_from_def("Duplicate type", enumdef).into());
+                }
+
+                all_types.insert(enumdef.name.clone());
+            }
+            Step::Service(servdef) => {
+                if serv_types.contains(&servdef.name) {
+                    return Err(IRError::new_from_def("Duplicate type", servdef).into());
+                }
+
+                serv_types.insert(servdef.name.clone());
+            }
+        }
+    }
+
+    // Verify dependencies
+
+    for step in &steps {
+        match step {
+            Step::Type(typedef) => {
+                for dep in &typedef.depends {
+                    if let Type::Custom(dep) = dep {
+                        if !all_types.contains(dep) {
+                            return Err(IRError::new_from_def(
+                                &format!("Type {} depends on unknown type {}", typedef.name, dep),
+                                typedef,
+                            )
+                            .into());
+                        }
+                    }
+                }
+            }
+            Step::Service(servdef) => {
+                for dep in &servdef.depends {
+                    if let Type::Custom(dep) = dep {
+                        if !all_types.contains(dep) {
+                            return Err(IRError::new_from_def(
+                                &format!(
+                                    "Service {} depends on unknown type {}",
+                                    servdef.name, dep
+                                ),
+                                servdef,
+                            )
+                            .into());
+                        }
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+
+    Ok(IR { options, steps })
+}
+
+#[derive(Debug, Clone)]
+pub struct IRError {
+    pub message: String,
+    pub position: ParserPosition,
+}
+
+impl IRError {
+    fn new(msg: &str, node: &impl Node) -> IRError {
+        IRError {
+            message: format!("{}: {}", msg, node.get_name()),
+            position: node.get_position(),
+        }
+    }
+
+    fn new_from_def(msg: &str, def: &impl Definition) -> IRError {
+        IRError {
+            message: msg.to_string(),
+            position: def.get_position(),
+        }
+    }
+}
+
+impl Error for IRError {}
+impl Display for IRError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "ParserError: {} at {:?}", self.message, self.position)
+    }
+}

crates/libjrpc/src/lib.rs

@@ -0,0 +1,39 @@
+mod ir;
+mod parser;
+mod process;
+mod shared;
+mod tokenizer;
+
+pub use tokenizer::{tokenize, Token, TokenError, TokenPosition, TokenType};
+
+#[cfg(test)]
+mod test {
+    use std::sync::Arc;
+
+    #[cfg(test)]
+    #[ctor::ctor]
+    fn init() {
+        env_logger::init();
+    }
+
+    #[test]
+    pub fn parse_jrpc() {
+        let tokens = crate::tokenizer::tokenize(
+            Arc::new("../test.jrpc".to_owned()),
+            include_str!("../test.jrpc").to_owned(),
+        )
+        .unwrap();
+
+        let parsed = crate::parser::Parser::new(tokens).parse().unwrap();
+        // println!("Parsed: {:?}", parsed);
+
+        let ir = crate::ir::build_ir(&parsed).unwrap();
+        println!("IR: {:?}", ir);
+
+        // let result = crate::JRPCParser::parse(crate::Rule::root, );
+        // match result {
+        //     Ok(result) => println!("{:?}", result),
+        //     Err(err) => println!("{:?}", err),
+        // }
+    }
+}

crates/libjrpc/src/parser.rs

@@ -0,0 +1,683 @@
+use anyhow::Result;
+use log::{debug, trace};
+use std::{collections::HashMap, error::Error, fmt::Display, sync::Arc};
+
+use crate::{Token, TokenPosition, TokenType};
+
+pub type PResult<T> = Result<T, ParserError>;
+
+#[derive(Debug, Clone)]
+pub struct ParserPosition {
+    path: Arc<String>,
+    position: usize,
+    token_positions: Vec<TokenPosition>,
+}
+
+impl ParserPosition {
+    pub fn from_token(token: &Token) -> Self {
+        Self {
+            path: token.2.path.clone(),
+            position: token.2.start,
+            token_positions: vec![token.2.clone()],
+        }
+    }
+}
+
+pub trait Node {
+    fn get_position(&self) -> ParserPosition;
+    fn get_name(&self) -> String {
+        String::from("")
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum RootNode {
+    Define(DefineStatement),
+    Import(ImportStatement),
+    Service(ServiceStatement),
+    Type(TypeStatement),
+    Enum(EnumStatement),
+}
+
+impl Node for RootNode {
+    fn get_position(&self) -> ParserPosition {
+        match self {
+            RootNode::Define(stmt) => stmt.get_position(),
+            RootNode::Import(stmt) => stmt.get_position(),
+            RootNode::Service(stmt) => stmt.get_position(),
+            RootNode::Type(stmt) => stmt.get_position(),
+            RootNode::Enum(stmt) => stmt.get_position(),
+        }
+    }
+
+    fn get_name(&self) -> String {
+        match self {
+            RootNode::Define(_) => String::from("define"),
+            RootNode::Import(_) => String::from("import"),
+            RootNode::Service(_) => String::from("service"),
+            RootNode::Type(_) => String::from("type"),
+            RootNode::Enum(_) => String::from("enum"),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct DefineStatement {
+    pub position: ParserPosition,
+    pub key: String,
+    pub value: String,
+}
+impl Node for DefineStatement {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+    fn get_name(&self) -> String {
+        format!("define {}", self.key)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ImportStatement {
+    pub position: ParserPosition,
+    pub path: String,
+}
+impl Node for ImportStatement {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+
+    fn get_name(&self) -> String {
+        format!("import {}", self.path)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TypeStatement {
+    pub position: ParserPosition,
+    pub name: String,
+    pub fields: Vec<TypeFieldNode>,
+}
+impl Node for TypeStatement {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+
+    fn get_name(&self) -> String {
+        format!("type {}", self.name)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TypeFieldNode {
+    pub position: ParserPosition,
+    pub name: String,
+    pub optional: bool,
+    pub fieldtype: String,
+    pub array: bool,
+    pub map: Option<String>,
+}
+impl Node for TypeFieldNode {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+
+    fn get_name(&self) -> String {
+        format!("type_field {}", self.name)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct EnumStatement {
+    pub position: ParserPosition,
+    pub name: String,
+    pub values: Vec<EnumValueNode>,
+}
+impl Node for EnumStatement {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+
+    fn get_name(&self) -> String {
+        format!("enum {}", self.name)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct EnumValueNode {
+    pub position: ParserPosition,
+    pub name: String,
+    pub value: Option<i64>,
+}
+impl Node for EnumValueNode {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+
+    fn get_name(&self) -> String {
+        format!("enum_field {}", self.name)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ServiceStatement {
+    pub position: ParserPosition,
+    pub name: String,
+    pub methods: Vec<ServiceMethodNode>,
+}
+impl Node for ServiceStatement {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+
+    fn get_name(&self) -> String {
+        format!("service {}", self.name)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ServiceMethodNode {
+    pub position: ParserPosition,
+    pub name: String,
+    pub inputs: Vec<ServiceMethodInputNode>,
+    pub return_type: Option<ServiceMethodReturnNode>,
+    pub decorators: Vec<Decorator>,
+}
+impl Node for ServiceMethodNode {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+
+    fn get_name(&self) -> String {
+        format!("method {}", self.name)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Decorator {
+    pub position: ParserPosition,
+    pub name: String,
+    pub args: Vec<String>,
+}
+impl Node for Decorator {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+
+    fn get_name(&self) -> String {
+        format!("decorator {}", self.name)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ServiceMethodInputNode {
+    pub position: ParserPosition,
+    pub name: String,
+    pub fieldtype: String,
+    pub optional: bool,
+    pub array: bool,
+}
+impl Node for ServiceMethodInputNode {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+
+    fn get_name(&self) -> String {
+        format!("method_arg {}", self.name)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ServiceMethodReturnNode {
+    pub position: ParserPosition,
+    pub fieldtype: String,
+    pub array: bool,
+}
+impl Node for ServiceMethodReturnNode {
+    fn get_position(&self) -> ParserPosition {
+        self.position.clone()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Parser {
+    tokens: Vec<Token>,
+    position: usize,
+}
+
+impl Parser {
+    pub fn new(tokens: Vec<Token>) -> Parser {
+        // Remove all comments and spaces at this stage
+        let tokens = tokens
+            .into_iter()
+            .filter(|e| e.0 != TokenType::Comment && e.0 != TokenType::Space)
+            .collect();
+        Parser {
+            tokens,
+            position: 0,
+        }
+    }
+
+    fn has_current_token(&self) -> bool {
+        self.position < self.tokens.len()
+    }
+
+    fn current_token(&mut self) -> &Token {
+        &self.tokens[self.position]
+    }
+
+    fn assert_types(&mut self, token_types: &[TokenType]) -> PResult<()> {
+        if token_types.iter().any(|t| *t == self.current_token().0) {
+            Ok(())
+        } else {
+            Err(ParserError::new(
+                &format!("Unexpected token. Expected on of {:?}", token_types),
+                self.current_token(),
+            ))
+        }
+    }
+
+    fn eat_token(&mut self) -> PResult<(String, ParserPosition)> {
+        debug!("Parser::eat_token()");
+        let pos = ParserPosition::from_token(self.current_token());
+        let value = self.current_token().1.clone();
+        self.position += 1;
+        Ok((value, pos))
+    }
+
+    fn eat_token_value(&mut self, value: &str) -> PResult<(String, ParserPosition)> {
+        debug!("Parser::eat_token_value({})", value);
+        let (val, pos) = self.eat_token()?;
+
+        if val != value {
+            return Err(ParserError::new(
+                &format!("Expected token of value {}", value),
+                self.current_token(),
+            ));
+        }
+
+        Ok((val, pos))
+    }
+
+    fn eat_token_type(&mut self, token_type: TokenType) -> PResult<(String, ParserPosition)> {
+        debug!("Parser::eat_token_type({:?})", token_type);
+        if self.current_token().0 != token_type {
+            return Err(ParserError::new(
+                &format!("Expected token of type {:?}", token_type),
+                self.current_token(),
+            ));
+        }
+
+        self.eat_token()
+    }
+
+    fn eat_text(&mut self) -> PResult<(String, ParserPosition)> {
+        debug!("Parser::eat_text()");
+        self.eat_token_type(TokenType::Text)
+    }
+
+    fn eat_text_with_keywords(&mut self) -> PResult<(String, ParserPosition)> {
+        debug!("Parser::eat_text_with_keywords()");
+        self.assert_types(&[TokenType::Text, TokenType::Keyword])?;
+        self.eat_token()
+    }
+
+    fn eat_string_or_text_as_raw(&mut self) -> PResult<String> {
+        debug!("Parser::eat_string_or_text_as_raw()");
+        self.assert_types(&[TokenType::String, TokenType::Text])?;
+        let token = self.current_token();
+        if token.0 == TokenType::String {
+            let (value, _) = self.eat_token()?;
+            let count = value.chars().count();
+            Ok(value.chars().skip(1).take(count - 2).collect())
+        } else {
+            let (value, _) = self.eat_token()?;
+            Ok(value)
+        }
+    }
+
+    fn eat_number(&mut self) -> PResult<i64> {
+        debug!("Parser::eat_number()");
+        self.assert_types(&[TokenType::Number])?;
+        let (value, _) = self.eat_token()?;
+        value
+            .parse()
+            .map_err(|_| ParserError::new("Invalid number", self.current_token()))
+    }
+
+    fn parse_define(&mut self) -> PResult<DefineStatement> {
+        debug!("Parser::parse_define()");
+        let (_, position) = self.eat_token_value("define")?;
+
+        let (key, _) = self.eat_token()?;
+        trace!("Parser::parse_define()::key = {}", key);
+
+        let value = self.eat_string_or_text_as_raw()?;
+
+        self.eat_token_type(TokenType::Semicolon)?;
+
+        Ok(DefineStatement {
+            position,
+            key,
+            value,
+        })
+    }
+
+    fn parse_import(&mut self) -> PResult<ImportStatement> {
+        debug!("Parser::parse_import()");
+        let (_, pos) = self.eat_token_value("import")?;
+        let path = self.eat_string_or_text_as_raw()?;
+
+        self.eat_token_type(TokenType::Semicolon)?;
+
+        Ok(ImportStatement {
+            position: pos,
+            path,
+        })
+    }
+
+    fn parse_enum_value(&mut self) -> PResult<EnumValueNode> {
+        debug!("Parser::parse_enum_value()");
+        let (name, pos) = self.eat_token()?;
+        let value = if self.current_token().0 == TokenType::Equals {
+            self.eat_token()?;
+            Some(self.eat_number()?)
+        } else {
+            None
+        };
+
+        Ok(EnumValueNode {
+            position: pos,
+            name,
+            value,
+        })
+    }
+
+    fn parse_enum(&mut self) -> PResult<EnumStatement> {
+        debug!("Parser::parse_enum()");
+        let (_, pos) = self.eat_token_value("enum")?;
+        let (name, _) = self.eat_token()?;
+        trace!("Parser::parse_enum()::name = {}", name);
+
+        self.eat_token_type(TokenType::CurlyOpen)?;
+        let mut values: Vec<EnumValueNode> = Vec::new();
+
+        while self.current_token().0 == TokenType::Text {
+            values.push(self.parse_enum_value()?);
+            if self.current_token().0 == TokenType::Comma {
+                //TODO: Maybe use a next flag or break or something with the commas
+                self.eat_token()?;
+            }
+        }
+
+        self.eat_token_type(TokenType::CurlyClose)?;
+
+        Ok(EnumStatement {
+            position: pos,
+            name,
+            values,
+        })
+    }
+
+    fn parse_type_field(&mut self) -> PResult<TypeFieldNode> {
+        debug!("Parser::parse_type_field()");
+        let (name, pos) = self.eat_text_with_keywords()?;
+        trace!("Parser::parse_type_field()::name = {}", name);
+
+        let mut optional = false;
+        let mut array = false;
+        let mut map = None;
+
+        if self.current_token().0 == TokenType::Questionmark {
+            optional = true;
+            self.eat_token()?;
+        }
+
+        _ = self.eat_token_type(TokenType::Colon);
+        let fieldtype = if self.current_token().0 == TokenType::CurlyOpen {
+            self.eat_token()?;
+            let (key_type, _) = self.eat_text()?;
+            self.eat_token_type(TokenType::Comma)?;
+            let (value_type, _) = self.eat_text()?;
+            self.eat_token_type(TokenType::CurlyClose)?;
+            map = Some(key_type);
+            value_type
+        } else {
+            let (type_name, _) = self.eat_text()?;
+            if self.current_token().0 == TokenType::Array {
+                array = true;
+                self.eat_token()?;
+            }
+
+            type_name
+        };
+
+        self.eat_token_type(TokenType::Semicolon)?;
+
+        Ok(TypeFieldNode {
+            position: pos,
+            name,
+            optional,
+            fieldtype,
+            map,
+            array,
+        })
+    }
+
+    fn parse_type(&mut self) -> PResult<TypeStatement> {
+        debug!("Parser::parse_type()");
+        let (_, pos) = self.eat_token_value("type")?;
+        let (name, _) = self.eat_text()?;
+        trace!("Parser::prase_type()::name = {}", name);
+        self.eat_token_type(TokenType::CurlyOpen)?;
+
+        let mut fields = Vec::new();
+        while self.current_token().0 == TokenType::Text
+            || self.current_token().0 == TokenType::Keyword
+        {
+            fields.push(self.parse_type_field()?);
+        }
+
+        self.eat_token_type(TokenType::CurlyClose)?;
+
+        Ok(TypeStatement {
+            position: pos,
+            name,
+            fields,
+        })
+    }
+
+    fn parse_decorator(&mut self) -> PResult<Decorator> {
+        debug!("Parser::parse_decorator()");
+        let (_, position) = self.eat_token_type(TokenType::At)?;
+        let (decorator, _) = self.eat_text()?;
+        trace!("Parser::parse_decorator()::name = {}", decorator);
+
+        self.eat_token_type(TokenType::BracketOpen)?;
+
+        let mut args = Vec::new();
+        let mut first = true;
+        while self.current_token().0 != TokenType::BracketClose {
+            if first {
+                first = false
+            } else {
+                self.eat_token_type(TokenType::Comma)?;
+            }
+
+            args.push(self.eat_string_or_text_as_raw()?);
+        }
+        self.eat_token_type(TokenType::BracketClose)?;
+
+        Ok(Decorator {
+            name: decorator,
+            args,
+            position,
+        })
+    }
+
+    fn parse_method(
+        &mut self,
+        decorators: Vec<Decorator>,
+        notification: bool,
+    ) -> PResult<ServiceMethodNode> {
+        debug!(
+            "Parser::parse_method({}, {})",
+            decorators.len(),
+            notification
+        );
+        let (name, pos) = self.eat_text()?;
+        trace!("Parser::parse_method()::name = {}", name);
+
+        self.eat_token_type(TokenType::BracketOpen)?;
+
+        let mut inputs = Vec::new();
+
+        while self.current_token().0 != TokenType::BracketClose {
+            let (name, position) = self.eat_text_with_keywords()?;
+
+            let mut optional = false;
+            if self.current_token().0 == TokenType::Questionmark {
+                optional = true;
+                self.eat_token()?;
+            }
+
+            self.eat_token_type(TokenType::Colon)?;
+            let (fieldtype, _) = self.eat_text()?;
+            let mut array = false;
+            if self.current_token().0 == TokenType::Array {
+                array = true;
+                self.eat_token()?;
+            }
+            inputs.push(ServiceMethodInputNode {
+                name,
+                fieldtype,
+                array,
+                optional,
+                position,
+            });
+
+            if self.current_token().0 == TokenType::Comma {
+                self.eat_token()?;
+            }
+
+            trace!(
+                "Parser::parse_method()::params_next_token: {}",
+                self.current_token().1
+            );
+        }
+
+        self.eat_token_type(TokenType::BracketClose)?;
+
+        let mut return_type = None;
+        if !notification {
+            self.eat_token_type(TokenType::Colon)?;
+
+            let (fieldtype, position) = self.eat_text()?;
+            let mut array = false;
+            if self.current_token().0 == TokenType::Array {
+                array = true;
+                self.eat_token()?;
+            }
+
+            return_type = Some(ServiceMethodReturnNode {
+                position,
+                fieldtype,
+                array,
+            });
+        }
+
+        self.eat_token_type(TokenType::Semicolon)?;
+
+        Ok(ServiceMethodNode {
+            position: pos,
+            name,
+            inputs,
+            return_type,
+            decorators,
+        })
+    }
+
+    fn parse_service(&mut self) -> PResult<ServiceStatement> {
+        debug!("Parser::parse_service()");
+        let (_, pos) = self.eat_token_value("service")?;
+        let (name, _) = self.eat_text()?;
+        trace!("Parser::parse_service()::name = {}", name);
+
+        self.eat_token_type(TokenType::CurlyOpen)?;
+
+        let mut methods = Vec::new();
+        while self.current_token().0 != TokenType::CurlyClose {
+            let mut decorators = Vec::new();
+            while self.current_token().0 == TokenType::At {
+                decorators.push(self.parse_decorator()?);
+            }
+
+            let mut notification = false;
+            if self.current_token().1 == "notification" {
+                self.eat_token()?;
+                notification = true;
+            }
+
+            methods.push(self.parse_method(decorators, notification)?);
+        }
+
+        self.eat_token_type(TokenType::CurlyClose)?;
+
+        Ok(ServiceStatement {
+            position: pos,
+            name,
+            methods,
+        })
+    }
+
+    fn parse_statement(&mut self) -> PResult<RootNode> {
+        debug!("Parser::parse_statement()");
+        let token = self.current_token();
+
+        if let TokenType::Keyword = token.0 {
+            trace!("Parser::parse_statement()::type = {}", token.1);
+            return match token.1.as_str() {
+                "define" => Ok(RootNode::Define(self.parse_define()?)),
+                "import" => Ok(RootNode::Import(self.parse_import()?)),
+                "type" => Ok(RootNode::Type(self.parse_type()?)),
+                "service" => Ok(RootNode::Service(self.parse_service()?)),
+                "enum" => Ok(RootNode::Enum(self.parse_enum()?)),
+                _ => Err(ParserError::new("Unknown keyword", token)),
+            };
+        } else {
+            Err(ParserError::new("Expected keyword", token))
+        }
+    }
+
+    pub fn parse(&mut self) -> PResult<Vec<RootNode>> {
+        debug!("Parser::parse()");
+        let mut nodes = Vec::new();
+
+        while self.has_current_token() {
+            nodes.push(self.parse_statement()?);
+        }
+
+        Ok(nodes)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ParserError {
+    pub message: String,
+    pub token: Token,
+}
+
+impl ParserError {
+    fn new(msg: &str, token: &Token) -> ParserError {
+        ParserError {
+            message: format!("{}: {}", msg, token.1),
+            token: token.clone(),
+        }
+    }
+}
+
+impl Error for ParserError {}
+impl Display for ParserError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "ParserError: {} at {:?}", self.message, self.token)
+    }
+}

crates/libjrpc/src/process.rs

@@ -0,0 +1,99 @@
+use std::{
+    collections::{HashMap, HashSet},
+    path::PathBuf,
+};
+
+use anyhow::Result;
+use url::Url;
+
+use crate::ir::IR;
+
+pub struct FileProcessor {
+    file_cache: HashMap<String, String>,
+    processed_files: HashSet<String>,
+}
+
+impl FileProcessor {
+    pub fn new() -> Self {
+        Self {
+            file_cache: HashMap::new(),
+            processed_files: HashSet::new(),
+        }
+    }
+
+    fn is_url(inp: Option<&str>) -> bool {
+        #[cfg(feature = "http")]
+        if let Some(inp) = inp {
+            if inp.starts_with("http://") || inp.starts_with("https://") {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn resolve_path(input: &str, context: Option<&str>) -> String {
+        let mut input = input.to_string();
+        #[cfg(feature = "http")]
+        if cfg!(feature = "http") && (Self::is_url(Some(&input)) || Self::is_url(context)) {
+            if Self::is_url(Some(&input)) {
+                input.to_string()
+            } else {
+                let mut url = Url::parse(context.unwrap()).unwrap();
+                if !input.ends_with(".jrpc") {
+                    input = format!("{}.jrpc", input);
+                }
+                url.join(&input).unwrap().to_string()
+            }
+        } else {
+            if !input.ends_with(".jrpc") {
+                input = format!("{}.jrpc", input);
+            }
+            if let Some(context) = context {
+                let mut path = PathBuf::from(context);
+                path = path.join(input);
+                path.to_str().unwrap().to_string()
+            } else {
+                input
+            }
+        }
+    }
+
+    fn get_file_url(&mut self, url: &str) -> Result<String> {
+        let resp = reqwest::blocking::get(url)?;
+        let body = resp.text()?;
+        self.file_cache.insert(url.to_string(), body.clone());
+        Ok(body)
+    }
+
+    fn get_file_path(&mut self, path: &str) -> Result<String> {
+        let body = std::fs::read_to_string(path)?;
+        self.file_cache.insert(path.to_string(), body.clone());
+        Ok(body)
+    }
+
+    fn get_file(&mut self, name: &str) -> Result<String> {
+        if self.file_cache.contains_key(name) {
+            return Ok(self.file_cache.get(name).unwrap().clone());
+        } else {
+            #[cfg(feature = "http")]
+            {
+                if name.starts_with("http://") || name.starts_with("https://") {
+                    return self.get_file_url(name);
+                } else {
+                    return self.get_file_path(name);
+                }
+            }
+
+            #[cfg(not(feature = "http"))]
+            {
+                return self.get_file_path(name);
+            }
+        }
+    }
+
+    fn process_file(&mut self, file: &str, root: bool) {}
+
+    pub fn start_compile(&mut self, file: &str) -> Result<IR> {
+        Err(anyhow::anyhow!("Not implemented"))
+    }
+}

crates/libjrpc/src/shared.rs

@@ -0,0 +1,7 @@
+pub enum Keywords {
+    Type,
+    Enum,
+    Import,
+    Service,
+    Define,
+}

crates/libjrpc/src/tokenizer.rs

@@ -0,0 +1,197 @@
+use anyhow::Result;
+use lazy_static::lazy_static;
+use regex::Regex;
+use std::{error::Error, fmt::Display, sync::Arc};
+
+fn match_regex(regex: &Regex, input: &str, index: usize) -> Option<String> {
+    // println!("Matching regex {:?} at index {}", regex, index);
+    let (_, b) = input.split_at(index);
+    let Some(caps) = regex.captures(b) else {
+        return None;
+    };
+
+    let capture = caps.get(0).unwrap();
+
+    if capture.start() != 0 {
+        return None;
+    }
+
+    Some(capture.as_str().to_owned())
+}
+
+struct TokenizerRegex(Regex, TokenType);
+
+lazy_static! {
+    static ref REGEXES: Vec<TokenizerRegex> = {
+        let mut regexes = Vec::new();
+
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\s+"#).unwrap(),
+            TokenType::Space,
+        ));
+        regexes.push(TokenizerRegex( //FIXME: This regex is not working
+            Regex::new(r#"(?s)^(\/\*)(.|\s)*?(\*\/)/"#).unwrap(),
+            TokenType::Comment,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\/\/.+"#).unwrap(),
+            TokenType::Comment,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^#.+"#).unwrap(),
+            TokenType::Comment,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^".*?""#).unwrap(),
+            TokenType::String,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^(type|enum|import|service|define)\b"#).unwrap(),
+            TokenType::Keyword,
+        ));
+        regexes.push(TokenizerRegex(Regex::new(r#"^\@"#).unwrap(), TokenType::At));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\:"#).unwrap(),
+            TokenType::Colon,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\;"#).unwrap(),
+            TokenType::Semicolon,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\,"#).unwrap(),
+            TokenType::Comma,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\="#).unwrap(),
+            TokenType::Equals,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\{"#).unwrap(),
+            TokenType::CurlyOpen,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\}"#).unwrap(),
+            TokenType::CurlyClose,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\("#).unwrap(),
+            TokenType::BracketOpen,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\)"#).unwrap(),
+            TokenType::BracketClose,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\[\]"#).unwrap(),
+            TokenType::Array,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^\?"#).unwrap(),
+            TokenType::Questionmark,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^[\.0-9]+"#).unwrap(),
+            TokenType::Number,
+        ));
+        regexes.push(TokenizerRegex(
+            Regex::new(r#"^[a-zA-Z_]([a-zA-Z0-9_]?)+"#).unwrap(),
+            TokenType::Text,
+        ));
+
+        regexes
+    };
+}
+
+#[derive(Debug, Clone)]
+pub struct TokenPosition {
+    pub path: Arc<String>,
+    pub start: usize,
+    pub end: usize,
+}
+
+impl TokenPosition {
+    pub fn new(path: Arc<String>, start: usize, end: usize) -> Self {
+        Self { path, start, end }
+    }
+}
+
+impl Display for TokenPosition {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        //TODO: Map index to line and column
+        write!(f, "{}:{}:{}", self.path, self.start, self.end)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum TokenType {
+    Space,
+    Comment,
+    String,
+    Keyword,
+    At,
+    Colon,
+    Semicolon,
+    Comma,
+    Equals,
+    CurlyOpen,
+    CurlyClose,
+    BracketOpen,
+    BracketClose,
+    Array,
+    Questionmark,
+    Number,
+    Text,
+}
+
+#[derive(Debug, Clone)]
+pub struct TokenError {
+    pub message: String,
+    pub position: TokenPosition,
+}
+
+impl Error for TokenError {}
+impl Display for TokenError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "TokenError: {} at {:?}", self.message, self.position)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Token(pub TokenType, pub String, pub TokenPosition);
+
+pub fn tokenize(path: Arc<String>, input: String) -> Result<Vec<Token>> {
+    let mut tokens = Vec::new();
+    let mut index = 0;
+
+    while index < input.len() {
+        let res = REGEXES
+            .iter()
+            .map(|regex| (match_regex(&regex.0, &input, index), regex.1))
+            .find(|x| x.0.is_some());
+
+        if let Some(mat) = res {
+            let value = mat.0.unwrap();
+            let value_len = value.len();
+            tokens.push(Token(
+                mat.1,
+                value,
+                TokenPosition::new(path.clone(), index, index + value_len),
+            ));
+            index += value_len;
+        } else {
+            return Err(TokenError {
+                message: format!(
+                    "Unexpected character: {}",
+                    input.chars().skip(index).take(4).collect::<String>()
+                ),
+                position: TokenPosition::new(path.clone(), index, index + 1),
+            }
+            .into());
+        }
+    }
+
+    // println!("{:?}", tokens);
+
+    Ok(tokens)
+}

crates/libjrpc/test.jrpc

@@ -0,0 +1,75 @@
+// Test
+// import "./import";
+
+define csharp_namespace Example;
+define rust_crate example;
+define dart_library_name example;
+
+enum TestEnum {
+   VAL1,
+   VAL2,
+   VAL10 = 10,
+   VAL11,
+   VAL12
+}
+
+type Test {
+   atom: TestAtom;
+   array: TestAtom[];
+   enumValue: TestEnum;
+   map: {int, TestAtom};
+}
+
+
+type AddValueRequest {
+   value1: float;
+   value2: float;
+}
+
+type AddValueResponse {
+   value: float;
+}
+
+service TestService {
+   @Description("Add two numbers")
+   @Param("request", "Parameter containing the two numbers")
+   @Returns("")
+   AddValuesSingleParam(request: AddValueRequest): AddValueResponse;
+
+   @Description("Add two numbers")
+   @Param("value1", "The first value")
+   @Param("value2", "The second value")
+   AddValuesMultipleParams(value1: float, value2: float): float;
+
+   @Description("Does literaly nothing")
+   @Param("param1", "Some number")
+   ReturningVoid(param1: float): void;
+
+   @Description("Just sends an Event with a String")
+   @Param("param1", "Parameter with some string for event")
+   notification OnEvent(param1: string);
+
+   ThrowingError(): void;
+
+
+   FunctionWithArrayAsParamAndReturn(values1: float[], values2: float[]): float[];
+
+   FunctionWithKeywords(type: float, static: float, event: float): float;
+}
+
+type Test2 {
+   name: string;
+   age:  int;
+}
+
+type TestKeywords {
+   type: float;
+   static: float;
+   event: float;
+}
+
+service SimpleTestService {
+   @Description("asdasdasd")
+   GetTest(name: string, age: int): Test2;
+   notification TestNot();
+}

crates/zed/.gitignore

@@ -0,0 +1,2 @@
+target/
+JsonRPC/

crates/zed/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "jrpc-syntax"
+version = "0.0.1"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+zed_extension_api = "0.1.0"

crates/zed/extension.toml

@@ -0,0 +1,7 @@
+id = "jrpc-syntax"
+name = "JRPC Syntax"
+version = "0.0.1"
+schema_version = 1
+authors = ["Fabian Stamm <dev@fabianstamm.de>"]
+description = "Adds JRPC syntax highlighting to Zed"
+repository = "https://github.com/"

crates/zed/src/lib.rs

@@ -0,0 +1,17 @@
+use zed_extension_api as zed;
+
+struct JRPCSyntaxExtension {
+    // ... state
+}
+
+impl zed::Extension for JRPCSyntaxExtension {
+    fn new() -> Self
+    where
+        Self: Sized,
+    {
+        todo!()
+    }
+    // ...
+}
+
+zed::register_extension!(JRPCSyntaxExtension);