【CarbonData】数据加载(Data Loading)过程的内部工作流程是怎样的?涉及哪些关键步骤?
2026/7/25 20:21:11
{}包裹,包含键值对- 数组(Array):用方括号[]包裹,包含有序元素- 字符串(String):用双引号""包裹- 数值(Number):整数或浮点数- 布尔值(Boolean):true或false- 空值(Null):null## 第二阶段:构建词法分析器(Tokenizer)词法分析是解析的第一步,将Json字符串分解为有意义的Token序列。以下是完整的Token定义和词法分析器实现:csharpusing System;using System.Collections.Generic;using System.Text;// 定义Token类型public enum TokenType{ String, // 字符串 Number, // 数值 True, // true False, // false Null, // null LeftBrace, // { RightBrace, // } LeftBracket, // [ RightBracket,// ] Colon, // : Comma, // , EOF // 文件结束}// Token类public class Token{ public TokenType Type { get; set; } public string Value { get; set; } public int Position { get; set; } // 用于错误定位 public Token(TokenType type, string value, int position) { Type = type; Value = value; Position = position; } public override string ToString() { return $"{Type}: {Value} (位置: {Position})"; }}// 词法分析器public class JsonTokenizer{ private readonly string _source; private int _position; public JsonTokenizer(string source) { _source = source; _position = 0; } // 获取下一个Token public Token NextToken() { SkipWhitespace(); if (_position >= _source.Length) return new Token(TokenType.EOF, "", _position); char current = _source[_position]; // 处理字符串 if (current == '"') return ReadString(); // 处理数值 if (char.IsDigit(current) || current == '-') return ReadNumber(); // 处理关键字 if (current == 't') return ReadKeyword("true", TokenType.True); if (current == 'f') return ReadKeyword("false", TokenType.False); if (current == 'n') return ReadKeyword("null", TokenType.Null); // 处理符号 switch (current) { case '{': return ConsumeChar(TokenType.LeftBrace); case '}': return ConsumeChar(TokenType.RightBrace); case '[': return ConsumeChar(TokenType.LeftBracket); case ']': return ConsumeChar(TokenType.RightBracket); case ':': return ConsumeChar(TokenType.Colon); case ',': return ConsumeChar(TokenType.Comma); default: throw new FormatException($"非法字符 '{current}' 在位置 {_position}"); } } // 跳过空白字符 private void SkipWhitespace() { while (_position < _source.Length && char.IsWhiteSpace(_source[_position])) _position++; } // 读取字符串 private Token ReadString() { _position++; // 跳过开头的引号 StringBuilder sb = new StringBuilder(); while (_position < _source.Length) { char c = _source[_position]; if (c == '"') // 字符串结束 { _position++; return new Token(TokenType.String, sb.ToString(), _position - sb.Length - 1); } if (c == '\\') // 转义字符处理 { _position++; if (_position >= _source.Length) throw new FormatException("字符串意外结束"); char next = _source[_position]; switch (next) { case '"': sb.Append('"'); break; case '\\': sb.Append('\\'); break; case '/': sb.Append('/'); break; case 'b': sb.Append('\b'); break; case 'f': sb.Append('\f'); break; case 'n': sb.Append('\n'); break; case 'r': sb.Append('\r'); break; case 't': sb.Append('\t'); break; case 'u': // Unicode转义 sb.Append((char)Convert.ToInt32(_source.Substring(_position + 1, 4), 16)); _position += 4; break; default: throw new FormatException($"无效转义字符 '\\{next}'"); } } else { sb.Append(c); } _position++; } throw new FormatException("未闭合的字符串"); } // 读取数值 private Token ReadNumber() { int start = _position; if (_source[_position] == '-') _position++; while (_position < _source.Length && char.IsDigit(_source[_position])) _position++; if (_position < _source.Length && _source[_position] == '.') { _position++; while (_position < _source.Length && char.IsDigit(_source[_position])) _position++; } if (_position < _source.Length && (_source[_position] == 'e' || _source[_position] == 'E')) { _position++; if (_position < _source.Length && (_source[_position] == '+' || _source[_position] == '-')) _position++; while (_position < _source.Length && char.IsDigit(_source[_position])) _position++; } string numberStr = _source.Substring(start, _position - start); return new Token(TokenType.Number, numberStr, start); } // 读取关键字 private Token ReadKeyword(string keyword, TokenType type) { int start = _position; foreach (char c in keyword) { if (_position >= _source.Length || _source[_position] != c) throw new FormatException($"期望关键字 '{keyword}' 在位置 {start}"); _position++; } return new Token(type, keyword, start); } // 消费单个字符Token private Token ConsumeChar(TokenType type) { return new Token(type, _source[_position++].ToString(), _position - 1); }}## 第三阶段:构建语法分析器(Parser)与Json对象模型有了Token序列后,我们需要构建语法分析器来生成Json对象模型。以下是完整的解析器实现:csharpusing System;using System.Collections.Generic;// Json值抽象基类public abstract class JsonValue{ public abstract object ToObject(); // 转换为C#原生对象}// 字符串值public class JsonString : JsonValue{ public string Value { get; set; } public override object ToObject() => Value; public override string ToString() => $"\"{Value}\"";}// 数值值public class JsonNumber : JsonValue{ public double Value { get; set; } public override object ToObject() => Value; public override string ToString() => Value.ToString();}// 布尔值public class JsonBoolean : JsonValue{ public bool Value { get; set; } public override object ToObject() => Value; public override string ToString() => Value ? "true" : "false";}// 空值public class JsonNull : JsonValue{ public override object ToObject() => null; public override string ToString() => "null";}// 数组值public class JsonArray : JsonValue{ public List<JsonValue> Items { get; set; } = new List<JsonValue>(); public override object ToObject() { var list = new List<object>(); foreach (var item in Items) list.Add(item.ToObject()); return list; } public override string ToString() => $"[{string.Join(", ", Items)}]";}// 对象值public class JsonObject : JsonValue{ public Dictionary<string, JsonValue> Properties { get; set; } = new Dictionary<string, JsonValue>(); public override object ToObject() { var dict = new Dictionary<string, object>(); foreach (var kvp in Properties) dict[kvp.Key] = kvp.Value.ToObject(); return dict; } public override string ToString() => $"{{{string.Join(", ", Properties)}}}";}// Json解析器public class JsonParser{ private readonly JsonTokenizer _tokenizer; private Token _currentToken; public JsonParser(string jsonString) { _tokenizer = new JsonTokenizer(jsonString); _currentToken = _tokenizer.NextToken(); // 初始化第一个Token } // 主解析入口 public JsonValue Parse() { var result = ParseValue(); if (_currentToken.Type != TokenType.EOF) throw new FormatException($"意外的Token '{_currentToken.Value}' 在位置 {_currentToken.Position}"); return result; } // 解析任意值 private JsonValue ParseValue() { switch (_currentToken.Type) { case TokenType.LeftBrace: return ParseObject(); case TokenType.LeftBracket: return ParseArray(); case TokenType.String: return ParseString(); case TokenType.Number: return ParseNumber(); case TokenType.True: case TokenType.False: return ParseBoolean(); case TokenType.Null: return ParseNull(); default: throw new FormatException($"意外的Token '{_currentToken.Value}' 在位置 {_currentToken.Position}"); } } // 解析对象 private JsonObject ParseObject() { var obj = new JsonObject(); ConsumeToken(TokenType.LeftBrace); // 吃掉 { if (_currentToken.Type == TokenType.RightBrace) { ConsumeToken(TokenType.RightBrace); return obj; // 空对象 } while (true) { // 解析键(必须是字符串) if (_currentToken.Type != TokenType.String) throw new FormatException($"期望字符串键在位置 {_currentToken.Position}"); string key = _currentToken.Value; ConsumeToken(TokenType.String); // 冒号 ConsumeToken(TokenType.Colon); // 解析值 JsonValue value = ParseValue(); obj.Properties[key] = value; // 检查逗号或结束 if (_currentToken.Type == TokenType.Comma) ConsumeToken(TokenType.Comma); else if (_currentToken.Type == TokenType.RightBrace) break; else throw new FormatException($"期望逗号或右花括号在位置 {_currentToken.Position}"); } ConsumeToken(TokenType.RightBrace); return obj; } // 解析数组 private JsonArray ParseArray() { var arr = new JsonArray(); ConsumeToken(TokenType.LeftBracket); // 吃掉 [ if (_currentToken.Type == TokenType.RightBracket) { ConsumeToken(TokenType.RightBracket); return arr; // 空数组 } while (true) { arr.Items.Add(ParseValue()); if (_currentToken.Type == TokenType.Comma) ConsumeToken(TokenType.Comma); else if (_currentToken.Type == TokenType.RightBracket) break; else throw new FormatException($"期望逗号或右方括号在位置 {_currentToken.Position}"); } ConsumeToken(TokenType.RightBracket); return arr; } // 基本类型解析 private JsonString ParseString() { var str = new JsonString { Value = _currentToken.Value }; ConsumeToken(TokenType.String); return str; } private JsonNumber ParseNumber() { var num = new JsonNumber { Value = double.Parse(_currentToken.Value) }; ConsumeToken(TokenType.Number); return num; } private JsonBoolean ParseBoolean() { var isTrue = _currentToken.Type == TokenType.True; ConsumeToken(isTrue ? TokenType.True : TokenType.False); return new JsonBoolean { Value = isTrue }; } private JsonNull ParseNull() { ConsumeToken(TokenType.Null); return new JsonNull(); } // 消费当前Token并获取下一个 private void ConsumeToken(TokenType expectedType) { if (_currentToken.Type != expectedType) throw new FormatException($"期望Token '{expectedType}' 但得到 '{_currentToken.Type}' 在位置 {_currentToken.Position}"); _currentToken = _tokenizer.NextToken(); }}## 第四阶段:添加自定义格式支持为了让解析器更实用,我们添加对自定义格式的支持,比如日期时间、GUID等:csharpusing System;using System.Collections.Generic;using System.Globalization;// 自定义格式处理器接口public interface ICustomFormatHandler{ bool CanParse(string value); object Parse(string value); string ToString(object value);}// 默认格式处理器(支持日期和GUID)public class DefaultCustomFormatHandler : ICustomFormatHandler{ public bool CanParse(string value) { // 检查是否为ISO日期格式 if (DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out _)) return true; // 检查是否为GUID格式 if (Guid.TryParse(value, out _)) return true; return false; } public object Parse(string value) { if (DateTime.TryParse(value, out DateTime dateTime)) return dateTime; if (Guid.TryParse(value, out Guid guid)) return guid; return value; // 无法识别则返回原字符串 } public string ToString(object value) { if (value is DateTime dt) return dt.ToString("o"); // ISO 8601格式 if (value is Guid guid) return guid.ToString(); return value?.ToString() ?? "null"; }}// 扩展的Json解析器,支持自定义格式转换public class ExtendedJsonParser : JsonParser{ private readonly ICustomFormatHandler _formatHandler; public ExtendedJsonParser(string jsonString, ICustomFormatHandler handler = null) : base(jsonString) { _formatHandler = handler ?? new DefaultCustomFormatHandler(); } // 重写解析方法以支持自定义格式 public new Dictionary<string, object> ParseToDictionary() { var jsonValue = base.Parse(); return ConvertToDictionary(jsonValue as JsonObject); } private Dictionary<string, object> ConvertToDictionary(JsonObject jsonObj) { var result = new Dictionary<string, object>(); foreach (var kvp in jsonObj.Properties) { object value = ConvertValue(kvp.Value); result[kvp.Key] = value; } return result; } private object ConvertValue(JsonValue value) { if (value is JsonString str) { // 尝试自定义格式解析 if (_formatHandler.CanParse(str.Value)) return _formatHandler.Parse(str.Value); return str.Value; } if (value is JsonNumber num) return num.Value; if (value is JsonBoolean boolean) return boolean.Value; if (value is JsonNull) return null; if (value is JsonArray arr) { var list = new List<object>(); foreach (var item in arr.Items) list.Add(ConvertValue(item)); return list; } if (value is JsonObject obj) return ConvertToDictionary(obj); throw new NotSupportedException($"不支持的Json值类型: {value.GetType()}"); }}## 第五阶段:完整使用示例以下是一个完整的示例,展示如何使用我们构建的Json解析器:```csharpusing System;using System.Collections.Generic;class Program{ static void Main() { // 测试Json字符串 string json = @" { ““name””: ““张三””, ““age””: 30, ““isStudent””: false, ““birthday””: ““1994-05-15T10:30:00"”, ““id””: ““a1b2c3d4-e5f6-7890-abcd-ef1234567890"”, ““address””: { ““city””: ““北京””, ““district””: ““海淀”” }, ““hobbies””: [”“阅读””, ““编程””, ““旅行””], ““metadata””: null }“; try { // 使用基础解析器 Console.WriteLine(”=== 基础解析器测试 =“); var parser = new JsonParser(json); JsonValue result = parser.Parse(); Console.WriteLine("解析结果:result");Console.WriteLine("解析结果: {result}"); Console.WriteLine("解析结果:result");Console.WriteLine(“转换为对象: {result.ToObject()}”); // 使用扩展解析器(支持自定义格式) Console.WriteLine(”\n= 扩展解析器测试 ==="); var extendedParser = new ExtendedJsonParser(json); Dictionary<string, object> dict = extendedParser.ParseToDictionary(); foreach (var kvp in dict) { Console.WriteLine(KaTeX parse error: Expected 'EOF', got '}' at position 83: …"); }̲ // …“生日类型: {dict[“birthday”].GetType().Name}”); // 应为 DateTime Console.WriteLine