From d5fb00a8a97ecc2575f85f097a5e9402a6ff958a Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Fri, 25 Jul 2025 02:58:13 +0800 Subject: [PATCH] :bug: Fix JSON serialization caused issue --- DysonNetwork.Drive/Storage/CloudFile.cs | 5 +- DysonNetwork.Drive/appsettings.json | 2 +- DysonNetwork.Gateway/appsettings.json | 1 + DysonNetwork.Pass/appsettings.json | 2 +- DysonNetwork.Shared/Cache/CacheService.cs | 43 ++--- .../Data/CloudFileReferenceObject.cs | 24 +-- DysonNetwork.Shared/Data/JsonExtensions.cs | 152 ++++++++++++++++++ .../DysonNetwork.Shared.csproj | 1 + DysonNetwork.Shared/Proto/GrpcTypeHelper.cs | 48 ++++-- DysonNetwork.Sphere/Post/PostController.cs | 1 - DysonNetwork.Sphere/Post/PostService.cs | 1 - DysonNetwork.sln.DotSettings.user | 3 + 12 files changed, 231 insertions(+), 52 deletions(-) create mode 100644 DysonNetwork.Shared/Data/JsonExtensions.cs diff --git a/DysonNetwork.Drive/Storage/CloudFile.cs b/DysonNetwork.Drive/Storage/CloudFile.cs index 8890215..4ba9803 100644 --- a/DysonNetwork.Drive/Storage/CloudFile.cs +++ b/DysonNetwork.Drive/Storage/CloudFile.cs @@ -3,7 +3,6 @@ using System.ComponentModel.DataAnnotations.Schema; using DysonNetwork.Shared.Data; using DysonNetwork.Shared.Proto; using Google.Protobuf; -using Newtonsoft.Json; using NodaTime; using NodaTime.Serialization.Protobuf; @@ -119,11 +118,11 @@ public class CloudFile : ModelBase, ICloudFile, IIdentifiedResource UploadedAt = UploadedAt?.ToTimestamp(), // Convert file metadata FileMeta = ByteString.CopyFromUtf8( - JsonConvert.SerializeObject(FileMeta, GrpcTypeHelper.SerializerSettings) + System.Text.Json.JsonSerializer.Serialize(FileMeta, GrpcTypeHelper.SerializerOptions) ), // Convert user metadata UserMeta = ByteString.CopyFromUtf8( - JsonConvert.SerializeObject(UserMeta, GrpcTypeHelper.SerializerSettings) + System.Text.Json.JsonSerializer.Serialize(UserMeta, GrpcTypeHelper.SerializerOptions) ) }; diff --git a/DysonNetwork.Drive/appsettings.json b/DysonNetwork.Drive/appsettings.json index 6538ff6..78b0850 100644 --- a/DysonNetwork.Drive/appsettings.json +++ b/DysonNetwork.Drive/appsettings.json @@ -1,7 +1,7 @@ { "Debug": true, "BaseUrl": "http://localhost:5071", - "GatewayUrl": "http://10.126.126.1:5094", + "GatewayUrl": "http://localhost:5094", "Logging": { "LogLevel": { "Default": "Information", diff --git a/DysonNetwork.Gateway/appsettings.json b/DysonNetwork.Gateway/appsettings.json index b8a754c..e43a949 100644 --- a/DysonNetwork.Gateway/appsettings.json +++ b/DysonNetwork.Gateway/appsettings.json @@ -1,5 +1,6 @@ { "LocalMode": true, + "CaCert": "../Certificates/ca.crt", "Logging": { "LogLevel": { "Default": "Information", diff --git a/DysonNetwork.Pass/appsettings.json b/DysonNetwork.Pass/appsettings.json index 446b82f..0374780 100644 --- a/DysonNetwork.Pass/appsettings.json +++ b/DysonNetwork.Pass/appsettings.json @@ -1,6 +1,6 @@ { "Debug": true, - "BaseUrl": "http://10.126.126.1:5216", + "BaseUrl": "http://localhost:5216", "Logging": { "LogLevel": { "Default": "Information", diff --git a/DysonNetwork.Shared/Cache/CacheService.cs b/DysonNetwork.Shared/Cache/CacheService.cs index 3ab8fcc..893ea17 100644 --- a/DysonNetwork.Shared/Cache/CacheService.cs +++ b/DysonNetwork.Shared/Cache/CacheService.cs @@ -1,7 +1,9 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using DysonNetwork.Shared.Data; using NodaTime; -using NodaTime.Serialization.JsonNet; +using NodaTime.Serialization.SystemTextJson; using StackExchange.Redis; namespace DysonNetwork.Shared.Cache; @@ -43,7 +45,7 @@ public interface ICacheService /// Gets a value from the cache /// Task GetAsync(string key); - + /// /// Get a value from the cache with the found status /// @@ -186,7 +188,7 @@ public class RedisDistributedLock : IDistributedLock public class CacheServiceRedis : ICacheService { private readonly IDatabase _database; - private readonly JsonSerializerSettings _serializerSettings; + private readonly JsonSerializerOptions _jsonOptions; // Global prefix for all cache keys public const string GlobalKeyPrefix = "dyson:"; @@ -199,19 +201,18 @@ public class CacheServiceRedis : ICacheService { var rds = redis ?? throw new ArgumentNullException(nameof(redis)); _database = rds.GetDatabase(); - - // Configure Newtonsoft.Json with proper NodaTime serialization - _serializerSettings = new JsonSerializerSettings + + // Configure System.Text.Json with proper NodaTime serialization + _jsonOptions = new JsonSerializerOptions { - ContractResolver = new CamelCasePropertyNamesContractResolver(), - PreserveReferencesHandling = PreserveReferencesHandling.None, - NullValueHandling = NullValueHandling.Include, - DateParseHandling = DateParseHandling.None, - ReferenceLoopHandling = ReferenceLoopHandling.Ignore + TypeInfoResolver = new DefaultJsonTypeInfoResolver + { + Modifiers = { JsonExtensions.UnignoreAllProperties() }, + }, + ReferenceHandler = ReferenceHandler.Preserve, }; - - // Configure NodaTime serializers - _serializerSettings.ConfigureForNodaTime(DateTimeZoneProviders.Tzdb); + _jsonOptions.ConfigureForNodaTime(DateTimeZoneProviders.Tzdb); + _jsonOptions.PropertyNameCaseInsensitive = true; } public async Task SetAsync(string key, T value, TimeSpan? expiry = null) @@ -220,7 +221,7 @@ public class CacheServiceRedis : ICacheService if (string.IsNullOrEmpty(key)) throw new ArgumentException("Key cannot be null or empty", nameof(key)); - var serializedValue = JsonConvert.SerializeObject(value, _serializerSettings); + var serializedValue = JsonSerializer.Serialize(value, _jsonOptions); return await _database.StringSetAsync(key, serializedValue, expiry); } @@ -235,8 +236,8 @@ public class CacheServiceRedis : ICacheService if (value.IsNullOrEmpty) return default; - // For NodaTime serialization, use the configured serializer settings - return JsonConvert.DeserializeObject(value!, _serializerSettings); + // For NodaTime serialization, use the configured JSON options + return JsonSerializer.Deserialize(value!, _jsonOptions); } public async Task<(bool found, T? value)> GetAsyncWithStatus(string key) @@ -250,8 +251,8 @@ public class CacheServiceRedis : ICacheService if (value.IsNullOrEmpty) return (false, default); - // For NodaTime serialization, use the configured serializer settings - return (true, JsonConvert.DeserializeObject(value!, _serializerSettings)); + // For NodaTime serialization, use the configured JSON options + return (true, JsonSerializer.Deserialize(value!, _jsonOptions)); } public async Task RemoveAsync(string key) diff --git a/DysonNetwork.Shared/Data/CloudFileReferenceObject.cs b/DysonNetwork.Shared/Data/CloudFileReferenceObject.cs index 6980771..498f517 100644 --- a/DysonNetwork.Shared/Data/CloudFileReferenceObject.cs +++ b/DysonNetwork.Shared/Data/CloudFileReferenceObject.cs @@ -1,6 +1,6 @@ +using System.Text.Json; using DysonNetwork.Shared.Proto; using Google.Protobuf; -using Newtonsoft.Json; namespace DysonNetwork.Shared.Data; @@ -42,14 +42,20 @@ public class CloudFileReferenceObject : ModelBase, ICloudFile { Id = proto.Id, Name = proto.Name, - FileMeta = JsonConvert.DeserializeObject>( + FileMeta = JsonSerializer.Deserialize>( proto.FileMeta.ToStringUtf8(), - GrpcTypeHelper.SerializerSettings - ) ?? [], - UserMeta = JsonConvert.DeserializeObject>( + GrpcTypeHelper.SerializerOptions + )?.ToDictionary( + kvp => kvp.Key, + kvp => kvp.Value.ValueKind == JsonValueKind.Undefined ? null : kvp.Value.Deserialize(GrpcTypeHelper.SerializerOptions) + ) ?? new Dictionary(), + UserMeta = JsonSerializer.Deserialize>( proto.UserMeta.ToStringUtf8(), - GrpcTypeHelper.SerializerSettings - ) ?? [], + GrpcTypeHelper.SerializerOptions + )?.ToDictionary( + kvp => kvp.Key, + kvp => kvp.Value.ValueKind == JsonValueKind.Undefined ? null : kvp.Value.Deserialize(GrpcTypeHelper.SerializerOptions) + ) ?? new Dictionary(), MimeType = proto.MimeType, Hash = proto.Hash, Size = proto.Size, @@ -76,12 +82,12 @@ public class CloudFileReferenceObject : ModelBase, ICloudFile // Convert file metadata proto.FileMeta = ByteString.CopyFromUtf8( - JsonConvert.SerializeObject(FileMeta, GrpcTypeHelper.SerializerSettings) + JsonSerializer.Serialize(FileMeta, GrpcTypeHelper.SerializerOptions) ); // Convert user metadata proto.UserMeta = ByteString.CopyFromUtf8( - JsonConvert.SerializeObject(UserMeta, GrpcTypeHelper.SerializerSettings) + JsonSerializer.Serialize(UserMeta, GrpcTypeHelper.SerializerOptions) ); return proto; diff --git a/DysonNetwork.Shared/Data/JsonExtensions.cs b/DysonNetwork.Shared/Data/JsonExtensions.cs new file mode 100644 index 0000000..4bc845d --- /dev/null +++ b/DysonNetwork.Shared/Data/JsonExtensions.cs @@ -0,0 +1,152 @@ +using System.Reflection; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace DysonNetwork.Shared.Data; + +public static class JsonExtensions +{ + public static Action UnignoreAllProperties() => + typeInfo => + { + if (typeInfo.Kind == JsonTypeInfoKind.Object) + // [JsonIgnore] is implemented by setting ShouldSerialize to a function that returns false. + foreach (var property in typeInfo.Properties.Where(ShouldUnignore)) + { + property.Get ??= CreatePropertyGetter(property); + property.Set ??= CreatePropertySetter(property); + if (property.Get != null) + property.ShouldSerialize = null; + } + }; + + public static Action UnignoreAllProperties(Type type) => + typeInfo => + { + if (type.IsAssignableFrom(typeInfo.Type) && typeInfo.Kind == JsonTypeInfoKind.Object) + // [JsonIgnore] is implemented by setting ShouldSerialize to a function that returns false. + foreach (var property in typeInfo.Properties.Where(ShouldUnignore)) + { + property.Get ??= CreatePropertyGetter(property); + property.Set ??= CreatePropertySetter(property); + if (property.Get != null) + property.ShouldSerialize = null; + } + }; + + public static Action UnignoreProperties(Type type, params string[] properties) => + typeInfo => + { + if (type.IsAssignableFrom(typeInfo.Type) && typeInfo.Kind == JsonTypeInfoKind.Object) + // [JsonIgnore] is implemented by setting ShouldSerialize to a function that returns false. + foreach (var property in typeInfo.Properties.Where(p => ShouldUnignore(p, properties))) + { + property.Get ??= CreatePropertyGetter(property); + property.Set ??= CreatePropertySetter(property); + if (property.Get != null) + property.ShouldSerialize = null; + } + }; + + public static Action UnignorePropertiesForDeserialize(Type type, params string[] properties) => + typeInfo => + { + if (type.IsAssignableFrom(typeInfo.Type) && typeInfo.Kind == JsonTypeInfoKind.Object) + // [JsonIgnore] is implemented by setting ShouldSerialize to a function that returns false. + foreach (var property in typeInfo.Properties.Where(p => ShouldUnignore(p, properties))) + { + property.Set ??= CreatePropertySetter(property); + } + }; + + static bool ShouldUnignore(JsonPropertyInfo property) => + property.ShouldSerialize != null && + property.AttributeProvider?.IsDefined(typeof(JsonIgnoreAttribute), true) == true; + + static bool ShouldUnignore(JsonPropertyInfo property, string[] properties) => + property.ShouldSerialize != null && + property.AttributeProvider?.IsDefined(typeof(JsonIgnoreAttribute), true) == true && + properties.Contains(property.GetMemberName()); + + // CreateGetter() and CreateSetter() taken from this answer https://stackoverflow.com/a/76296944/3744182 + // To https://stackoverflow.com/questions/61869393/get-net-core-jsonserializer-to-serialize-private-members + + delegate TValue RefFunc(ref TObject arg); + + static Func? CreatePropertyGetter(JsonPropertyInfo property) => + property.GetPropertyInfo() is { } info && info.ReflectedType != null && info.GetGetMethod() is { } getMethod + ? CreateGetter(info.ReflectedType, getMethod) + : null; + + static Func? CreateGetter(Type type, MethodInfo? method) + { + if (method == null) + return null; + var myMethod = typeof(JsonExtensions).GetMethod(nameof(JsonExtensions.CreateGetterGeneric), + BindingFlags.NonPublic | BindingFlags.Static)!; + return (Func)(myMethod.MakeGenericMethod(new[] { type, method.ReturnType }) + .Invoke(null, new[] { method })!); + } + + static Func CreateGetterGeneric(MethodInfo method) + { + if (method == null) + throw new ArgumentNullException(); + if (typeof(TObject).IsValueType) + { + // https://stackoverflow.com/questions/4326736/how-can-i-create-an-open-delegate-from-a-structs-instance-method + // https://stackoverflow.com/questions/1212346/uncurrying-an-instance-method-in-net/1212396#1212396 + var func = (RefFunc)Delegate.CreateDelegate(typeof(RefFunc), null, + method); + return (o) => + { + var tObj = (TObject)o; + return func(ref tObj); + }; + } + else + { + var func = (Func)Delegate.CreateDelegate(typeof(Func), method); + return (o) => func((TObject)o); + } + } + + static Action? CreatePropertySetter(JsonPropertyInfo property) => + property.GetPropertyInfo() is { } info && info.ReflectedType != null && info.GetSetMethod() is { } setMethod + ? CreateSetter(info.ReflectedType, setMethod) + : null; + + static Action? CreateSetter(Type type, MethodInfo? method) + { + if (method == null) + return null; + var myMethod = typeof(JsonExtensions).GetMethod(nameof(JsonExtensions.CreateSetterGeneric), + BindingFlags.NonPublic | BindingFlags.Static)!; + return (Action)(myMethod + .MakeGenericMethod(new[] { type, method.GetParameters().Single().ParameterType }) + .Invoke(null, new[] { method })!); + } + + static Action? CreateSetterGeneric(MethodInfo method) + { + if (method == null) + throw new ArgumentNullException(); + if (typeof(TObject).IsValueType) + { + // TODO: find a performant way to do this. Possibilities: + // Box from Microsoft.Toolkit.HighPerformance + // https://stackoverflow.com/questions/18937935/how-to-mutate-a-boxed-struct-using-il + return (o, v) => method.Invoke(o, new[] { v }); + } + else + { + var func = (Action)Delegate.CreateDelegate(typeof(Action), method); + return (o, v) => func((TObject)o, (TValue?)v); + } + } + + static PropertyInfo? GetPropertyInfo(this JsonPropertyInfo property) => + (property.AttributeProvider as PropertyInfo); + + static string? GetMemberName(this JsonPropertyInfo property) => (property.AttributeProvider as MemberInfo)?.Name; +} \ No newline at end of file diff --git a/DysonNetwork.Shared/DysonNetwork.Shared.csproj b/DysonNetwork.Shared/DysonNetwork.Shared.csproj index 92e78ee..c813c91 100644 --- a/DysonNetwork.Shared/DysonNetwork.Shared.csproj +++ b/DysonNetwork.Shared/DysonNetwork.Shared.csproj @@ -26,6 +26,7 @@ + diff --git a/DysonNetwork.Shared/Proto/GrpcTypeHelper.cs b/DysonNetwork.Shared/Proto/GrpcTypeHelper.cs index dc5c324..30e70d1 100644 --- a/DysonNetwork.Shared/Proto/GrpcTypeHelper.cs +++ b/DysonNetwork.Shared/Proto/GrpcTypeHelper.cs @@ -1,20 +1,26 @@ using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; using Google.Protobuf.Collections; using Google.Protobuf.WellKnownTypes; +using DysonNetwork.Shared.Data; using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; +using JsonSerializer = System.Text.Json.JsonSerializer; namespace DysonNetwork.Shared.Proto; public abstract class GrpcTypeHelper { - public static readonly JsonSerializerSettings SerializerSettings = new() + public static readonly JsonSerializerOptions SerializerOptions = new() { - ContractResolver = new DefaultContractResolver { NamingStrategy = new SnakeCaseNamingStrategy() }, - PreserveReferencesHandling = PreserveReferencesHandling.None, - NullValueHandling = NullValueHandling.Include, - DateParseHandling = DateParseHandling.None + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + PropertyNameCaseInsensitive = true, + TypeInfoResolver = new DefaultJsonTypeInfoResolver + { + Modifiers = { JsonExtensions.UnignoreAllProperties() } + } }; public static MapField ConvertToValueMap(Dictionary source) @@ -31,7 +37,7 @@ public abstract class GrpcTypeHelper double d => Value.ForNumber(d), bool b => Value.ForBool(b), null => Value.ForNull(), - _ => Value.ForString(JsonConvert.SerializeObject(kvp.Value, SerializerSettings)) // fallback to JSON string + _ => Value.ForString(JsonSerializer.Serialize(kvp.Value, SerializerOptions)) // fallback to JSON string }; } return result; @@ -49,7 +55,7 @@ public abstract class GrpcTypeHelper try { // Try to parse as JSON object or primitive - result[kvp.Key] = JsonConvert.DeserializeObject(value.StringValue, SerializerSettings); + result[kvp.Key] = JsonNode.Parse(value.StringValue)?.AsObject() ?? JsonObject.Create(new JsonElement()); } catch { @@ -67,10 +73,22 @@ public abstract class GrpcTypeHelper result[kvp.Key] = null; break; case Value.KindOneofCase.StructValue: - result[kvp.Key] = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(value.StructValue.Fields.ToDictionary(f => f.Key, f => ConvertValueToObject(f.Value)), SerializerSettings)); + result[kvp.Key] = JsonSerializer.Deserialize( + JsonSerializer.Serialize( + value.StructValue.Fields.ToDictionary(f => f.Key, f => ConvertValueToObject(f.Value)), + SerializerOptions + ), + SerializerOptions + ); break; case Value.KindOneofCase.ListValue: - result[kvp.Key] = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(value.ListValue.Values.Select(ConvertValueToObject).ToList(), SerializerSettings), SerializerSettings); + result[kvp.Key] = JsonSerializer.Deserialize( + JsonSerializer.Serialize( + value.ListValue.Values.Select(ConvertValueToObject).ToList(), + SerializerOptions + ), + SerializerOptions + ); break; default: result[kvp.Key] = null; @@ -88,7 +106,7 @@ public abstract class GrpcTypeHelper Value.KindOneofCase.NumberValue => value.NumberValue, Value.KindOneofCase.BoolValue => value.BoolValue, Value.KindOneofCase.NullValue => null, - _ => JsonConvert.DeserializeObject(JsonConvert.SerializeObject(value, SerializerSettings)) + _ => JsonSerializer.Deserialize(JsonSerializer.Serialize(value, SerializerOptions), SerializerOptions) }; } @@ -96,8 +114,8 @@ public abstract class GrpcTypeHelper { return value.KindCase switch { - Value.KindOneofCase.StringValue => JsonConvert.DeserializeObject(value.StringValue, SerializerSettings), - _ => JsonConvert.DeserializeObject(JsonConvert.SerializeObject(value, SerializerSettings)) + Value.KindOneofCase.StringValue => JsonSerializer.Deserialize(value.StringValue, SerializerOptions), + _ => JsonSerializer.Deserialize(JsonSerializer.Serialize(value, SerializerOptions), SerializerOptions) }; } @@ -112,7 +130,7 @@ public abstract class GrpcTypeHelper double d => Value.ForNumber(d), bool b => Value.ForBool(b), null => Value.ForNull(), - _ => Value.ForString(JsonConvert.SerializeObject(obj, SerializerSettings)) // fallback to JSON string + _ => Value.ForString(JsonSerializer.Serialize(obj, SerializerOptions)) // fallback to JSON string }; } @@ -123,7 +141,7 @@ public abstract class GrpcTypeHelper return obj switch { null => Value.ForNull(), - _ => Value.ForString(JsonConvert.SerializeObject(obj, SerializerSettings)) + _ => Value.ForString(JsonSerializer.Serialize(obj, SerializerOptions)) }; } } \ No newline at end of file diff --git a/DysonNetwork.Sphere/Post/PostController.cs b/DysonNetwork.Sphere/Post/PostController.cs index 751bb62..edb35b2 100644 --- a/DysonNetwork.Sphere/Post/PostController.cs +++ b/DysonNetwork.Sphere/Post/PostController.cs @@ -281,7 +281,6 @@ public class PostController( try { post = await ps.PostAsync( - currentUser, post, attachments: request.Attachments, tags: request.Tags, diff --git a/DysonNetwork.Sphere/Post/PostService.cs b/DysonNetwork.Sphere/Post/PostService.cs index d8f60e0..8f245ad 100644 --- a/DysonNetwork.Sphere/Post/PostService.cs +++ b/DysonNetwork.Sphere/Post/PostService.cs @@ -72,7 +72,6 @@ public partial class PostService( } public async Task PostAsync( - Account user, Post post, List? attachments = null, List? tags = null, diff --git a/DysonNetwork.sln.DotSettings.user b/DysonNetwork.sln.DotSettings.user index 2630216..98f1329 100644 --- a/DysonNetwork.sln.DotSettings.user +++ b/DysonNetwork.sln.DotSettings.user @@ -65,7 +65,10 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded