diff --git a/src/Luban.Core/BuiltinOptionNames.cs b/src/Luban.Core/BuiltinOptionNames.cs index 13c69768..3b0f3489 100644 --- a/src/Luban.Core/BuiltinOptionNames.cs +++ b/src/Luban.Core/BuiltinOptionNames.cs @@ -24,13 +24,19 @@ public static class BuiltinOptionNames public const string L10NFamily = "l10n"; - public const string TextProviderName = "textProviderName"; + public const string L10NProviderName = "provider"; - public const string TextProviderFile = "textProviderFile"; + public const string L10NTextFilePath = "textFile.path"; - public const string TextKeyListFile = "textListFile"; + public const string L10NTextFileKeyFieldName = "textFile.keyFieldName"; - public const string TextKeyFieldName = "key"; + public const string L10NTextFileLanguageFieldName = "textFile.languageFieldName"; + + public const string L10NConvertTextKeyToValue = "convertTextKeyToValue"; + + //public const string L10NUnknownTextKeyListOutputFile = "unknownTextKeyListOutputFile"; + + public const string L10NTextListFile = "textListFile"; public const string TypeMapperType = "type"; diff --git a/src/Luban.Core/DataTransformer/DataTransfomerBase.cs b/src/Luban.Core/DataTransformer/DataTransfomerBase.cs new file mode 100644 index 00000000..0748e7c9 --- /dev/null +++ b/src/Luban.Core/DataTransformer/DataTransfomerBase.cs @@ -0,0 +1,209 @@ +using Luban.Datas; +using Luban.DataVisitors; +using Luban.Types; +using NLog.LayoutRenderers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.AccessControl; +using System.Text; +using System.Threading.Tasks; + +namespace Luban.DataTransformer; + +public abstract class DataTransfomerBase : IDataTransformer, IDataFuncVisitor2 +{ + public DType Transform(DType originalData, TType type) + { + if (originalData == null) + { + return null; + } + return originalData.Apply(this, type); + } + + DType IDataFuncVisitor2.Accept(DBool data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DByte data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DShort data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DInt data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DLong data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DFloat data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DDouble data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DEnum data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DString data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DDateTime data, TType type) + { + return data; + } + + DType IDataFuncVisitor2.Accept(DBean data, TType type) + { + var defFields = data.ImplType.HierarchyFields; + int i = 0; + List newFields = null; + foreach (var fieldValue in data.Fields) + { + if (fieldValue == null) + { + i++; + continue; + } + var defField = defFields[i]; + var fieldType = defField.CType; + DType newFieldValue = fieldValue.Apply(this, fieldType); + if (newFieldValue != fieldValue) + { + if (newFields == null) + { + newFields = new List(data.Fields); + } + newFields[i] = newFieldValue; + } + ++i; + } + return newFields == null ? data : new DBean(data.TType, data.ImplType, newFields); + } + + DType IDataFuncVisitor2.Accept(DArray data, TType type) + { + TType eleType = type.ElementType; + List newDatas = null; + int index = 0; + foreach (var ele in data.Datas) + { + if (ele == null) + { + ++index; + continue; + } + DType newEle = ele.Apply(this, eleType); + if (newEle != ele) + { + if (newDatas == null) + { + newDatas = new List(data.Datas); + } + newDatas[index] = newEle; + } + ++index; + } + return newDatas == null ? data : new DArray(data.Type, newDatas); + } + + DType IDataFuncVisitor2.Accept(DList data, TType type) + { + TType eleType = type.ElementType; + List newDatas = null; + int index = 0; + foreach (var ele in data.Datas) + { + if (ele == null) + { + ++index; + continue; + } + DType newEle = ele.Apply(this, eleType); + if (newEle != ele) + { + if (newDatas == null) + { + newDatas = new List(data.Datas); + } + newDatas[index] = newEle; + } + ++index; + } + return newDatas == null ? data : new DList(data.Type, newDatas); + } + + DType IDataFuncVisitor2.Accept(DSet data, TType type) + { + TType eleType = type.ElementType; + List newDatas = null; + int index = 0; + foreach (var ele in data.Datas) + { + if (ele == null) + { + ++index; + continue; + } + DType newEle = ele.Apply(this, eleType); + if (newEle != ele) + { + if (newDatas == null) + { + newDatas = new List(data.Datas); + } + newDatas[index] = newEle; + } + ++index; + } + return newDatas == null ? data : new DSet(data.Type, newDatas); + } + + DType IDataFuncVisitor2.Accept(DMap data, TType type) + { + TMap mapType = (TMap)type; + bool dirty = false; + foreach (var ele in data.Datas) + { + DType newKey = ele.Key.Apply(this, mapType.KeyType); + DType newValue = ele.Value.Apply(this, mapType.ValueType); + if (newKey != ele.Key || newValue != ele.Value) + { + dirty = true; + break; + } + } + if (!dirty) + { + return data; + } + + var newDatas = new Dictionary(); + foreach (var ele in data.Datas) + { + DType newKey = ele.Key.Apply(this, mapType.KeyType); + DType newValue = ele.Value.Apply(this, mapType.ValueType); + newDatas[newKey] = newValue; + } + return new DMap(data.Type, newDatas); + } +} diff --git a/src/Luban.Core/DataTransformer/DataTransformerAttribute.cs b/src/Luban.Core/DataTransformer/DataTransformerAttribute.cs new file mode 100644 index 00000000..727b2dc2 --- /dev/null +++ b/src/Luban.Core/DataTransformer/DataTransformerAttribute.cs @@ -0,0 +1,16 @@ +using Luban.CustomBehaviour; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Luban.DataTransformer; + +[AttributeUsage(AttributeTargets.Class, Inherited = false, AllowMultiple = false)] +public class DataTransformerAttribute : BehaviourBaseAttribute +{ + public DataTransformerAttribute(string name) : base(name) + { + } +} diff --git a/src/Luban.Core/DataTransformer/IDataTransformer.cs b/src/Luban.Core/DataTransformer/IDataTransformer.cs new file mode 100644 index 00000000..88be9f03 --- /dev/null +++ b/src/Luban.Core/DataTransformer/IDataTransformer.cs @@ -0,0 +1,14 @@ +using Luban.Datas; +using Luban.Types; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Luban.DataTransformer; + +public interface IDataTransformer +{ + DType Transform(DType originalData, TType type); +} diff --git a/src/Luban.Core/DataVisitors/DataActionHelpVisitor2.cs b/src/Luban.Core/DataVisitors/DataActionHelpVisitor2.cs index 90679e65..69244c51 100644 --- a/src/Luban.Core/DataVisitors/DataActionHelpVisitor2.cs +++ b/src/Luban.Core/DataVisitors/DataActionHelpVisitor2.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using static System.Runtime.InteropServices.JavaScript.JSType; namespace Luban.DataVisitors; diff --git a/src/Luban.Core/Defs/Record.cs b/src/Luban.Core/Defs/Record.cs index 4ec87bbf..5ec59f6b 100644 --- a/src/Luban.Core/Defs/Record.cs +++ b/src/Luban.Core/Defs/Record.cs @@ -6,7 +6,7 @@ public class Record { public int AutoIndex { get; set; } - public DBean Data { get; } + public DBean Data { get; set; } public string Source { get; } diff --git a/src/Luban.Core/GenerationContext.cs b/src/Luban.Core/GenerationContext.cs index 69859293..71f954f2 100644 --- a/src/Luban.Core/GenerationContext.cs +++ b/src/Luban.Core/GenerationContext.cs @@ -5,6 +5,7 @@ using Luban.DataLoader; using Luban.Datas; using Luban.Defs; +using Luban.L10N; using Luban.RawDefs; using Luban.Schema; using Luban.Types; @@ -59,6 +60,8 @@ public class GenerationContext public TimeZoneInfo TimeZone { get; private set; } + public ITextProvider TextProvider { get; private set; } + private readonly Dictionary _uniqueObjects = new(); private readonly HashSet _failedValidatorTypes = new(); @@ -66,6 +69,7 @@ public class GenerationContext public void LoadDatas() { s_logger.Info("load datas begin"); + TextProvider?.Load(); DataLoaderManager.Ins.LoadDatas(this); s_logger.Info("load datas end"); } @@ -82,6 +86,9 @@ public void Init(GenerationContextBuilder builder) ExcludeTags = builder.ExcludeTags; TimeZone = TimeZoneUtil.GetTimeZone(builder.TimeZone); + TextProvider = EnvManager.Current.TryGetOption(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.L10NProviderName, false, out string providerName) ? + L10NManager.Ins.CreateTextProvider(providerName) : null; + ExportTables = Assembly.ExportTables; ExportTypes = CalculateExportTypes(); ExportBeans = ExportTypes.OfType().ToList(); @@ -90,7 +97,7 @@ public void Init(GenerationContextBuilder builder) private bool NeedExportNotDefault(List groups) { - return groups.Any(g => Target.Groups.Contains(g)); + return groups.Any(Target.Groups.Contains); } private List CalculateExportTypes() diff --git a/src/Luban.Core/L10N/ITextProvider.cs b/src/Luban.Core/L10N/ITextProvider.cs index 5fb5f8f5..997afddd 100644 --- a/src/Luban.Core/L10N/ITextProvider.cs +++ b/src/Luban.Core/L10N/ITextProvider.cs @@ -2,11 +2,15 @@ namespace Luban.L10N; public interface ITextProvider { - bool Enable { get; } - void Load(); + void ProcessDatas(); + bool IsValidKey(string key); - string GetText(string key, string language); + bool TryGetText(string key, out string text); + + void AddUnknownKey(string key); + + bool ConvertTextKeyToValue { get; } } diff --git a/src/Luban.Core/L10N/L10NManager.cs b/src/Luban.Core/L10N/L10NManager.cs index c18e1e8d..8ad4a34c 100644 --- a/src/Luban.Core/L10N/L10NManager.cs +++ b/src/Luban.Core/L10N/L10NManager.cs @@ -13,13 +13,6 @@ public void Init() public ITextProvider CreateTextProvider(string name) { - ITextProvider provider = CustomBehaviourManager.Ins.CreateBehaviour(name); - provider.Load(); - return provider; - } - - public ITextProvider GetOrCreateContextUniqueTextProvider(string name) - { - return (ITextProvider)GenerationContext.Current.GetOrAddUniqueObject($"{BuiltinOptionNames.TextProviderName}.{name}", () => CreateTextProvider(name)); + return CustomBehaviourManager.Ins.CreateBehaviour(name); } } diff --git a/src/Luban.Core/Pipeline/DefaultPipeline.cs b/src/Luban.Core/Pipeline/DefaultPipeline.cs index 46f64375..1bc87eb8 100644 --- a/src/Luban.Core/Pipeline/DefaultPipeline.cs +++ b/src/Luban.Core/Pipeline/DefaultPipeline.cs @@ -1,6 +1,7 @@ using Luban.CodeTarget; using Luban.DataTarget; using Luban.Defs; +using Luban.L10N; using Luban.OutputSaver; using Luban.PostProcess; using Luban.RawDefs; @@ -69,6 +70,7 @@ protected void LoadDatas() { _genCtx.LoadDatas(); DoValidate(); + ProcessL10N(); } protected void DoValidate() @@ -79,6 +81,14 @@ protected void DoValidate() s_logger.Info("validation end"); } + protected void ProcessL10N() + { + if (_genCtx.TextProvider != null) + { + _genCtx.TextProvider.ProcessDatas(); + } + } + protected void ProcessTargets() { var tasks = new List(); diff --git a/src/Luban.Core/Utils/DataUtil.cs b/src/Luban.Core/Utils/DataUtil.cs index d1fce1bd..4244ed10 100644 --- a/src/Luban.Core/Utils/DataUtil.cs +++ b/src/Luban.Core/Utils/DataUtil.cs @@ -7,6 +7,22 @@ namespace Luban.Utils; public static class DataUtil { + public static bool ParseBool(string s) + { + s = s.ToLower().Trim(); + switch (s) + { + case "true": + case "1": + return true; + case "false": + case "0": + return false; + default: + throw new Exception($"{s} 不是 bool 类型的值 (true|1 或 false|0)"); + } + } + private static readonly string[] dateTimeFormats = new string[] { "yyyy-M-d HH:mm:ss", "yyyy-M-d HH:mm", "yyyy-M-d HH", "yyyy-M-d", //"yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyy/MM/dd HH", "yyyy/MM/dd", diff --git a/src/Luban.L10N/DataTarget/TextKeyListDataTarget.cs b/src/Luban.L10N/DataTarget/TextKeyListDataTarget.cs index b4411548..2b593f14 100644 --- a/src/Luban.L10N/DataTarget/TextKeyListDataTarget.cs +++ b/src/Luban.L10N/DataTarget/TextKeyListDataTarget.cs @@ -33,7 +33,7 @@ public override OutputFile ExportTables(List tables) keys.Sort((a, b) => string.Compare(a, b, StringComparison.Ordinal)); var content = string.Join("\n", keys); - string outputFile = EnvManager.Current.GetOption(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.TextKeyListFile, false); + string outputFile = EnvManager.Current.GetOption(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.L10NTextListFile, false); return new OutputFile { File = outputFile, Content = content }; } diff --git a/src/Luban.L10N/DefaultTextProvider.cs b/src/Luban.L10N/DefaultTextProvider.cs index 53ed15d5..e0ca53b6 100644 --- a/src/Luban.L10N/DefaultTextProvider.cs +++ b/src/Luban.L10N/DefaultTextProvider.cs @@ -4,6 +4,7 @@ using Luban.RawDefs; using Luban.Types; using Luban.Utils; +using System.Linq; namespace Luban.L10N; @@ -12,34 +13,49 @@ public class DefaultTextProvider : ITextProvider { private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); - private bool _enableTextValidation; private string _keyFieldName; - private readonly HashSet _keys = new(); + private string _ValueFieldName; - public bool Enable => _enableTextValidation; + private bool _convertTextKeyToValue; + + private readonly Dictionary _texts = new(); + + private readonly HashSet _unknownTextKeys = new(); public void Load() { - if (!EnvManager.Current.TryGetOption(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.TextProviderFile, true, - out string textProviderFile)) + EnvManager env = EnvManager.Current; + + _keyFieldName = env.GetOptionOrDefault(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.L10NTextFileKeyFieldName, false, ""); + if (string.IsNullOrWhiteSpace(_keyFieldName)) { - s_logger.Warn("option: '-x {0}.{1}=' not found, text validation is disabled", BuiltinOptionNames.L10NFamily, BuiltinOptionNames.TextProviderFile); - _enableTextValidation = false; - return; + throw new Exception($"'-x {BuiltinOptionNames.L10NFamily}.{BuiltinOptionNames.L10NTextFileKeyFieldName}=xxx' missing"); + } + + _convertTextKeyToValue = DataUtil.ParseBool(env.GetOptionOrDefault(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.L10NConvertTextKeyToValue, false, "false")); + if (_convertTextKeyToValue) + { + _ValueFieldName = env.GetOptionOrDefault(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.L10NTextFileLanguageFieldName, false, ""); + if (string.IsNullOrWhiteSpace(_ValueFieldName)) + { + throw new Exception($"'-x {BuiltinOptionNames.L10NFamily}.{BuiltinOptionNames.L10NTextFileLanguageFieldName}=xxx' missing"); + } } - _keyFieldName = EnvManager.Current.GetOptionOrDefault(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.TextKeyFieldName, false, "key"); + + string textProviderFile = env.GetOption(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.L10NTextFilePath, false); LoadTextListFromFile(textProviderFile); - _enableTextValidation = true; } + public bool ConvertTextKeyToValue => _convertTextKeyToValue; + public bool IsValidKey(string key) { - return _keys.Contains(key); + return _texts.ContainsKey(key); } - public string GetText(string key, string language) + public bool TryGetText(string key, out string text) { - throw new NotSupportedException("default text provider not support get text"); + return _texts.TryGetValue(key, out text); } private void LoadTextListFromFile(string fileName) @@ -49,6 +65,12 @@ private void LoadTextListFromFile(string fileName) Targets = new List { new() { Name = "default", Manager = "Tables" } }, }, "default", new List()); + + var rawFields = new List { new() { Name = _keyFieldName, Type = "string" }, }; + if (_convertTextKeyToValue) + { + rawFields.Add(new() { Name = _ValueFieldName, Type = "string" }); + } var defTableRecordType = new DefBean(new RawBean() { Namespace = "__intern__", @@ -57,14 +79,12 @@ private void LoadTextListFromFile(string fileName) Alias = "", IsValueType = false, Sep = "", - Fields = new List - { - new() { Name = _keyFieldName, Type = "string" }, - } + Fields = rawFields, }) { Assembly = ass, }; + ass.AddType(defTableRecordType); defTableRecordType.PreCompile(); defTableRecordType.Compile(); @@ -79,10 +99,36 @@ private void LoadTextListFromFile(string fileName) DBean data = r.Data; string key = ((DString)data.GetField(_keyFieldName)).Value; - if (!_keys.Add(key)) + string value = _convertTextKeyToValue ? ((DString)data.GetField(_ValueFieldName)).Value : key; + if (string.IsNullOrEmpty(key)) + { + s_logger.Error("textFile:{} key:{} is empty. ignore it!", fileName, key); + continue; + } + if (!_texts.TryAdd(key, value)) { - s_logger.Warn("textProviderFile:{} key:{} duplicated", fileName, key); + s_logger.Error("textFile:{} key:{} is duplicated", fileName, key); } }; } + + public void AddUnknownKey(string key) + { + _unknownTextKeys.Add(key); + } + + public void ProcessDatas() + { + if (_convertTextKeyToValue) + { + var trans = new TextKeyToValueTransformer(this); + foreach (var table in GenerationContext.Current.Tables) + { + foreach (var record in GenerationContext.Current.GetTableAllDataList(table)) + { + record.Data = (DBean)record.Data.Apply(trans,table.ValueTType); + } + } + } + } } diff --git a/src/Luban.L10N/TextKeyToValueTransformer.cs b/src/Luban.L10N/TextKeyToValueTransformer.cs new file mode 100644 index 00000000..fcbcc7b7 --- /dev/null +++ b/src/Luban.L10N/TextKeyToValueTransformer.cs @@ -0,0 +1,39 @@ +using Luban.Datas; +using Luban.DataTransformer; +using Luban.DataVisitors; +using Luban.Types; +using Luban.Utils; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Luban.L10N; + +public class TextKeyToValueTransformer : DataTransfomerBase, IDataFuncVisitor2 +{ + private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); + + private readonly ITextProvider _provider; + + public TextKeyToValueTransformer(ITextProvider provider) + { + _provider = provider; + } + + DType IDataFuncVisitor2.Accept(DString data, TType type) + { + if(string.IsNullOrEmpty(data.Value) || !type.HasTag("text")) + { + return data; + } + if (_provider.TryGetText(data.Value, out var text)) + { + return DString.ValueOf(type, text); + } + s_logger.Error("can't find target language text of text id:{} ", data.Value); + //_provider.AddUnknownKey(data.Value); + return data; + } +} diff --git a/src/Luban.L10N/TextValidator.cs b/src/Luban.L10N/TextValidator.cs index b33d1c06..1ae02f41 100644 --- a/src/Luban.L10N/TextValidator.cs +++ b/src/Luban.L10N/TextValidator.cs @@ -1,6 +1,7 @@ using Luban.Datas; using Luban.Defs; using Luban.Types; +using Luban.Utils; using Luban.Validator; namespace Luban.L10N; @@ -10,38 +11,28 @@ public class TextValidator : DataValidatorBase { private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); - private ITextProvider _provider; - - private ITextProvider Provider - { - get - { - if (_provider == null) - { - string textProviderName = EnvManager.Current.GetOptionOrDefault(BuiltinOptionNames.L10NFamily, BuiltinOptionNames.TextProviderName, false, "default"); - _provider = L10NManager.Ins.GetOrCreateContextUniqueTextProvider(textProviderName); - } - return _provider; - } - } - public override void Compile(DefField field, TType type) { if (type is not TString) { throw new Exception($"field:{field} text validator supports string type only"); } - } public override void Validate(DataValidatorContext ctx, TType type, DType data) { + ITextProvider provider = GenerationContext.Current.TextProvider; + // dont' check when convertTextKeyToValue is true + if (provider == null || provider.ConvertTextKeyToValue) + { + return; + } string key = ((DString)data).Value; if (string.IsNullOrEmpty(key)) { return; } - if (Provider.Enable && !Provider.IsValidKey(key)) + if (!provider.IsValidKey(key)) { s_logger.Error("记录 {}:{} (来自文件:{}) 不是一个有效的文本key", DataValidatorContext.CurrentRecordPath, data, Source); GenerationContext.Current.LogValidatorFail(this);