8000 feat: description generator of op code. · SciSharp/TensorFlow.NET@c1b6731 · GitHub
[go: up one dir, main page]
Skip to content

Commit c1b6731

Browse files
AsakusaRinneOceania2018
AsakusaRinne
authored and
Oceania2018
committed
feat: description generator of op code.
1 parent 6c651c9 commit c1b6731

File tree

6 files changed

+482
-212
lines changed

6 files changed

+482
-212
lines changed

‎Tensorflow.CodeGen/DescriptionGenerator.cs

Lines changed: 263 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,263 @@
1+
using Microsoft.CodeAnalysis.CSharp;
2+
using Protobuf.Text;
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Linq;
6+
using System.Reflection.Metadata.Ecma335;
7+
using System.Text;
8+
using System.Text.RegularExpressions;
9+
using System.Threading.Tasks;
10+
11+
namespace Tensorflow.CodeGen
12+
{
13+
public class DescriptionGenerator
14+
{
15+
private static readonly string replaceStrInner = "~~%~~";
16+
private static readonly string replaceStrInnerQuotationMarks = "^%^";
17+
Dictionary<string, Dictionary<string, string>> _opDescriptions = new Dictionary<string, Dictionary<string, string>>();
18+
Dictionary<string, OpDef> _opDescriptionDefs = new Dictionary<string, OpDef>();
19+
public DescriptionGenerator(string apiDefDirectory)
20+
{
21+
DirectoryInfo directory = new DirectoryInfo(apiDefDirectory);
22+
23+
int errors = 0;
24+
foreach (FileInfo file in directory.GetFiles())
25+
{
26+
string target = file.Name.Split('.')[0].Split('_').Last();
27+
OpDef op = null;
28+
try
29+
{
30+
op = ReadOpDefs(file.FullName).Op[0];
31+
}
32+
catch
33+
{
34+
errors++;
35+
continue;
36+
}
37+
_opDescriptionDefs[target] = op;
38+
_opDescriptions[target] = new Dictionary<string, string>();
39+
foreach (var arg in op.InputArg)
40+
{
41+
string argName = arg.Name;
42+
var token = SyntaxFactory.ParseToken(argName);
43+
if (token.IsKeyword())
44+
{
45+
argName = $"{argName}_";
46+
}
47+
_opDescriptions[target][argName] = arg.Description ?? "";
48+
}
49+
foreach (var arg in op.Attr)
50+
{
51+
var token = SyntaxFactory.ParseToken(arg.Name);
52+
string realKey = arg.Name;
53+
if (token.IsKeyword())
54+
{
55+
realKey += "_";
56+
}
57+
_opDescriptions[target][realKey] = arg.Description ?? "";
58+
}
59+
_opDescriptions[target]["SUMMARY"] = op.Summary ?? "";
60+
_opDescriptions[target]["DESC"] = op.Description ?? "";
61+
}
62+
Console.Writ 7802 eLine($"Warning: {errors} description files cannot be analyzed! Please revise it if " +
63+
$"the failed files number is large, or ignore it.");
64+
}
65+
66+
/// <summary>
67+
///
68+
/// </summary>
69+
/// <param name="op"></param>
70+
/// <param name="sb"></param>
71+
public void AppendDescription(OpDef fullOp, StringBuilder sb)
72+
{
73+
var opName = fullOp.Name;
74+
if(_opDescriptions.TryGetValue(opName, out var op))
75+
{
76+
var def = _opDescriptionDefs[opName];
77+
sb.AppendLine("/// <summary>");
78+
sb.AppendLine($"/// {op["SUMMARY"]}");
79+
sb.AppendLine("/// </summary>");
80+
81+
string totalDesc = op["DESC"];
82+
if (!string.IsNullOrEmpty(totalDesc))
83+
{
84+
totalDesc = totalDesc.Replace(replaceStrInnerQuotationMarks, "\"");
85+
sb.AppendLine("/// <remarks>");
86+
string[] lines = totalDesc.Split(replaceStrInner);
87+
foreach (var line in lines)
88+
{
89+
sb.AppendLine($"/// {line}");
90+
}
91+
sb.AppendLine("/// </remarks>");
92+
}
93+
94+
var argNames = GetInputArgNames(fullOp);
95+
foreach (var argName in argNames)
96+
{
97+
if(op.TryGetValue(argName, out var desc))
98+
{
99+
desc = desc.Replace(replaceStrInnerQuotationMarks, "\"");
100+
string[] lines = desc.Split(replaceStrInner);
101+
sb.AppendLine($"/// <param name=\"{argName}\">");
102+
foreach (var line in lines)
103+
{
104+
sb.AppendLine($"/// {line}");
105+
}
106+
sb.AppendLine("/// </param>");
107+
}
108+
else
109+
{
110+
sb.AppendLine($"/// <param name=\"{argName}\"></param>");
111+
}
112+
}
113+
114+
List<string> returnValueDescs = new();
115+
foreach (var arg in def.OutputArg)
116+
{
117+
if (!string.IsNullOrEmpty(arg.Description))
118+
{
119+
returnValueDescs.Add($"{arg.Name}: {arg.Description}");
120+
}
121+
}
122+
string returnValueDesc = "";
123+
if (returnValueDescs.Count > 0)
124+
{
125+
returnValueDesc = string.Join(" && ", returnValueDescs);
126+
}
127+
sb.AppendLine($"/// <returns>{returnValueDesc}</returns>");
128+
}
129+
else
130+
{
131+
sb.AppendLine("/// <summary>");
132+
sb.AppendLine($"///");
133+
sb.AppendLine("/// </summary>");
134+
135+
var argNames = GetInputArgNames(fullOp);
136+
foreach (var argName in argNames)
137+
{
138+
sb.AppendLine($"/// <param name=\"{argName}\"></param>");
139+
}
140+
141+
sb.AppendLine($"/// <returns></returns>");
142+
}
143+
}
144+
145+
/// <summary>
146+
///
147+
/// </summary>
148+
/// <param name="op">
149+
/// </param>
150+
/// <returns></returns>
151+
/// <remarks></remarks>
152+
public List<string> GetInputArgNames(OpDef op)
153+
{
154+
List<string> names = new();
155+
foreach (var arg in op.InputArg)
156+
{
157+
string argName = arg.Name;
158+
var token = SyntaxFactory.ParseToken(argName);
159+
if (token.IsKeyword())
160+
{
161+
argName = $"{argName}_";
162+
}
163+
names.Add(argName);
164+
}
165+
var attrValueDic = Utils.GetAttrsDefaultValue(op, out var dynamicDefaultValues);
166+
foreach (var (key, typeStr, value) in attrValueDic)
167+
{
168+
var token = SyntaxFactory.ParseToken(key);
169+
string realKey = key;
170+
if (token.IsKeyword())
171+
{
172+
realKey += "_";
173+
}
174+
names.Add(realKey);
175+
}
176+
return names;
177+
}
178+
179+
private static OpList ReadOpDefs(string path)
180+
{
181+
var text = File.ReadAllText(path);
182+
text = RemoveLintTags(text);
183+
text = PreProcessText(text);
184+
185+
string pattern = @"<<END([\s\S]*?)END";
186+
187+
// 定义用于替换的字符串
188+
string replaceStrPrefix = "\"";
189+
string replaceStrSuffix = "\"";
190+
191+
// 将匹配到的文本段全部替换
192+
string replacedText = Regex.Replace(text, pattern, match => {
193+
string matchedText = match.Value;
194+
string innerText = match.Groups[1].Value;
195+
innerText = innerText.Replace("\"", replaceStrInnerQuotationMarks)
196+
.Replace("\r\n", replaceStrInner).Replace("\n", replaceStrInner); // 替换内部换行符
197+
return replaceStrPrefix + innerText + replaceStrSuffix; // 替换首尾
198+
}, RegexOptions.Multiline);
199+
200+
var opDefs = new TextParser(TextParser.Settings.Default.WithIgnoreUnknownFields(true)).Parse<OpList>(replacedText);
201+
return opDefs;
202+
}
203+
204+
static string PreProcessText(string input)
205+
{
206+
int depth = 0;
207+
int endBlockDepth = -1;
208+
StringBuilder sb = new StringBuilder();
209+
for (int i = 0; i < input.Length; i++)
210+
{
211+
char c = input[i];
212+
if (c == '{')
213+
{
214+
depth++;
215+
sb.Append(c);
216+
}
217+
else if (c == '}')
218+
{
219+
if (depth == endBlockDepth)
220+
{
221+
sb.Append("END\n");
222+
endBlockDepth = -1;
223+
}
224+
sb.Append(c);
225+
depth--;
226+
}
227+
else if (c == '<' && i + 5 < input.Length && input.Substring(i, 5) == "<<END")
228+
{
229+
endBlockDepth = depth;
230+
sb.Append("<<END");
231+
i += 4;
232+
}
233+
else if (c == 'E' && i + 3 < input.Length && input.Substring(i, 3) == "END")
234+
{
235+
endBlockDepth = -1;
236+
sb.Append("END");
237+
i += 2;
238+
}
239+
else
240+
{
241+
sb.Append(c);
242+
}
243+
}
244+
245+
string output = sb.ToString();
246+
return output;
247+
}
248+
249+
static string RemoveLintTags(string input)
250+
{
251+
string[] lines = input.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None);
252+
StringBuilder sb = new StringBuilder();
253+
foreach (string line in lines)
254+
{
255+
if (!line.TrimStart().StartsWith("# LINT"))
256+
{
257+
sb.AppendLine(line);
258+
}
259+
}
260+
return sb.ToString().TrimEnd();
261+
}
262+
}
263+
}

0 commit comments

Comments
 (0)
0