Skip to content

Commit e61bc7c

Browse files
TalZaccaiTalZaccai
authored
Commands Info/Docs Update (#913)
* Some small changes to CommandInfoUpdater tool + updating CommandsInfo/Docs JSON files * format * test fix * fix * Added command metadata coverage tests * added comment
1 parent 9f90803 commit e61bc7c

8 files changed

+1575
-561
lines changed

‎libs/resources/RespCommandsDocs.json

+640-332
Large diffs are not rendered by default.

‎libs/resources/RespCommandsInfo.json

+164-165
Large diffs are not rendered by default.

‎playground/CommandInfoUpdater/CommandDocsUpdater.cs

+26-23
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ namespace CommandInfoUpdater
1111
{
1212
public class CommandDocsUpdater
1313
{
14+
const int QUERY_CMD_BATCH_SIZE = 25;
1415
private static readonly string CommandDocsFileName = "RespCommandsDocs.json";
1516
private static readonly string GarnetCommandDocsJsonPath = "GarnetCommandsDocs.json";
1617

@@ -59,36 +60,43 @@ public static bool TryUpdateCommandDocs(string outputDir, int respServerPort, IP
5960
IDictionary<string, RespCommandDocs> queriedCommandsDocs = new Dictionary<string, RespCommandDocs>();
6061
var commandsToQuery = commandsToAdd.Keys.Select(k => k.Command)
6162
.Where(c => updatedCommandsInfo.ContainsKey(c) && !updatedCommandsInfo[c].IsInternal).ToArray();
62-
if (commandsToQuery.Length > 0 && !TryGetCommandsDocs(commandsToQuery, respServerPort, respServerHost,
63-
logger, out queriedCommandsDocs))
63+
if (commandsToQuery.Length > 0)
6464
{
65-
logger.LogError("Unable to get RESP command docs from local RESP server.");
66-
return false;
65+
for (var i = 0; i < commandsToQuery.Length; i += QUERY_CMD_BATCH_SIZE)
66+
{
67+
var batchToQuery = commandsToQuery.Skip(i).Take(QUERY_CMD_BATCH_SIZE).ToArray();
68+
if (!TryGetCommandsDocs(batchToQuery, respServerPort, respServerHost,
69+
logger, ref queriedCommandsDocs))
70+
{
71+
logger.LogError("Unable to get RESP command info from local RESP server.");
72+
return false;
73+
}
74+
}
6775
}
6876

6977
var additionalCommandsDocs = new Dictionary<string, RespCommandDocs>();
7078
foreach (var cmd in garnetCommandsDocs.Keys.Union(queriedCommandsDocs.Keys))
7179
{
7280
if (!additionalCommandsDocs.ContainsKey(cmd))
7381
{
74-
var baseCommandDocs = queriedCommandsDocs.TryGetValue(cmd, out var doc)
75-
? doc
76-
: garnetCommandsDocs[cmd];
82+
var inQueried = queriedCommandsDocs.TryGetValue(cmd, out var queriedCommandDocs);
83+
var inGarnet = garnetCommandsDocs.TryGetValue(cmd, out var garnetCommandDocs);
84+
var baseCommandDocs = inGarnet ? garnetCommandDocs : queriedCommandDocs;
7785

7886
RespCommandDocs[] subCommandsDocs;
79-
if (garnetCommandsDocs.ContainsKey(cmd) && queriedCommandsDocs.ContainsKey(cmd))
87+
if (inQueried && inGarnet)
8088
{
8189
var subCommandsInfoMap = new Dictionary<string, RespCommandDocs>();
8290

83-
if (garnetCommandsDocs.TryGetValue(cmd, out var garnetCmdDocs) && garnetCmdDocs.SubCommands != null)
91+
if (garnetCommandDocs.SubCommands != null)
8492
{
85-
foreach (var sc in garnetCmdDocs.SubCommands)
93+
foreach (var sc in garnetCommandDocs.SubCommands)
8694
subCommandsInfoMap.Add(sc.Name, sc);
8795
}
8896

89-
if (queriedCommandsDocs.TryGetValue(cmd, out var queriedCmdDocs) && queriedCmdDocs.SubCommands != null)
97+
if (queriedCommandDocs.SubCommands != null)
9098
{
91-
foreach (var sc in queriedCmdDocs.SubCommands)
99+
foreach (var sc in queriedCommandDocs.SubCommands)
92100
{
93101
subCommandsInfoMap.TryAdd(sc.Name, sc);
94102
}
@@ -134,15 +142,11 @@ public static bool TryUpdateCommandDocs(string outputDir, int respServerPort, IP
134142
/// <param name="commandDocs">Queried commands docs</param>
135143
/// <returns>True if succeeded</returns>
136144
private static unsafe bool TryGetCommandsDocs(string[] commandsToQuery, int respServerPort,
137-
IPAddress respServerHost, ILogger logger, out IDictionary<string, RespCommandDocs> commandDocs)
145+
IPAddress respServerHost, ILogger logger, ref IDictionary<string, RespCommandDocs> commandDocs)
138146
{
139-
commandDocs = default;
140-
141147
// If there are no commands to query, return
142148
if (commandsToQuery.Length == 0) return true;
143149

144-
var tmpCommandsDocs = new Dictionary<string, RespCommandDocs>();
145-
146150
// Get a map of supported commands to Garnet's RespCommand & ArrayCommand for the parser
147151
var supportedCommands = new ReadOnlyDictionary<string, RespCommand>(
148152
SupportedCommand.SupportedCommandsFlattenedMap.ToDictionary(kvp => kvp.Key,
@@ -171,10 +175,9 @@ private static unsafe bool TryGetCommandsDocs(string[] commandsToQuery, int resp
171175
return false;
172176
}
173177

174-
tmpCommandsDocs.Add(cmdName, cmdDocs);
178+
commandDocs.Add(cmdName, cmdDocs);
175179
}
176180

177-
commandDocs = tmpCommandsDocs;
178181
return true;
179182
}
180183

@@ -212,7 +215,7 @@ private static IReadOnlyDictionary<string, RespCommandDocs> GetUpdatedCommandsDo
212215
: existingCommandsDocs[command.Command].SubCommands.Select(sc => sc.Name).ToArray();
213216
var remainingSubCommands = existingSubCommands == null ? null :
214217
command.SubCommands == null ? existingSubCommands :
215-
existingSubCommands.Except(command.SubCommands.Keys).ToArray();
218+
[.. existingSubCommands.Except(command.SubCommands.Keys)];
216219

217220
// Create updated command docs based on existing command
218221
var existingCommandDoc = existingCommandsDocs[command.Command];
@@ -227,7 +230,7 @@ private static IReadOnlyDictionary<string, RespCommandDocs> GetUpdatedCommandsDo
227230
existingCommandDoc.Arguments,
228231
remainingSubCommands == null || remainingSubCommands.Length == 0
229232
? null
230-
: existingCommandDoc.SubCommands.Where(sc => remainingSubCommands.Contains(sc.Name)).ToArray());
233+
: [.. existingCommandDoc.SubCommands.Where(sc => remainingSubCommands.Contains(sc.Name))]);
231234

232235
updatedCommandsDocs.Add(updatedCommandDoc.Name, updatedCommandDoc);
233236
}
@@ -267,7 +270,7 @@ private static IReadOnlyDictionary<string, RespCommandDocs> GetUpdatedCommandsDo
267270
// Update sub-commands to contain supported sub-commands only
268271
updatedSubCommandsDocs = command.SubCommands == null
269272
? null
270-
: baseCommandDocs.SubCommands.Where(sc => command.SubCommands.Keys.Contains(sc.Name)).ToList();
273+
: [.. baseCommandDocs.SubCommands.Where(sc => command.SubCommands.Keys.Contains(sc.Name))];
271274
}
272275

273276
// Create updated command docs based on base command & updated sub-commands
@@ -280,7 +283,7 @@ private static IReadOnlyDictionary<string, RespCommandDocs> GetUpdatedCommandsDo
280283
baseCommandDocs.DocFlags,
281284
baseCommandDocs.ReplacedBy,
282285
baseCommandDocs.Arguments,
283-
updatedSubCommandsDocs?.ToArray());
286+
updatedSubCommandsDocs?.OrderBy(sc => sc.Name).ToArray());
284287

285288
updatedCommandsDocs.Add(updatedCommandDocs.Name, updatedCommandDocs);
286289
}

‎playground/CommandInfoUpdater/CommandInfoUpdater.cs

+27-23
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ namespace CommandInfoUpdater
1515
/// </summary>
1616
public class CommandInfoUpdater
1717
{
18+
const int QUERY_CMD_BATCH_SIZE = 25;
1819
private static readonly string CommandInfoFileName = "RespCommandsInfo.json";
1920
private static readonly string GarnetCommandInfoJsonPath = "GarnetCommandsInfo.json";
2021

@@ -63,35 +64,43 @@ public static bool TryUpdateCommandInfo(string outputDir, int respServerPort, IP
6364
var commandsToQuery = commandsToAdd.Keys.Select(k => k.Command)
6465
.Where(c => !garnetCommandsInfo.ContainsKey(c) || !garnetCommandsInfo[c].IsInternal).ToArray();
6566

66-
if (commandsToQuery.Length > 0 && !TryGetCommandsInfo(commandsToQuery, respServerPort, respServerHost,
67-
logger, out queriedCommandsInfo))
67+
if (commandsToQuery.Length > 0)
6868
{
69-
logger.LogError("Unable to get RESP command info from local RESP server.");
70-
return false;
69+
for (var i = 0; i < commandsToQuery.Length; i += QUERY_CMD_BATCH_SIZE)
70+
{
71+
var batchToQuery = commandsToQuery.Skip(i).Take(QUERY_CMD_BATCH_SIZE).ToArray();
72+
if (!TryGetCommandsInfo(batchToQuery, respServerPort, respServerHost,
73+
logger, ref queriedCommandsInfo))
74+
{
75+
logger.LogError("Unable to get RESP command info from local RESP server.");
76+
return false;
77+
}
78+
}
7179
}
7280

7381
var additionalCommandsInfo = new Dictionary<string, RespCommandsInfo>();
7482
foreach (var cmd in garnetCommandsInfo.Keys.Union(queriedCommandsInfo.Keys))
7583
{
7684
if (!additionalCommandsInfo.ContainsKey(cmd))
7785
{
78-
var baseCommandInfo = queriedCommandsInfo.TryGetValue(cmd, out var cmdInfo)
79-
? cmdInfo : garnetCommandsInfo[cmd];
86+
var inQueried = queriedCommandsInfo.TryGetValue(cmd, out var queriedCommandInfo);
87+
var inGarnet = garnetCommandsInfo.TryGetValue(cmd, out var garnetCommandInfo);
88+
var baseCommandInfo = inGarnet ? garnetCommandInfo : queriedCommandInfo;
8089

8190
RespCommandsInfo[] subCommandsInfo;
82-
if (garnetCommandsInfo.ContainsKey(cmd) && queriedCommandsInfo.ContainsKey(cmd))
91+
if (inQueried && inGarnet)
8392
{
8493
var subCommandsInfoMap = new Dictionary<string, RespCommandsInfo>();
8594

86-
if (garnetCommandsInfo.TryGetValue(cmd, out var garnetCmdInfo) && garnetCmdInfo.SubCommands != null)
95+
if (garnetCommandInfo.SubCommands != null)
8796
{
88-
foreach (var sc in garnetCmdInfo.SubCommands)
97+
foreach (var sc in garnetCommandInfo.SubCommands)
8998
subCommandsInfoMap.Add(sc.Name, sc);
9099
}
91100

92-
if (queriedCommandsInfo.TryGetValue(cmd, out var queriedCmdInfo) && queriedCmdInfo.SubCommands != null)
101+
if (queriedCommandInfo.SubCommands != null)
93102
{
94-
foreach (var sc in queriedCmdInfo.SubCommands)
103+
foreach (var sc in queriedCommandInfo.SubCommands)
95104
{
96105
subCommandsInfoMap.TryAdd(sc.Name, sc);
97106
}
@@ -104,7 +113,7 @@ public static bool TryUpdateCommandInfo(string outputDir, int respServerPort, IP
104113
subCommandsInfo = baseCommandInfo.SubCommands;
105114
}
106115

107-
additionalCommandsInfo.Add(cmd, new RespCommandsInfo()
116+
additionalCommandsInfo.Add(cmd, new RespCommandsInfo
108117
{
109118
Command = baseCommandInfo.Command,
110119
Name = baseCommandInfo.Name,
@@ -117,7 +126,7 @@ public static bool TryUpdateCommandInfo(string outputDir, int respServerPort, IP
117126
AclCategories = baseCommandInfo.AclCategories,
118127
Tips = baseCommandInfo.Tips,
119128
KeySpecifications = baseCommandInfo.KeySpecifications,
120-
SubCommands = subCommandsInfo
129+
SubCommands = subCommandsInfo?.OrderBy(sc => sc.Name).ToArray()
121130
});
122131
}
123132
}
@@ -147,10 +156,8 @@ public static bool TryUpdateCommandInfo(string outputDir, int respServerPort, IP
147156
/// <param name="commandsInfo">Queried commands info</param>
148157
/// <returns>True if succeeded</returns>
149158
private static unsafe bool TryGetCommandsInfo(string[] commandsToQuery, int respServerPort,
150-
IPAddress respServerHost, ILogger logger, out IDictionary<string, RespCommandsInfo> commandsInfo)
159+
IPAddress respServerHost, ILogger logger, ref IDictionary<string, RespCommandsInfo> commandsInfo)
151160
{
152-
commandsInfo = default;
153-
154161
// If there are no commands to query, return
155162
if (commandsToQuery.Length == 0) return true;
156163

@@ -167,8 +174,6 @@ private static unsafe bool TryGetCommandsInfo(string[] commandsToQuery, int resp
167174
return false;
168175
}
169176

170-
var tmpCommandsInfo = new Dictionary<string, RespCommandsInfo>();
171-
172177
// Get a map of supported commands to Garnet's RespCommand & ArrayCommand for the parser
173178
var supportedCommands = new ReadOnlyDictionary<string, RespCommand>(
174179
SupportedCommand.SupportedCommandsFlattenedMap.ToDictionary(kvp => kvp.Key,
@@ -197,11 +202,10 @@ private static unsafe bool TryGetCommandsInfo(string[] commandsToQuery, int resp
197202
}
198203

199204
if (command != null)
200-
tmpCommandsInfo.Add(command.Name, command);
205+
commandsInfo.Add(command.Name, command);
201206
}
202207
}
203208

204-
commandsInfo = tmpCommandsInfo;
205209
return true;
206210
}
207211

@@ -238,7 +242,7 @@ private static IReadOnlyDictionary<string, RespCommandsInfo> GetUpdatedCommandsI
238242
: existingCommandsInfo[command.Command].SubCommands.Select(sc => sc.Name).ToArray();
239243
var remainingSubCommands = existingSubCommands == null ? null :
240244
command.SubCommands == null ? existingSubCommands :
241-
existingSubCommands.Except(command.SubCommands.Keys).ToArray();
245+
[.. existingSubCommands.Except(command.SubCommands.Keys)];
242246

243247
// Create updated command info based on existing command
244248
var existingCommand = existingCommandsInfo[command.Command];
@@ -257,7 +261,7 @@ private static IReadOnlyDictionary<string, RespCommandsInfo> GetUpdatedCommandsI
257261
KeySpecifications = existingCommand.KeySpecifications,
258262
SubCommands = remainingSubCommands == null || remainingSubCommands.Length == 0
259263
? null
260-
: existingCommand.SubCommands.Where(sc => remainingSubCommands.Contains(sc.Name)).ToArray()
264+
: [.. existingCommand.SubCommands.Where(sc => remainingSubCommands.Contains(sc.Name))]
261265
};
262266

263267
updatedCommandsInfo.Add(updatedCommand.Name, updatedCommand);
@@ -294,7 +298,7 @@ private static IReadOnlyDictionary<string, RespCommandsInfo> GetUpdatedCommandsI
294298
// Update sub-commands to contain supported sub-commands only
295299
updatedSubCommands = command.SubCommands == null
296300
? null
297-
: baseCommand.SubCommands.Where(sc => command.SubCommands.ContainsKey(sc.Name)).ToList();
301+
: [.. baseCommand.SubCommands.Where(sc => command.SubCommands.ContainsKey(sc.Name))];
298302
}
299303

300304
// Create updated command info based on base command & updated sub-commands

‎playground/CommandInfoUpdater/CommonUtils.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,8 @@ internal static (IDictionary<SupportedCommand, bool>, IDictionary<SupportedComma
105105
var existingSubCommands = new HashSet<string>(existingCommandsInfo[supportedCommand.Command]
106106
.SubCommands
107107
.Select(sc => sc.Name));
108-
subCommandsToAdd = supportedCommand.SubCommands
109-
.Where(subCommand => !existingSubCommands.Contains(subCommand.Key)).Select(sc => sc.Value).ToArray();
108+
subCommandsToAdd = [.. supportedCommand.SubCommands
109+
.Where(subCommand => !existingSubCommands.Contains(subCommand.Key)).Select(sc => sc.Value)];
110110
}
111111

112112
// If there are sub-commands to add, add a new supported command with the sub-commands to add

0 commit comments

Comments
 (0)