Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add job queue status to describe API #2464

Merged
merged 5 commits into from
Aug 2, 2023
Merged

Add job queue status to describe API #2464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0dd07ba
Add support for job queue metrics in describe API
Jun 27, 2023
b9642f1
add test
Jul 13, 2023
38f2e45
update documentation for describe API
Jul 13, 2023
ad3693e
Retain describe API metrics implementation
Jul 14, 2023
c40005e
Merge branch 'master' into naman-job-queue-metrics
agunapal Aug 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 24 additions & 4 deletions docs/management_api.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,11 @@ curl http://localhost:8081/models/noop
"gpu": false,
"memoryUsage": 89247744
}
]
],
"jobQueueStatus": {
"remainingCapacity": 100,
"pendingRequests": 0
}
}
]
```
Expand Down Expand Up @@ -234,7 +238,11 @@ curl http://localhost:8081/models/noop/2.0
"gpu": false,
"memoryUsage": 89247744
}
]
],
"jobQueueStatus": {
"remainingCapacity": 100,
"pendingRequests": 0
}
}
]
```
Expand Down Expand Up @@ -264,7 +272,11 @@ curl http://localhost:8081/models/noop/all
"gpu": false,
"memoryUsage": 89247744
}
]
],
"jobQueueStatus": {
"remainingCapacity": 100,
"pendingRequests": 0
}
},
{
"modelName": "noop",
Expand All @@ -284,7 +296,11 @@ curl http://localhost:8081/models/noop/all
"gpu": false,
"memoryUsage": 89247744
}
]
],
"jobQueueStatus": {
"remainingCapacity": 100,
"pendingRequests": 0
}
}
]
```
Expand Down Expand Up @@ -401,6 +417,10 @@ curl http://localhost:8081/models/noop-customized/1.0?customized=true
"gpuUsage": "N/A"
}
],
"jobQueueStatus": {
"remainingCapacity": 100,
"pendingRequests": 0
},
"customizedMetadata": "{\n \"data1\": \"1\",\n \"data2\": \"2\"\n}"
}
]
Expand Down
31 changes: 31 additions & 0 deletions frontend/server/src/main/java/org/pytorch/serve/http/messages/DescribeModelResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class DescribeModelResponse {

private List<Worker> workers;
private Metrics metrics;
private JobQueueStatus jobQueueStatus;
private String customizedMetadata;

public DescribeModelResponse() {
Expand Down Expand Up @@ -150,6 +151,14 @@ public void setMetrics(Metrics metrics) {
this.metrics = metrics;
}

public JobQueueStatus getJobQueueStatus() {
return jobQueueStatus;
}

public void setJobQueueStatus(JobQueueStatus jobQueueStatus) {
this.jobQueueStatus = jobQueueStatus;
}

public void setCustomizedMetadata(byte[] customizedMetadata) {
this.customizedMetadata = new String(customizedMetadata, Charset.forName("UTF-8"));
}
Expand Down Expand Up @@ -257,4 +266,26 @@ public void setRequests(int requests) {
this.requests = requests;
}
}

public static final class JobQueueStatus {

private int remainingCapacity;
private int pendingRequests;

public int getRemainingCapacity() {
return remainingCapacity;
}

public void setRemainingCapacity(int remainingCapacity) {
this.remainingCapacity = remainingCapacity;
}

public int getPendingRequests() {
return pendingRequests;
}

public void setPendingRequests(int pendingRequests) {
this.pendingRequests = pendingRequests;
}
}
}
11 changes: 10 additions & 1 deletion frontend/server/src/main/java/org/pytorch/serve/openapi/OpenApiUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,6 @@ private static Operation getDescribeModelOperation(boolean version) {
workerStatus.setEnumeration(status);
worker.addProperty("status", workerStatus, true);
workers.setItems(worker);

schema.addProperty("workers", workers, true);
Schema metrics = new Schema("object");
metrics.addProperty(
Expand All @@ -449,6 +448,16 @@ private static Operation getDescribeModelOperation(boolean version) {
new Schema("integer", "Number requests processed in last 10 minutes."),
true);
schema.addProperty("metrics", metrics, true);
Schema jobQueueStatus = new Schema("object");
jobQueueStatus.addProperty(
"remainingCapacity",
new Schema("integer", "Number of new requests that can be queued."),
true);
jobQueueStatus.addProperty(
"pendingRequests",
new Schema("integer", "Number of requests waiting in the queue."),
true);
schema.addProperty("jobQueueStatus", jobQueueStatus, true);

MediaType mediaType = new MediaType(HttpHeaderValues.APPLICATION_JSON.toString(), schema);
MediaType error = getErrorResponse();
Expand Down
6 changes: 6 additions & 0 deletions frontend/server/src/main/java/org/pytorch/serve/util/ApiUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,12 @@ private static DescribeModelResponse createModelResponse(
resp.addWorker(workerId, startTime, isRunning, gpuId, memory, pid, gpuUsage);
}

DescribeModelResponse.JobQueueStatus jobQueueStatus =
new DescribeModelResponse.JobQueueStatus();
jobQueueStatus.setRemainingCapacity(model.getJobQueueRemainingCapacity());
jobQueueStatus.setPendingRequests(model.getPendingRequestsInJobQueue());
resp.setJobQueueStatus(jobQueueStatus);

return resp;
}

Expand Down
18 changes: 18 additions & 0 deletions frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,4 +402,22 @@ public synchronized boolean getJobTickets() {
this.numJobTickets.decrementAndGet();
return true;
}

public int getJobQueueRemainingCapacity() {
LinkedBlockingDeque<Job> jobsQueue = jobsDb.get(DEFAULT_DATA_QUEUE);
if (jobsQueue != null) {
return jobsQueue.remainingCapacity();
}

return 0;
}

public int getPendingRequestsInJobQueue() {
LinkedBlockingDeque<Job> jobsQueue = jobsDb.get(DEFAULT_DATA_QUEUE);
if (jobsQueue != null) {
return jobsQueue.size();
}

return 0;
}
}
18 changes: 18 additions & 0 deletions frontend/server/src/test/java/org/pytorch/serve/ModelServerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,24 @@ public void testDescribeSpecificModelVersion() throws InterruptedException {
@Test(
alwaysRun = true,
dependsOnMethods = {"testDescribeSpecificModelVersion"})
public void testDescribeModelJobQueueStatus() throws InterruptedException {
testLoadModelWithInitialWorkers("noop.mar", "noop_describe", "1.11");

Channel channel = TestUtils.getManagementChannel(configManager);
TestUtils.setResult(null);
TestUtils.setLatch(new CountDownLatch(1));
TestUtils.describeModel(channel, "noop_describe", "1.11", false);
TestUtils.getLatch().await();

DescribeModelResponse[] resp =
JsonUtils.GSON.fromJson(TestUtils.getResult(), DescribeModelResponse[].class);
Assert.assertEquals(resp[0].getJobQueueStatus().getRemainingCapacity(), 100);
Assert.assertEquals(resp[0].getJobQueueStatus().getPendingRequests(), 0);
}

@Test(
alwaysRun = true,
dependsOnMethods = {"testDescribeModelJobQueueStatus"})
public void testNoopVersionedPrediction() throws InterruptedException {
testPredictions("noopversioned", "OK", "1.11");
}
Expand Down
40 changes: 38 additions & 2 deletions frontend/server/src/test/resources/management_open_api.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,8 @@
"maxWorkers",
"status",
"workers",
"metrics"
"metrics",
"jobQueueStatus"
],
"properties": {
"modelName": {
Expand Down Expand Up @@ -570,6 +571,23 @@
"description": "Number requests processed in last 10 minutes."
}
}
},
"jobQueueStatus": {
"type": "object",
"required": [
"remainingCapacity",
"pendingRequests"
],
"properties": {
"remainingCapacity": {
"type": "integer",
"description": "Number of new requests that can be queued."
},
"pendingRequests": {
"type": "integer",
"description": "Number of requests waiting in the queue."
}
}
}
}
}
Expand Down Expand Up @@ -1049,7 +1067,8 @@
"maxWorkers",
"status",
"workers",
"metrics"
"metrics",
"jobQueueStatus"
],
"properties": {
"modelName": {
Expand Down Expand Up @@ -1140,6 +1159,23 @@
"description": "Number requests processed in last 10 minutes."
}
}
},
"jobQueueStatus": {
"type": "object",
"required": [
"remainingCapacity",
"pendingRequests"
],
"properties": {
"remainingCapacity": {
"type": "integer",
"description": "Number of new requests that can be queued."
},
"pendingRequests": {
"type": "integer",
"description": "Number of requests waiting in the queue."
}
}
}
}
}
Expand Down
40 changes: 38 additions & 2 deletions frontend/server/src/test/resources/model_management_api.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@
"maxWorkers",
"status",
"workers",
"metrics"
"metrics",
"jobQueueStatus"
],
"properties": {
"modelName": {
Expand Down Expand Up @@ -126,6 +127,23 @@
"description": "Number requests processed in last 10 minutes."
}
}
},
"jobQueueStatus": {
"type": "object",
"required": [
"remainingCapacity",
"pendingRequests"
],
"properties": {
"remainingCapacity": {
"type": "integer",
"description": "Number of new requests that can be queued."
},
"pendingRequests": {
"type": "integer",
"description": "Number of requests waiting in the queue."
}
}
}
}
}
Expand Down Expand Up @@ -605,7 +623,8 @@
"maxWorkers",
"status",
"workers",
"metrics"
"metrics",
"jobQueueStatus"
],
"properties": {
"modelName": {
Expand Down Expand Up @@ -696,6 +715,23 @@
"description": "Number requests processed in last 10 minutes."
}
}
},
"jobQueueStatus": {
"type": "object",
"required": [
"remainingCapacity",
"pendingRequests"
],
"properties": {
"remainingCapacity": {
"type": "integer",
"description": "Number of new requests that can be queued."
},
"pendingRequests": {
"type": "integer",
"description": "Number of requests waiting in the queue."
}
}
}
}
}
Expand Down