Skip to content

Commit 003e2d0

Browse files
coeuvrecopybara-github
coeuvre
authored and
copybara-github
committedSep 22, 2021
Remote: Fixes a confusion that background upload counter could increase after build finished.
At the end of a build, the number of files waiting to be uploaded could increase as other ones finished. This PR fixes that. Also, changes to only emit profile block `upload outputs` for blocking uploads. Fixes bazelbuild#13655 (comment). Closes bazelbuild#13954. PiperOrigin-RevId: 398161750
·
7.0.0-pre.20230504.4
1 parent de0c6bd commit 003e2d0

16 files changed

+158
-193
lines changed
 

‎src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java

+13-69
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,7 @@
3030
import com.google.common.util.concurrent.ListenableFuture;
3131
import com.google.common.util.concurrent.MoreExecutors;
3232
import com.google.common.util.concurrent.SettableFuture;
33-
import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
34-
import com.google.devtools.build.lib.actions.ActionUploadFinishedEvent;
35-
import com.google.devtools.build.lib.actions.ActionUploadStartedEvent;
3633
import com.google.devtools.build.lib.concurrent.ThreadSafety;
37-
import com.google.devtools.build.lib.events.ExtendedEventHandler;
3834
import com.google.devtools.build.lib.exec.SpawnProgressEvent;
3935
import com.google.devtools.build.lib.remote.common.BulkTransferException;
4036
import com.google.devtools.build.lib.remote.common.LazyFileOutputStream;
@@ -66,7 +62,6 @@
6662
import java.util.concurrent.atomic.AtomicLong;
6763
import java.util.regex.Matcher;
6864
import java.util.regex.Pattern;
69-
import javax.annotation.Nullable;
7065

7166
/**
7267
* A cache for storing artifacts (input and output) as well as the output of running an action.
@@ -85,7 +80,6 @@ public class RemoteCache extends AbstractReferenceCounted {
8580
private static final ListenableFuture<Void> COMPLETED_SUCCESS = immediateFuture(null);
8681
private static final ListenableFuture<byte[]> EMPTY_BYTES = immediateFuture(new byte[0]);
8782

88-
private final ExtendedEventHandler reporter;
8983
private final CountDownLatch closeCountDownLatch = new CountDownLatch(1);
9084
protected final AsyncTaskCache.NoResult<Digest> casUploadCache = AsyncTaskCache.NoResult.create();
9185

@@ -94,11 +88,9 @@ public class RemoteCache extends AbstractReferenceCounted {
9488
protected final DigestUtil digestUtil;
9589

9690
public RemoteCache(
97-
ExtendedEventHandler reporter,
9891
RemoteCacheClient cacheProtocol,
9992
RemoteOptions options,
10093
DigestUtil digestUtil) {
101-
this.reporter = reporter;
10294
this.cacheProtocol = cacheProtocol;
10395
this.options = options;
10496
this.digestUtil = digestUtil;
@@ -110,23 +102,6 @@ public CachedActionResult downloadActionResult(
110102
return getFromFuture(cacheProtocol.downloadActionResult(context, actionKey, inlineOutErr));
111103
}
112104

113-
private void postUploadStartedEvent(@Nullable ActionExecutionMetadata action, String resourceId) {
114-
if (action == null) {
115-
return;
116-
}
117-
118-
reporter.post(ActionUploadStartedEvent.create(action, resourceId));
119-
}
120-
121-
private void postUploadFinishedEvent(
122-
@Nullable ActionExecutionMetadata action, String resourceId) {
123-
if (action == null) {
124-
return;
125-
}
126-
127-
reporter.post(ActionUploadFinishedEvent.create(action, resourceId));
128-
}
129-
130105
/**
131106
* Returns a set of digests that the remote cache does not know about. The returned set is
132107
* guaranteed to be a subset of {@code digests}.
@@ -143,38 +118,14 @@ public ListenableFuture<ImmutableSet<Digest>> findMissingDigests(
143118
public ListenableFuture<Void> uploadActionResult(
144119
RemoteActionExecutionContext context, ActionKey actionKey, ActionResult actionResult) {
145120

146-
ActionExecutionMetadata action = context.getSpawnOwner();
147-
148121
Completable upload =
149-
Completable.using(
150-
() -> {
151-
String resourceId = "ac/" + actionKey.getDigest().getHash();
152-
postUploadStartedEvent(action, resourceId);
153-
return resourceId;
154-
},
155-
resourceId ->
156-
RxFutures.toCompletable(
157-
() -> cacheProtocol.uploadActionResult(context, actionKey, actionResult),
158-
directExecutor()),
159-
resourceId -> postUploadFinishedEvent(action, resourceId));
122+
RxFutures.toCompletable(
123+
() -> cacheProtocol.uploadActionResult(context, actionKey, actionResult),
124+
directExecutor());
160125

161126
return RxFutures.toListenableFuture(upload);
162127
}
163128

164-
private Completable doUploadFile(RemoteActionExecutionContext context, Digest digest, Path file) {
165-
ActionExecutionMetadata action = context.getSpawnOwner();
166-
return Completable.using(
167-
() -> {
168-
String resourceId = "cas/" + digest.getHash();
169-
postUploadStartedEvent(action, resourceId);
170-
return resourceId;
171-
},
172-
resourceId ->
173-
RxFutures.toCompletable(
174-
() -> cacheProtocol.uploadFile(context, digest, file), directExecutor()),
175-
resourceId -> postUploadFinishedEvent(action, resourceId));
176-
}
177-
178129
/**
179130
* Upload a local file to the remote cache.
180131
*
@@ -191,26 +142,15 @@ public final ListenableFuture<Void> uploadFile(
191142
return COMPLETED_SUCCESS;
192143
}
193144

194-
Completable upload = casUploadCache.executeIfNot(digest, doUploadFile(context, digest, file));
145+
Completable upload =
146+
casUploadCache.executeIfNot(
147+
digest,
148+
RxFutures.toCompletable(
149+
() -> cacheProtocol.uploadFile(context, digest, file), directExecutor()));
195150

196151
return RxFutures.toListenableFuture(upload);
197152
}
198153

199-
private Completable doUploadBlob(
200-
RemoteActionExecutionContext context, Digest digest, ByteString data) {
201-
ActionExecutionMetadata action = context.getSpawnOwner();
202-
return Completable.using(
203-
() -> {
204-
String resourceId = "cas/" + digest.getHash();
205-
postUploadStartedEvent(action, resourceId);
206-
return resourceId;
207-
},
208-
resourceId ->
209-
RxFutures.toCompletable(
210-
() -> cacheProtocol.uploadBlob(context, digest, data), directExecutor()),
211-
resourceId -> postUploadFinishedEvent(action, resourceId));
212-
}
213-
214154
/**
215155
* Upload sequence of bytes to the remote cache.
216156
*
@@ -227,7 +167,11 @@ public final ListenableFuture<Void> uploadBlob(
227167
return COMPLETED_SUCCESS;
228168
}
229169

230-
Completable upload = casUploadCache.executeIfNot(digest, doUploadBlob(context, digest, data));
170+
Completable upload =
171+
casUploadCache.executeIfNot(
172+
digest,
173+
RxFutures.toCompletable(
174+
() -> cacheProtocol.uploadBlob(context, digest, data), directExecutor()));
231175

232176
return RxFutures.toListenableFuture(upload);
233177
}

‎src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionCache.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import com.google.common.util.concurrent.Futures;
2323
import com.google.common.util.concurrent.ListenableFuture;
2424
import com.google.common.util.concurrent.MoreExecutors;
25-
import com.google.devtools.build.lib.events.ExtendedEventHandler;
2625
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
2726
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
2827
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
@@ -43,11 +42,10 @@
4342
public class RemoteExecutionCache extends RemoteCache {
4443

4544
public RemoteExecutionCache(
46-
ExtendedEventHandler reporter,
4745
RemoteCacheClient protocolImpl,
4846
RemoteOptions options,
4947
DigestUtil digestUtil) {
50-
super(reporter, protocolImpl, options, digestUtil);
48+
super(protocolImpl, options, digestUtil);
5149
}
5250

5351
/**

‎src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java

+7-2
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484
import com.google.devtools.build.lib.events.Reporter;
8585
import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionContext;
8686
import com.google.devtools.build.lib.profiler.Profiler;
87+
import com.google.devtools.build.lib.profiler.ProfilerTask;
8788
import com.google.devtools.build.lib.profiler.SilentCloseable;
8889
import com.google.devtools.build.lib.remote.RemoteExecutionService.ActionResultMetadata.DirectoryMetadata;
8990
import com.google.devtools.build.lib.remote.RemoteExecutionService.ActionResultMetadata.FileMetadata;
@@ -1070,7 +1071,8 @@ public void uploadOutputs(RemoteAction action, SpawnResult spawnResult)
10701071
Single.using(
10711072
remoteCache::retain,
10721073
remoteCache ->
1073-
manifest.uploadAsync(action.getRemoteActionExecutionContext(), remoteCache),
1074+
manifest.uploadAsync(
1075+
action.getRemoteActionExecutionContext(), remoteCache, reporter),
10741076
RemoteCache::release)
10751077
.subscribeOn(scheduler)
10761078
.subscribe(
@@ -1087,7 +1089,10 @@ public void onError(@NonNull Throwable e) {
10871089
}
10881090
});
10891091
} else {
1090-
manifest.upload(action.getRemoteActionExecutionContext(), remoteCache);
1092+
try (SilentCloseable c =
1093+
Profiler.instance().profile(ProfilerTask.UPLOAD_TIME, "upload outputs")) {
1094+
manifest.upload(action.getRemoteActionExecutionContext(), remoteCache, reporter);
1095+
}
10911096
}
10921097
} catch (IOException e) {
10931098
reportUploadError(e);

‎src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java

+3-5
Original file line numberDiff line numberDiff line change
@@ -234,8 +234,7 @@ private void initHttpAndDiskCache(
234234
handleInitFailure(env, e, Code.CACHE_INIT_FAILURE);
235235
return;
236236
}
237-
RemoteCache remoteCache =
238-
new RemoteCache(env.getReporter(), cacheClient, remoteOptions, digestUtil);
237+
RemoteCache remoteCache = new RemoteCache(cacheClient, remoteOptions, digestUtil);
239238
actionContextProvider =
240239
RemoteActionContextProvider.createForRemoteCaching(
241240
executorService, env, remoteCache, /* retryScheduler= */ null, digestUtil);
@@ -573,7 +572,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
573572
}
574573
execChannel.release();
575574
RemoteExecutionCache remoteCache =
576-
new RemoteExecutionCache(env.getReporter(), cacheClient, remoteOptions, digestUtil);
575+
new RemoteExecutionCache(cacheClient, remoteOptions, digestUtil);
577576
actionContextProvider =
578577
RemoteActionContextProvider.createForRemoteExecution(
579578
executorService,
@@ -609,8 +608,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
609608
}
610609
}
611610

612-
RemoteCache remoteCache =
613-
new RemoteCache(env.getReporter(), cacheClient, remoteOptions, digestUtil);
611+
RemoteCache remoteCache = new RemoteCache(cacheClient, remoteOptions, digestUtil);
614612
actionContextProvider =
615613
RemoteActionContextProvider.createForRemoteCaching(
616614
executorService, env, remoteCache, retryScheduler, digestUtil);

‎src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -198,9 +198,7 @@ public void store(SpawnResult result) throws ExecException, InterruptedException
198198
}
199199
}
200200

201-
try (SilentCloseable c = prof.profile(ProfilerTask.UPLOAD_TIME, "upload outputs")) {
202-
remoteExecutionService.uploadOutputs(action, result);
203-
}
201+
remoteExecutionService.uploadOutputs(action, result);
204202
}
205203

206204
@Override

‎src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -575,9 +575,7 @@ SpawnResult execLocallyAndUpload(
575575
}
576576
}
577577

578-
try (SilentCloseable c = Profiler.instance().profile(UPLOAD_TIME, "upload outputs")) {
579-
remoteExecutionService.uploadOutputs(action, result);
580-
}
578+
remoteExecutionService.uploadOutputs(action, result);
581579
return result;
582580
}
583581

‎src/main/java/com/google/devtools/build/lib/remote/UploadManifest.java

+80-10
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,20 @@
2828
import build.bazel.remote.execution.v2.Tree;
2929
import com.google.common.annotations.VisibleForTesting;
3030
import com.google.common.base.Preconditions;
31+
import com.google.common.collect.ImmutableList;
32+
import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
33+
import com.google.devtools.build.lib.actions.ActionUploadFinishedEvent;
34+
import com.google.devtools.build.lib.actions.ActionUploadStartedEvent;
3135
import com.google.devtools.build.lib.actions.ExecException;
3236
import com.google.devtools.build.lib.actions.UserExecException;
37+
import com.google.devtools.build.lib.events.ExtendedEventHandler;
3338
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
3439
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
3540
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey;
3641
import com.google.devtools.build.lib.remote.common.RemotePathResolver;
3742
import com.google.devtools.build.lib.remote.options.RemoteOptions;
3843
import com.google.devtools.build.lib.remote.util.DigestUtil;
44+
import com.google.devtools.build.lib.remote.util.RxUtils;
3945
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
4046
import com.google.devtools.build.lib.server.FailureDetails.RemoteExecution;
4147
import com.google.devtools.build.lib.server.FailureDetails.RemoteExecution.Code;
@@ -56,6 +62,7 @@
5662
import java.util.HashMap;
5763
import java.util.List;
5864
import java.util.Map;
65+
import java.util.stream.Collectors;
5966
import javax.annotation.Nullable;
6067

6168
/** UploadManifest adds output metadata to a {@link ActionResult}. */
@@ -341,10 +348,11 @@ ActionResult getActionResult() {
341348
}
342349

343350
/** Uploads outputs and action result (if exit code is 0) to remote cache. */
344-
public ActionResult upload(RemoteActionExecutionContext context, RemoteCache remoteCache)
351+
public ActionResult upload(
352+
RemoteActionExecutionContext context, RemoteCache remoteCache, ExtendedEventHandler reporter)
345353
throws IOException, InterruptedException {
346354
try {
347-
return uploadAsync(context, remoteCache).blockingGet();
355+
return uploadAsync(context, remoteCache, reporter).blockingGet();
348356
} catch (RuntimeException e) {
349357
throwIfInstanceOf(e.getCause(), InterruptedException.class);
350358
throwIfInstanceOf(e.getCause(), IOException.class);
@@ -368,29 +376,91 @@ private Completable upload(
368376
return toCompletable(() -> remoteCache.uploadBlob(context, digest, blob), directExecutor());
369377
}
370378

379+
private static void reportUploadStarted(
380+
ExtendedEventHandler reporter,
381+
@Nullable ActionExecutionMetadata action,
382+
String prefix,
383+
Iterable<Digest> digests) {
384+
if (action != null) {
385+
for (Digest digest : digests) {
386+
reporter.post(ActionUploadStartedEvent.create(action, prefix + digest.getHash()));
387+
}
388+
}
389+
}
390+
391+
private static void reportUploadFinished(
392+
ExtendedEventHandler reporter,
393+
@Nullable ActionExecutionMetadata action,
394+
String resourceIdPrefix,
395+
Iterable<Digest> digests) {
396+
if (action != null) {
397+
for (Digest digest : digests) {
398+
reporter.post(
399+
ActionUploadFinishedEvent.create(action, resourceIdPrefix + digest.getHash()));
400+
}
401+
}
402+
}
403+
371404
/**
372405
* Returns a {@link Single} which upon subscription will upload outputs and action result (if exit
373406
* code is 0) to remote cache.
374407
*/
375408
public Single<ActionResult> uploadAsync(
376-
RemoteActionExecutionContext context, RemoteCache remoteCache) {
409+
RemoteActionExecutionContext context,
410+
RemoteCache remoteCache,
411+
ExtendedEventHandler reporter) {
377412
Collection<Digest> digests = new ArrayList<>();
378413
digests.addAll(digestToFile.keySet());
379414
digests.addAll(digestToBlobs.keySet());
380415

381-
Completable uploadOutputs =
382-
mergeBulkTransfer(
383-
toSingle(() -> remoteCache.findMissingDigests(context, digests), directExecutor())
384-
.flatMapPublisher(Flowable::fromIterable)
385-
.flatMapSingle(digest -> toTransferResult(upload(context, remoteCache, digest))));
416+
ActionExecutionMetadata action = context.getSpawnOwner();
417+
418+
String outputPrefix = "cas/";
419+
Flowable<RxUtils.TransferResult> bulkTransfers =
420+
toSingle(() -> remoteCache.findMissingDigests(context, digests), directExecutor())
421+
.doOnSubscribe(d -> reportUploadStarted(reporter, action, outputPrefix, digests))
422+
.doOnError(error -> reportUploadFinished(reporter, action, outputPrefix, digests))
423+
.doOnDispose(() -> reportUploadFinished(reporter, action, outputPrefix, digests))
424+
.doOnSuccess(
425+
missingDigests -> {
426+
List<Digest> existedDigests =
427+
digests.stream()
428+
.filter(digest -> !missingDigests.contains(digest))
429+
.collect(Collectors.toList());
430+
reportUploadFinished(reporter, action, outputPrefix, existedDigests);
431+
})
432+
.flatMapPublisher(Flowable::fromIterable)
433+
.flatMapSingle(
434+
digest ->
435+
toTransferResult(upload(context, remoteCache, digest))
436+
.doFinally(
437+
() ->
438+
reportUploadFinished(
439+
reporter, action, outputPrefix, ImmutableList.of(digest))));
440+
Completable uploadOutputs = mergeBulkTransfer(bulkTransfers);
386441

387442
ActionResult actionResult = result.build();
388443
Completable uploadActionResult = Completable.complete();
389444
if (actionResult.getExitCode() == 0 && actionKey != null) {
445+
String actionResultPrefix = "ac/";
390446
uploadActionResult =
391447
toCompletable(
392-
() -> remoteCache.uploadActionResult(context, actionKey, actionResult),
393-
directExecutor());
448+
() -> remoteCache.uploadActionResult(context, actionKey, actionResult),
449+
directExecutor())
450+
.doOnSubscribe(
451+
d ->
452+
reportUploadStarted(
453+
reporter,
454+
action,
455+
actionResultPrefix,
456+
ImmutableList.of(actionKey.getDigest())))
457+
.doFinally(
458+
() ->
459+
reportUploadFinished(
460+
reporter,
461+
action,
462+
actionResultPrefix,
463+
ImmutableList.of(actionKey.getDigest())));
394464
}
395465

396466
return Completable.concatArray(uploadOutputs, uploadActionResult).toSingleDefault(actionResult);

‎src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,7 @@ private ByteStreamBuildEventArtifactUploader newArtifactUploader(
414414
invocationOnMock.getArgument(0), invocationOnMock.getArgument(1)))
415415
.when(cacheClient)
416416
.findMissingDigests(any(), any());
417-
RemoteCache remoteCache = new RemoteCache(reporter, cacheClient, remoteOptions, DIGEST_UTIL);
417+
RemoteCache remoteCache = new RemoteCache(cacheClient, remoteOptions, DIGEST_UTIL);
418418

419419
return new ByteStreamBuildEventArtifactUploader(
420420
MoreExecutors.directExecutor(),

‎src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java

+11-13
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@
5151
import com.google.common.collect.ImmutableMap;
5252
import com.google.common.collect.ImmutableSortedMap;
5353
import com.google.common.collect.Maps;
54-
import com.google.common.eventbus.EventBus;
5554
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
5655
import com.google.common.util.concurrent.MoreExecutors;
5756
import com.google.devtools.build.lib.actions.ActionInputHelper;
@@ -61,7 +60,7 @@
6160
import com.google.devtools.build.lib.authandtls.CallCredentialsProvider;
6261
import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
6362
import com.google.devtools.build.lib.clock.JavaClock;
64-
import com.google.devtools.build.lib.events.Reporter;
63+
import com.google.devtools.build.lib.events.NullEventHandler;
6564
import com.google.devtools.build.lib.remote.RemoteRetrier.ExponentialBackoff;
6665
import com.google.devtools.build.lib.remote.Retrier.Backoff;
6766
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
@@ -132,7 +131,6 @@ public class GrpcCacheClientTest {
132131
private Path execRoot;
133132
private FileOutErr outErr;
134133
private FakeActionInputFileCache fakeFileCache;
135-
private final Reporter reporter = new Reporter(new EventBus());
136134
private final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry();
137135
private final String fakeServerName = "fake server for " + getClass();
138136
private Server fakeServer;
@@ -271,7 +269,7 @@ private static byte[] downloadBlob(
271269
public void testVirtualActionInputSupport() throws Exception {
272270
RemoteOptions options = Options.getDefaults(RemoteOptions.class);
273271
RemoteExecutionCache client =
274-
new RemoteExecutionCache(reporter, newClient(options), options, DIGEST_UTIL);
272+
new RemoteExecutionCache(newClient(options), options, DIGEST_UTIL);
275273
PathFragment execPath = PathFragment.create("my/exec/path");
276274
VirtualActionInput virtualActionInput =
277275
ActionsTestUtil.createVirtualActionInput(execPath, "hello");
@@ -381,7 +379,7 @@ public void testDownloadAllResults() throws Exception {
381379
// arrange
382380
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
383381
GrpcCacheClient client = newClient(remoteOptions);
384-
RemoteCache remoteCache = new RemoteCache(reporter, client, remoteOptions, DIGEST_UTIL);
382+
RemoteCache remoteCache = new RemoteCache(client, remoteOptions, DIGEST_UTIL);
385383

386384
Digest fooDigest = DIGEST_UTIL.computeAsUtf8("foo-contents");
387385
Digest barDigest = DIGEST_UTIL.computeAsUtf8("bar-contents");
@@ -404,7 +402,7 @@ public void testDownloadAllResults() throws Exception {
404402
public void testUploadDirectory() throws Exception {
405403
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
406404
GrpcCacheClient client = newClient(remoteOptions);
407-
RemoteCache remoteCache = new RemoteCache(reporter, client, remoteOptions, DIGEST_UTIL);
405+
RemoteCache remoteCache = new RemoteCache(client, remoteOptions, DIGEST_UTIL);
408406

409407
final Digest fooDigest =
410408
fakeFileCache.createScratchInput(ActionInputHelper.fromPath("a/foo"), "xyz");
@@ -468,7 +466,7 @@ public void updateActionResult(
468466
public void testUploadDirectoryEmpty() throws Exception {
469467
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
470468
GrpcCacheClient client = newClient(remoteOptions);
471-
RemoteCache remoteCache = new RemoteCache(reporter, client, remoteOptions, DIGEST_UTIL);
469+
RemoteCache remoteCache = new RemoteCache(client, remoteOptions, DIGEST_UTIL);
472470

473471
final Digest barDigest =
474472
fakeFileCache.createScratchInputDirectory(
@@ -507,7 +505,7 @@ public void updateActionResult(
507505
public void testUploadDirectoryNested() throws Exception {
508506
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
509507
GrpcCacheClient client = newClient(remoteOptions);
510-
RemoteCache remoteCache = new RemoteCache(reporter, client, remoteOptions, DIGEST_UTIL);
508+
RemoteCache remoteCache = new RemoteCache(client, remoteOptions, DIGEST_UTIL);
511509

512510
final Digest wobbleDigest =
513511
fakeFileCache.createScratchInput(ActionInputHelper.fromPath("bar/test/wobble"), "xyz");
@@ -583,7 +581,7 @@ private ActionResult upload(
583581
outputs,
584582
outErr,
585583
0);
586-
return uploadManifest.upload(context, remoteCache);
584+
return uploadManifest.upload(context, remoteCache, NullEventHandler.INSTANCE);
587585
}
588586

589587
private ActionResult uploadDirectory(RemoteCache remoteCache, List<Path> outputs)
@@ -658,7 +656,7 @@ public void getActionResult(
658656
serviceRegistry.addService(ServerInterceptors.intercept(actionCache, interceptor));
659657

660658
GrpcCacheClient client = newClient(remoteOptions);
661-
RemoteCache remoteCache = new RemoteCache(reporter, client, remoteOptions, DIGEST_UTIL);
659+
RemoteCache remoteCache = new RemoteCache(client, remoteOptions, DIGEST_UTIL);
662660
remoteCache.downloadActionResult(
663661
context,
664662
DIGEST_UTIL.asActionKey(DIGEST_UTIL.computeAsUtf8("key")),
@@ -669,7 +667,7 @@ public void getActionResult(
669667
public void testUpload() throws Exception {
670668
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
671669
GrpcCacheClient client = newClient(remoteOptions);
672-
RemoteCache remoteCache = new RemoteCache(reporter, client, remoteOptions, DIGEST_UTIL);
670+
RemoteCache remoteCache = new RemoteCache(client, remoteOptions, DIGEST_UTIL);
673671

674672
final Digest fooDigest =
675673
fakeFileCache.createScratchInput(ActionInputHelper.fromPath("a/foo"), "xyz");
@@ -745,7 +743,7 @@ public void testUploadSplitMissingDigestsCall() throws Exception {
745743
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
746744
remoteOptions.maxOutboundMessageSize = 80; // Enough for one digest, but not two.
747745
GrpcCacheClient client = newClient(remoteOptions);
748-
RemoteCache remoteCache = new RemoteCache(reporter, client, remoteOptions, DIGEST_UTIL);
746+
RemoteCache remoteCache = new RemoteCache(client, remoteOptions, DIGEST_UTIL);
749747

750748
final Digest fooDigest =
751749
fakeFileCache.createScratchInput(ActionInputHelper.fromPath("a/foo"), "xyz");
@@ -810,7 +808,7 @@ public void updateActionResult(
810808
public void testUploadCacheMissesWithRetries() throws Exception {
811809
RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
812810
GrpcCacheClient client = newClient(remoteOptions);
813-
RemoteCache remoteCache = new RemoteCache(reporter, client, remoteOptions, DIGEST_UTIL);
811+
RemoteCache remoteCache = new RemoteCache(client, remoteOptions, DIGEST_UTIL);
814812

815813
final Digest fooDigest =
816814
fakeFileCache.createScratchInput(ActionInputHelper.fromPath("a/foo"), "xyz");

‎src/test/java/com/google/devtools/build/lib/remote/InMemoryRemoteCache.java

+3-5
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
import static java.nio.charset.StandardCharsets.UTF_8;
1717

1818
import build.bazel.remote.execution.v2.Digest;
19-
import com.google.devtools.build.lib.events.Reporter;
2019
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
2120
import com.google.devtools.build.lib.remote.options.RemoteOptions;
2221
import com.google.devtools.build.lib.remote.util.DigestUtil;
@@ -30,15 +29,14 @@
3029
class InMemoryRemoteCache extends RemoteExecutionCache {
3130

3231
InMemoryRemoteCache(
33-
Reporter reporter,
3432
Map<Digest, byte[]> casEntries,
3533
RemoteOptions options,
3634
DigestUtil digestUtil) {
37-
super(reporter, new InMemoryCacheClient(casEntries), options, digestUtil);
35+
super(new InMemoryCacheClient(casEntries), options, digestUtil);
3836
}
3937

40-
InMemoryRemoteCache(Reporter reporter, RemoteOptions options, DigestUtil digestUtil) {
41-
super(reporter, new InMemoryCacheClient(), options, digestUtil);
38+
InMemoryRemoteCache(RemoteOptions options, DigestUtil digestUtil) {
39+
super(new InMemoryCacheClient(), options, digestUtil);
4240
}
4341

4442
Digest addContents(RemoteActionExecutionContext context, String txt)

‎src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java

+1-5
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import com.google.common.collect.ImmutableList;
2424
import com.google.common.collect.ImmutableMap;
2525
import com.google.common.collect.Maps;
26-
import com.google.common.eventbus.EventBus;
2726
import com.google.common.hash.HashCode;
2827
import com.google.common.util.concurrent.Futures;
2928
import com.google.common.util.concurrent.SettableFuture;
@@ -37,7 +36,6 @@
3736
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
3837
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
3938
import com.google.devtools.build.lib.clock.JavaClock;
40-
import com.google.devtools.build.lib.events.Reporter;
4139
import com.google.devtools.build.lib.remote.common.BulkTransferException;
4240
import com.google.devtools.build.lib.remote.options.RemoteOptions;
4341
import com.google.devtools.build.lib.remote.util.DigestUtil;
@@ -67,7 +65,6 @@ public class RemoteActionInputFetcherTest {
6765

6866
private static final DigestHashFunction HASH_FUNCTION = DigestHashFunction.SHA256;
6967

70-
private final Reporter reporter = new Reporter(new EventBus());
7168
private Path execRoot;
7269
private ArtifactRoot artifactRoot;
7370
private RemoteOptions options;
@@ -395,7 +392,6 @@ private RemoteCache newCache(
395392
for (Map.Entry<Digest, ByteString> entry : cacheEntries.entrySet()) {
396393
cacheEntriesByteArray.put(entry.getKey(), entry.getValue().toByteArray());
397394
}
398-
return new RemoteCache(
399-
reporter, new InMemoryCacheClient(cacheEntriesByteArray), options, digestUtil);
395+
return new RemoteCache(new InMemoryCacheClient(cacheEntriesByteArray), options, digestUtil);
400396
}
401397
}

‎src/test/java/com/google/devtools/build/lib/remote/RemoteCacheTest.java

+2-58
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,15 @@
1616
import static com.google.common.truth.Truth.assertThat;
1717
import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
1818

19-
import build.bazel.remote.execution.v2.Action;
2019
import build.bazel.remote.execution.v2.ActionResult;
2120
import build.bazel.remote.execution.v2.Digest;
2221
import build.bazel.remote.execution.v2.RequestMetadata;
2322
import com.google.common.collect.ImmutableList;
2423
import com.google.common.collect.ImmutableMap;
2524
import com.google.common.collect.ImmutableSet;
26-
import com.google.common.eventbus.EventBus;
2725
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
2826
import com.google.common.util.concurrent.MoreExecutors;
2927
import com.google.devtools.build.lib.actions.ActionInputHelper;
30-
import com.google.devtools.build.lib.actions.ActionUploadFinishedEvent;
31-
import com.google.devtools.build.lib.actions.ActionUploadStartedEvent;
3228
import com.google.devtools.build.lib.actions.ArtifactRoot;
3329
import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
3430
import com.google.devtools.build.lib.actions.ResourceSet;
@@ -37,11 +33,8 @@
3733
import com.google.devtools.build.lib.clock.JavaClock;
3834
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
3935
import com.google.devtools.build.lib.collect.nestedset.Order;
40-
import com.google.devtools.build.lib.events.Reporter;
41-
import com.google.devtools.build.lib.events.StoredEventHandler;
4236
import com.google.devtools.build.lib.exec.util.FakeOwner;
4337
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
44-
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey;
4538
import com.google.devtools.build.lib.remote.options.RemoteOptions;
4639
import com.google.devtools.build.lib.remote.util.DigestUtil;
4740
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
@@ -71,8 +64,6 @@
7164
@RunWith(JUnit4.class)
7265
public class RemoteCacheTest {
7366

74-
private final Reporter reporter = new Reporter(new EventBus());
75-
private final StoredEventHandler eventHandler = new StoredEventHandler();
7667
private RemoteActionExecutionContext context;
7768
private FileSystem fs;
7869
private Path execRoot;
@@ -84,8 +75,6 @@ public class RemoteCacheTest {
8475

8576
@Before
8677
public void setUp() throws Exception {
87-
reporter.addHandler(eventHandler);
88-
8978
MockitoAnnotations.initMocks(this);
9079
RequestMetadata metadata =
9180
TracingMetadataUtils.buildMetadata("none", "none", "action-id", null);
@@ -170,7 +159,7 @@ public void testDownloadFileWithSymlinkTemplate() throws Exception {
170159
Path file = fs.getPath("/execroot/symlink-to-file");
171160
RemoteOptions options = Options.getDefaults(RemoteOptions.class);
172161
options.remoteDownloadSymlinkTemplate = "/home/alice/cas/{hash}-{size_bytes}";
173-
RemoteCache remoteCache = new InMemoryRemoteCache(reporter, cas, options, digestUtil);
162+
RemoteCache remoteCache = new InMemoryRemoteCache(cas, options, digestUtil);
174163

175164
// act
176165
getFromFuture(remoteCache.downloadFile(context, file, helloDigest));
@@ -199,53 +188,8 @@ public void upload_emptyBlobAndFile_doNotPerformUpload() throws Exception {
199188
.containsExactly(emptyDigest);
200189
}
201190

202-
@Test
203-
public void uploadActionResult_firesUploadEvents() throws Exception {
204-
InMemoryRemoteCache remoteCache = newRemoteCache();
205-
ActionKey actionKey = new ActionKey(digestUtil.compute(Action.getDefaultInstance()));
206-
ActionResult actionResult = ActionResult.getDefaultInstance();
207-
208-
getFromFuture(remoteCache.uploadActionResult(context, actionKey, actionResult));
209-
210-
String resourceId = "ac/" + actionKey.getDigest().getHash();
211-
assertThat(eventHandler.getPosts())
212-
.containsExactly(
213-
ActionUploadStartedEvent.create(context.getSpawn().getResourceOwner(), resourceId),
214-
ActionUploadFinishedEvent.create(context.getSpawn().getResourceOwner(), resourceId));
215-
}
216-
217-
@Test
218-
public void uploadBlob_firesUploadEvents() throws Exception {
219-
InMemoryRemoteCache remoteCache = newRemoteCache();
220-
ByteString content = ByteString.copyFromUtf8("content");
221-
Digest digest = digestUtil.compute(content.toByteArray());
222-
223-
getFromFuture(remoteCache.uploadBlob(context, digest, content));
224-
225-
String resourceId = "cas/" + digest.getHash();
226-
assertThat(eventHandler.getPosts())
227-
.containsExactly(
228-
ActionUploadStartedEvent.create(context.getSpawn().getResourceOwner(), resourceId),
229-
ActionUploadFinishedEvent.create(context.getSpawn().getResourceOwner(), resourceId));
230-
}
231-
232-
@Test
233-
public void uploadFile_firesUploadEvents() throws Exception {
234-
InMemoryRemoteCache remoteCache = newRemoteCache();
235-
Digest digest = fakeFileCache.createScratchInput(ActionInputHelper.fromPath("file"), "content");
236-
Path file = execRoot.getRelative("file");
237-
238-
getFromFuture(remoteCache.uploadFile(context, digest, file));
239-
240-
String resourceId = "cas/" + digest.getHash();
241-
assertThat(eventHandler.getPosts())
242-
.containsExactly(
243-
ActionUploadStartedEvent.create(context.getSpawn().getResourceOwner(), resourceId),
244-
ActionUploadFinishedEvent.create(context.getSpawn().getResourceOwner(), resourceId));
245-
}
246-
247191
private InMemoryRemoteCache newRemoteCache() {
248192
RemoteOptions options = Options.getDefaults(RemoteOptions.class);
249-
return new InMemoryRemoteCache(reporter, options, digestUtil);
193+
return new InMemoryRemoteCache(options, digestUtil);
250194
}
251195
}

‎src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java

+31-1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
import com.google.common.util.concurrent.Futures;
5353
import com.google.devtools.build.lib.actions.ActionInput;
5454
import com.google.devtools.build.lib.actions.ActionInputHelper;
55+
import com.google.devtools.build.lib.actions.ActionUploadFinishedEvent;
56+
import com.google.devtools.build.lib.actions.ActionUploadStartedEvent;
5557
import com.google.devtools.build.lib.actions.Artifact;
5658
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
5759
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
@@ -154,7 +156,7 @@ public final void setUp() throws Exception {
154156
checkNotNull(stderr.getParentDirectory()).createDirectoryAndParents();
155157
outErr = new FileOutErr(stdout, stderr);
156158

157-
cache = spy(new InMemoryRemoteCache(reporter, remoteOptions, digestUtil));
159+
cache = spy(new InMemoryRemoteCache(remoteOptions, digestUtil));
158160
executor = mock(RemoteExecutionClient.class);
159161

160162
RequestMetadata metadata =
@@ -1314,6 +1316,34 @@ public void uploadOutputs_uploadFails_printWarning() throws Exception {
13141316
assertThat(evt.getMessage()).contains("cache down");
13151317
}
13161318

1319+
@Test
1320+
public void uploadOutputs_firesUploadEvents() throws Exception {
1321+
Digest digest =
1322+
fakeFileCache.createScratchInput(ActionInputHelper.fromPath("outputs/file"), "content");
1323+
Path file = execRoot.getRelative("outputs/file");
1324+
Artifact outputFile = ActionsTestUtil.createArtifact(artifactRoot, file);
1325+
RemoteExecutionService service = newRemoteExecutionService();
1326+
Spawn spawn = newSpawn(ImmutableMap.of(), ImmutableSet.of(outputFile));
1327+
FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1328+
RemoteAction action = service.buildRemoteAction(spawn, context);
1329+
SpawnResult spawnResult =
1330+
new SpawnResult.Builder()
1331+
.setExitCode(0)
1332+
.setStatus(SpawnResult.Status.SUCCESS)
1333+
.setRunnerName("test")
1334+
.build();
1335+
1336+
service.uploadOutputs(action, spawnResult);
1337+
1338+
assertThat(eventHandler.getPosts())
1339+
.containsAtLeast(
1340+
ActionUploadStartedEvent.create(spawn.getResourceOwner(), "cas/" + digest.getHash()),
1341+
ActionUploadFinishedEvent.create(spawn.getResourceOwner(), "cas/" + digest.getHash()),
1342+
ActionUploadStartedEvent.create(spawn.getResourceOwner(), "ac/" + action.getActionId()),
1343+
ActionUploadFinishedEvent.create(
1344+
spawn.getResourceOwner(), "ac/" + action.getActionId()));
1345+
}
1346+
13171347
@Test
13181348
public void uploadInputsIfNotPresent_deduplicateFindMissingBlobCalls() throws Exception {
13191349
int taskCount = 100;

‎src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,7 @@ public int maxConcurrency() {
299299
DIGEST_UTIL,
300300
uploader);
301301
RemoteExecutionCache remoteCache =
302-
new RemoteExecutionCache(reporter, cacheProtocol, remoteOptions, DIGEST_UTIL);
302+
new RemoteExecutionCache(cacheProtocol, remoteOptions, DIGEST_UTIL);
303303
RemoteExecutionService remoteExecutionService =
304304
new RemoteExecutionService(
305305
directExecutor(),

‎src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import com.google.common.util.concurrent.MoreExecutors;
3636
import com.google.common.util.concurrent.ThreadFactoryBuilder;
3737
import com.google.devtools.build.lib.actions.ExecException;
38+
import com.google.devtools.build.lib.events.NullEventHandler;
3839
import com.google.devtools.build.lib.remote.ExecutionStatusException;
3940
import com.google.devtools.build.lib.remote.UploadManifest;
4041
import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
@@ -356,7 +357,7 @@ private ActionResult execute(
356357
outputs,
357358
outErr,
358359
exitCode);
359-
result = manifest.upload(context, cache);
360+
result = manifest.upload(context, cache, NullEventHandler.INSTANCE);
360361
} catch (ExecException e) {
361362
if (errStatus == null) {
362363
errStatus =

‎src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/OnDiskBlobStoreCache.java

-13
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919
import build.bazel.remote.execution.v2.Directory;
2020
import build.bazel.remote.execution.v2.DirectoryNode;
2121
import build.bazel.remote.execution.v2.FileNode;
22-
import com.google.devtools.build.lib.events.Event;
23-
import com.google.devtools.build.lib.events.ExtendedEventHandler;
2422
import com.google.devtools.build.lib.remote.RemoteCache;
2523
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
2624
import com.google.devtools.build.lib.remote.disk.DiskCacheClient;
@@ -34,17 +32,6 @@ class OnDiskBlobStoreCache extends RemoteCache {
3432

3533
public OnDiskBlobStoreCache(RemoteOptions options, Path cacheDir, DigestUtil digestUtil) {
3634
super(
37-
new ExtendedEventHandler() {
38-
@Override
39-
public void post(Postable obj) {
40-
// do nothing
41-
}
42-
43-
@Override
44-
public void handle(Event event) {
45-
// do nothing
46-
}
47-
},
4835
new DiskCacheClient(cacheDir, /* verifyDownloads= */ true, digestUtil),
4936
options,
5037
digestUtil);

0 commit comments

Comments
 (0)
Please sign in to comment.