@@ -259,9 +259,7 @@ func (lt *LocalTracer) PushAll() error {
259
259
260
260
// S3Download downloads files that match some prefix from an S3 bucket to a
261
261
// local directory dst.
262
- // fileNames is a list of traced jsonl file names to download. If it is empty, all traces are downloaded.
263
- // fileNames should not have .jsonl suffix.
264
- func S3Download (dst , prefix string , fileNames []string , cfg S3Config ) error {
262
+ func S3Download (dst , prefix string , cfg S3Config ) error {
265
263
// Ensure local directory structure exists
266
264
err := os .MkdirAll (dst , os .ModePerm )
267
265
if err != nil {
@@ -290,51 +288,37 @@ func S3Download(dst, prefix string, fileNames []string, cfg S3Config) error {
290
288
291
289
err = s3Svc .ListObjectsV2Pages (input , func (page * s3.ListObjectsV2Output , lastPage bool ) bool {
292
290
for _ , content := range page .Contents {
293
- key := * content .Key
291
+ localFilePath := filepath .Join (dst , prefix , strings .TrimPrefix (* content .Key , prefix ))
292
+ fmt .Printf ("Downloading %s to %s\n " , * content .Key , localFilePath )
294
293
295
- // If no fileNames are specified, download all files
296
- if len ( fileNames ) == 0 {
297
- fileNames = append ( fileNames , strings . TrimPrefix ( key , prefix ))
294
+ // Create the directories in the path
295
+ if err := os . MkdirAll ( filepath . Dir ( localFilePath ), os . ModePerm ); err != nil {
296
+ return false
298
297
}
299
298
300
- for _ , filename := range fileNames {
301
- // Add .jsonl suffix to the fileNames
302
- fullFilename := filename + ".jsonl"
303
- if strings .HasSuffix (key , fullFilename ) {
304
- localFilePath := filepath .Join (dst , prefix , strings .TrimPrefix (key , prefix ))
305
- fmt .Printf ("Downloading %s to %s\n " , key , localFilePath )
306
-
307
- // Create the directories in the path
308
- if err := os .MkdirAll (filepath .Dir (localFilePath ), os .ModePerm ); err != nil {
309
- return false
310
- }
311
-
312
- // Create a file to write the S3 Object contents to.
313
- f , err := os .Create (localFilePath )
314
- if err != nil {
315
- return false
316
- }
317
-
318
- resp , err := s3Svc .GetObject (& s3.GetObjectInput {
319
- Bucket : aws .String (cfg .BucketName ),
320
- Key : aws .String (key ),
321
- })
322
- if err != nil {
323
- f .Close ()
324
- continue
325
- }
326
- defer resp .Body .Close ()
327
-
328
- // Copy the contents of the S3 object to the local file
329
- if _ , err := io .Copy (f , resp .Body ); err != nil {
330
- f .Close ()
331
- return false
332
- }
333
-
334
- fmt .Printf ("Successfully downloaded %s to %s\n " , key , localFilePath )
335
- f .Close ()
336
- }
299
+ // Create a file to write the S3 Object contents to.
300
+ f , err := os .Create (localFilePath )
301
+ if err != nil {
302
+ return false
303
+ }
304
+
305
+ resp , err := s3Svc .GetObject (& s3.GetObjectInput {
306
+ Bucket : aws .String (cfg .BucketName ),
307
+ Key : aws .String (* content .Key ),
308
+ })
309
+ if err != nil {
310
+ f .Close ()
311
+ continue
337
312
}
313
+ defer resp .Body .Close ()
314
+
315
+ // Copy the contents of the S3 object to the local file
316
+ if _ , err := io .Copy (f , resp .Body ); err != nil {
317
+ return false
318
+ }
319
+
320
+ fmt .Printf ("Successfully downloaded %s to %s\n " , * content .Key , localFilePath )
321
+ f .Close ()
338
322
}
339
323
return ! lastPage // continue paging
340
324
})
0 commit comments