Skip to content

Commit a501422

Browse files
authoredJul 29, 2024··
revert: enhances S3Download to filter by traced table names (#1430)
Reverts #1374
1 parent f03f8ec commit a501422

File tree

1 file changed

+28
-44
lines changed

1 file changed

+28
-44
lines changed
 

‎pkg/trace/fileserver.go

+28-44
Original file line numberDiff line numberDiff line change
@@ -259,9 +259,7 @@ func (lt *LocalTracer) PushAll() error {
259259

260260
// S3Download downloads files that match some prefix from an S3 bucket to a
261261
// local directory dst.
262-
// fileNames is a list of traced jsonl file names to download. If it is empty, all traces are downloaded.
263-
// fileNames should not have .jsonl suffix.
264-
func S3Download(dst, prefix string, fileNames []string, cfg S3Config) error {
262+
func S3Download(dst, prefix string, cfg S3Config) error {
265263
// Ensure local directory structure exists
266264
err := os.MkdirAll(dst, os.ModePerm)
267265
if err != nil {
@@ -290,51 +288,37 @@ func S3Download(dst, prefix string, fileNames []string, cfg S3Config) error {
290288

291289
err = s3Svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
292290
for _, content := range page.Contents {
293-
key := *content.Key
291+
localFilePath := filepath.Join(dst, prefix, strings.TrimPrefix(*content.Key, prefix))
292+
fmt.Printf("Downloading %s to %s\n", *content.Key, localFilePath)
294293

295-
// If no fileNames are specified, download all files
296-
if len(fileNames) == 0 {
297-
fileNames = append(fileNames, strings.TrimPrefix(key, prefix))
294+
// Create the directories in the path
295+
if err := os.MkdirAll(filepath.Dir(localFilePath), os.ModePerm); err != nil {
296+
return false
298297
}
299298

300-
for _, filename := range fileNames {
301-
// Add .jsonl suffix to the fileNames
302-
fullFilename := filename + ".jsonl"
303-
if strings.HasSuffix(key, fullFilename) {
304-
localFilePath := filepath.Join(dst, prefix, strings.TrimPrefix(key, prefix))
305-
fmt.Printf("Downloading %s to %s\n", key, localFilePath)
306-
307-
// Create the directories in the path
308-
if err := os.MkdirAll(filepath.Dir(localFilePath), os.ModePerm); err != nil {
309-
return false
310-
}
311-
312-
// Create a file to write the S3 Object contents to.
313-
f, err := os.Create(localFilePath)
314-
if err != nil {
315-
return false
316-
}
317-
318-
resp, err := s3Svc.GetObject(&s3.GetObjectInput{
319-
Bucket: aws.String(cfg.BucketName),
320-
Key: aws.String(key),
321-
})
322-
if err != nil {
323-
f.Close()
324-
continue
325-
}
326-
defer resp.Body.Close()
327-
328-
// Copy the contents of the S3 object to the local file
329-
if _, err := io.Copy(f, resp.Body); err != nil {
330-
f.Close()
331-
return false
332-
}
333-
334-
fmt.Printf("Successfully downloaded %s to %s\n", key, localFilePath)
335-
f.Close()
336-
}
299+
// Create a file to write the S3 Object contents to.
300+
f, err := os.Create(localFilePath)
301+
if err != nil {
302+
return false
303+
}
304+
305+
resp, err := s3Svc.GetObject(&s3.GetObjectInput{
306+
Bucket: aws.String(cfg.BucketName),
307+
Key: aws.String(*content.Key),
308+
})
309+
if err != nil {
310+
f.Close()
311+
continue
337312
}
313+
defer resp.Body.Close()
314+
315+
// Copy the contents of the S3 object to the local file
316+
if _, err := io.Copy(f, resp.Body); err != nil {
317+
return false
318+
}
319+
320+
fmt.Printf("Successfully downloaded %s to %s\n", *content.Key, localFilePath)
321+
f.Close()
338322
}
339323
return !lastPage // continue paging
340324
})

0 commit comments

Comments
 (0)
Please sign in to comment.