From 6f9b80751e055d86b1c2c8641a2b53dbe1f572e5 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 5 May 2023 09:25:48 -0600 Subject: [PATCH 1/2] Test also compressed s3 files with a fragment in the uri --- .../io/s3/TestS3ExternalCompressionRead.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3ExternalCompressionRead.java b/cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3ExternalCompressionRead.java index af11246f10..e79d3726c9 100644 --- a/cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3ExternalCompressionRead.java +++ b/cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3ExternalCompressionRead.java @@ -19,13 +19,20 @@ @Category(NeedsExternalResource.class) public class TestS3ExternalCompressionRead { + private static final String compressedObject = "cdms3:noaa-nexrad-level2?1991/07/20/KTLX/KTLX19910720_160529.gz"; + private static final String fragment = "#delimiter=/"; @Test public void testCompressedObjectRead() throws IOException { - String bucket = "noaa-nexrad-level2"; - String key = "1991/07/20/KTLX/KTLX19910720_160529.gz"; - String s3uri = "cdms3:" + bucket + "?" + key; + testCompressedObjectRead(compressedObject); + } + + @Test + public void testCompressedObjectReadWithDelimiterFragment() throws IOException { + testCompressedObjectRead(compressedObject + fragment); + } + private static void testCompressedObjectRead(String s3uri) throws IOException { System.setProperty(S3TestsCommon.AWS_REGION_PROP_NAME, S3TestsCommon.AWS_G16_REGION); try (NetcdfFile ncfile = NetcdfFiles.open(s3uri)) { From b7af5d076f3e1c669c5f70c2ba0cf22a4177ab4b Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 5 May 2023 09:26:54 -0600 Subject: [PATCH 2/2] Remove possible fragment in the uri (like for s3 files) before checking the ending to see if its compressed --- cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java b/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java index 9aa9c3dba2..f616fcf64f 100644 --- a/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java +++ b/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java @@ -400,7 +400,7 @@ private static ucar.unidata.io.RandomAccessFile downloadAndDecompress(ucar.unida * @return RandomAccessFile for the object at location */ public static ucar.unidata.io.RandomAccessFile getRaf(String location, int buffer_size) throws IOException { - String uriString = location.trim(); + String uriString = removeFragment(location.trim()); if (buffer_size <= 0) buffer_size = default_buffersize; @@ -473,6 +473,10 @@ public static ucar.unidata.io.RandomAccessFile getRaf(String location, int buffe return raf; } + private static String removeFragment(String uriString) { + return uriString.split("#")[0]; + } + private static boolean looksCompressed(String filename) { // return true if filename ends with a compressed suffix or contains a compressed suffix followed by a delimiter // (i.e. path to a compressed entry)