Merge pull request openzfs#611 from delphix/projects/merge-upstream/m…

…aster Merge remote-tracking branch '6.0/stage' into 'master'
pcd1193182 · Sep 14, 2022 · d2f8fb6 · d2f8fb6
2 parents 7ddf346 + dfe1624
commit d2f8fb6
Show file tree

Hide file tree

Showing 10 changed files with 404 additions and 620 deletions.
diff --git a/cmd/zfs_object_agent/Cargo.lock b/cmd/zfs_object_agent/Cargo.lock
diff --git a/cmd/zfs_object_agent/client/Cargo.toml b/cmd/zfs_object_agent/client/Cargo.toml
@@ -9,10 +9,10 @@ publish = false
 
 [dependencies]
 anyhow = "1.0"
-azure_core = "0.1.1"
-azure_identity = "0.1.1"
-azure_storage = "0.1.0"
-azure_storage_blobs = "0.1.0"
+azure_core = "0.2.2"
+azure_identity = "0.3.0"
+azure_storage = "0.2.0"
+azure_storage_blobs = "0.2.0"
 chrono = "0.4.19"
 clap = { version = "3.1.6", features = ["derive"] }
 futures = "0.3.13"

diff --git a/cmd/zfs_object_agent/deny.toml b/cmd/zfs_object_agent/deny.toml
@@ -18,7 +18,7 @@ allow = [
 	"CC0-1.0",
 	"BSD-2-Clause",
 	"MPL-2.0",
-	"OpenSSL",
+	"Unicode-DFS-2016",
 ]
 #
 # XXX This is here because our packages are currently unlicensed; when they
@@ -51,5 +51,4 @@ ignore = [
        # threads setting the time environment variables while we run.
        #
        "RUSTSEC-2020-0071",
-       "RUSTSEC-2020-0159",
-]
+]
diff --git a/cmd/zfs_object_agent/object_perf/Cargo.toml b/cmd/zfs_object_agent/object_perf/Cargo.toml
@@ -11,7 +11,7 @@ publish = false
 clap = { version = "3.1.6", features = ["derive"] }
 git-version = "0.3.5"
 log = "0.4"
-metered = "0.8"
+metered = "0.9"
 futures = "0.3.16"
 serde = { version = "1.0.125", features = ["derive"] }
 uuid = {version = "1.0.0"}

diff --git a/cmd/zfs_object_agent/util/Cargo.toml b/cmd/zfs_object_agent/util/Cargo.toml
@@ -28,15 +28,15 @@ log = "0.4"
 log4rs = "1.0.0"
 libc = "0.2"
 lru = "0.7.0"
-more-asserts = "0.2.1"
+more-asserts = "0.3.0"
 num-traits = "0.2.14"
 paste = "1.0.6"
 rand = "0.8.3"
 rusoto_credential = "0.48.0"
-safer-ffi = { version = "0.0.7", features = ["proc_macros"] }
+safer-ffi = { version = "0.0.10", features = ["proc_macros"] }
 serde = { version = "1.0.125", features = ["derive"] }
 serde_json = "1.0.64"
-serial_test = "0.6.0"
+serial_test = "0.9.0"
 signal-hook = "0.3.13"
 tokio = { version = "1.4", features = ["full"] }
 uuid = {version = "1.0.0", features = ["v4", "serde"]}
diff --git a/cmd/zfs_object_agent/zcache/Cargo.toml b/cmd/zfs_object_agent/zcache/Cargo.toml
@@ -16,7 +16,7 @@ exitcode = "1.1.2"
 git-version = "0.3.5"
 humantime = "2.1.0"
 log = "0.4"
-more-asserts = "0.2.1"
+more-asserts = "0.3.0"
 num-traits = "0.2.14"
 nvpair = { git = "https://github.com/delphix/rust-libzfs"}
 semver = "1.0.4"

diff --git a/cmd/zfs_object_agent/zettacache/Cargo.toml b/cmd/zfs_object_agent/zettacache/Cargo.toml
@@ -25,15 +25,15 @@ libc = "0.2"
 log = "0.4"
 lru = "0.7.0"
 lz4 = "1.23.2"
-more-asserts = "0.2.1"
-nix = "0.24.1"
+more-asserts = "0.3.0"
+nix = "0.25.0"
 num-traits = "0.2.14"
 rand = "0.8.3"
-safer-ffi = { version = "0.0.7", features = ["proc_macros"] }
+safer-ffi = { version = "0.0.10", features = ["proc_macros"] }
 seahash = "4.1.0"
 serde = { version = "1.0.125", features = ["derive", "rc"] }
 serde_json = "1.0.64"
-sysinfo = "0.23.5"
+sysinfo = "0.26.0"
 tokio = { version = "1.4", features = ["full"] }
 tokio-stream = "0.1.8"
 util = { path = "../util" }

diff --git a/cmd/zfs_object_agent/zettaobject/Cargo.toml b/cmd/zfs_object_agent/zettaobject/Cargo.toml
@@ -36,16 +36,16 @@ lazy_static = "1.4.0"
 libc = "0.2"
 log = "0.4"
 log4rs = "1.0.0"
-more-asserts = "0.2.1"
-nix = "0.24.1"
+more-asserts = "0.3.0"
+nix = "0.25.0"
 num-traits = "0.2.14"
 nvpair = { git = "https://github.com/delphix/rust-libzfs"}
 rand = "0.8.3"
 rusoto_core = "0.48.0"
 rusoto_credential = "0.48.0"
 rusoto_s3 = "0.48.0"
 rust-ini = "0.18.0"
-safer-ffi = { version = "0.0.7", features = ["proc_macros"] }
+safer-ffi = { version = "0.0.10", features = ["proc_macros"] }
 semver = "1.0.4"
 serde = { version = "1.0.125", features = ["derive"] }
 serde_bytes = "0.11"

diff --git a/cmd/zfs_object_agent/zettaobject/src/reclaim.rs b/cmd/zfs_object_agent/zettaobject/src/reclaim.rs
@@ -65,7 +65,8 @@ tunable! {
     // When reclaiming free blocks, allow this many concurrent GetObject+PutObject requests.
     static ref RECLAIM_QUEUE_DEPTH: usize = 200;
 
-    static ref VERIFY_RECLAIM_PUT: bool = true;
+    static ref VERIFY_RECLAIM_PUT: bool = false;
+    static ref MITIGATE_OVERWRITE_BUG: bool = true;
 }
 
 lazy_static! {
@@ -698,6 +699,18 @@ impl Reclaim {
                     freed_blocks_count += u64::from(later_blocks_freed);
                     freed_blocks_bytes += u64::from(later_bytes_freed);
                     objects_to_consolidate.push((later_object_new_size, frees));
+                    if *MITIGATE_OVERWRITE_BUG {
+                        // A timed-out PutObject may take effect at an unknown later time.  If a
+                        // subsequent PutObject overwrites the same object with different data,
+                        // the first PutObject may be applied at a later time, overwriting the
+                        // second PutObject's data.  If the overwritten data contains blocks from
+                        // a different (consolidated) object, those blocks will be lost.  To
+                        // mitigate this, we do not consolidate objects.  This way, if a
+                        // timed-out PutObject takes effect later, the reverted state of the
+                        // object only contains additional blocks (which will be leaked, at least
+                        // until the entire object is delted).
+                        break;
+                    }
                 }
                 // XXX look for earlier objects too?
 
@@ -869,15 +882,30 @@ async fn reclaim_frees_object(
                     .blocks
                     .retain(|block, _| block < &next_block);
 
-                assert_ge!(phys.header.blocks_size, object_size.num_bytes);
+                if !*MITIGATE_OVERWRITE_BUG {
+                    assert_ge!(phys.header.blocks_size, object_size.num_bytes);
+                }
                 phys.header.blocks_size = object_size.num_bytes;
 
                 assert_ge!(phys.header.next_block, next_block);
                 phys.header.next_block = next_block;
             }
-            assert_eq!(phys.header.blocks_size, phys.calculate_blocks_size());
-            assert_eq!(phys.header.blocks_size, object_size.num_bytes);
-            assert_eq!(phys.blocks_len(), object_size.num_blocks);
+            if *MITIGATE_OVERWRITE_BUG {
+                // A timed-out and then later-applied PutObject may have reverted the object to a
+                // previous state which has more blocks than we expect.
+                if phys.header.blocks_size != object_size.num_bytes ||
+                    phys.blocks_len() != object_size.num_blocks
+                {
+                    warn!("reclaim: {object:?} expected size {} blocks {}, found size {} blocks {}",
+                        object_size.num_bytes, object_size.num_blocks,
+                        phys.header.blocks_size, phys.blocks_len());
+                }
+                phys.header.blocks_size = phys.calculate_blocks_size();
+            } else {
+                assert_eq!(phys.header.blocks_size, phys.calculate_blocks_size());
+                assert_eq!(phys.header.blocks_size, object_size.num_bytes);
+                assert_eq!(phys.blocks_len(), object_size.num_blocks);
+            }
 
             phys
         })

diff --git a/cmd/zfs_object_agent/zoa/Cargo.toml b/cmd/zfs_object_agent/zoa/Cargo.toml
@@ -7,7 +7,7 @@ build = "src/build.rs"
 publish = false
 
 [build-dependencies]
-cbindgen = "0.23.0"
+cbindgen = "0.24.3"
 
 [lib]
 crate-type = ["cdylib"]      # Creates dynamic lib
@@ -21,7 +21,7 @@ futures = "0.3.13"
 libc = "0.2"
 log = "0.4"
 log4rs = "1.0.0"
-nix = "0.24.1"
+nix = "0.25.0"
 nvpair = { git = "https://github.com/delphix/rust-libzfs"}
 nvpair-sys = { git = "https://github.com/delphix/rust-libzfs"}
 tokio = { version = "1.4", features = ["full"] }