ref(projects): Jitter project state requests

common/src/config.rs

@@ -307,6 +307,9 @@ struct Cache {
     batch_interval: u32,
     /// Interval for watching local cache override files in seconds.
     file_interval: u32,
+    /// Jitter to be applied to cache timeouts, such as projects. `0` for no jitter, `1` for 100%
+    /// jitter.
+    cache_jitter: f64,
 }
 
 impl Default for Cache {
@@ -319,6 +322,7 @@ impl Default for Cache {
             miss_expiry: 60,     // 1 minute
             batch_interval: 100, // 100ms
             file_interval: 10,   // 10 seconds
+            cache_jitter: 0.2,   // 20%
         }
     }
 }
@@ -777,6 +781,15 @@ impl Config {
         Duration::from_secs(self.values.cache.file_interval.into())
     }
 
+    /// Returns a jitter percentage that should be applied to cache timeouts.
+    pub fn cache_timeout_jitter(&self) -> f64 {
+        match self.values.cache.cache_jitter {
+            jitter if jitter > 1.0 => 1.0,
+            jitter if jitter < 0.0 => 0.0,
+            jitter => jitter,
+        }
+    }
+
     /// Returns the maximum size of an event payload in bytes.
     pub fn max_event_payload_size(&self) -> usize {
         self.values.limits.max_event_payload_size.as_bytes() as usize

server/Cargo.toml

@@ -33,6 +33,7 @@ native-tls = { version = "0.2.3", optional = true }
 num_cpus = "1.10.1"
 parking_lot = "0.9.0"
 r2d2 = { version = "0.8.5", optional = true }
+rand = "0.7.2"
 rdkafka = { version = "0.21.0", git = "https://github.com/fede1024/rust-rdkafka", optional = true, rev = "b1a391b3264ca8e0b0e053c6aec7a6db09af3131" }
 redis = { version = "0.12.1-alpha.0", git = "https://github.com/mitsuhiko/redis-rs", optional = true, branch = "feature/cluster", features = ["cluster", "r2d2"] }
 regex = "1.2.0"

server/src/actors/project.rs

@@ -246,6 +246,10 @@ impl ProjectConfig {
     }
 }
 
+fn project_state_seed() -> f64 {
+    rand::random()
+}
+
 /// The project state is a cached server state of a project.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -277,6 +281,9 @@ pub struct ProjectState {
     /// The organization id.
     #[serde(default)]
     pub organization_id: Option<u64>,
+    /// Random seed for jitter and related operations.
+    #[serde(skip, default = "project_state_seed")]
+    pub seed: f64,
 }
 
 impl ProjectState {
@@ -291,6 +298,7 @@ impl ProjectState {
             config: Default::default(),
             rev: None,
             organization_id: None,
+            seed: project_state_seed(),
         }
     }
 
@@ -334,11 +342,15 @@ impl ProjectState {
 
     /// Returns whether this state is outdated and needs to be refetched.
     pub fn outdated(&self, config: &Config) -> bool {
+        // Jitter is used to compute a random interval between [0; 2 * expiry_interval]
+        let jitter = config.cache_timeout_jitter();
+        let factor = jitter * (self.seed * 2.0);
+
         SystemTime::from(self.last_fetch)
             .elapsed()
             .map(|e| match self.slug {
-                Some(_) => e > config.project_cache_expiry(),
-                None => e > config.cache_miss_expiry(),
+                Some(_) => e > config.project_cache_expiry().mul_f64(factor),
+                None => e > config.cache_miss_expiry().mul_f64(factor),
             })
             .unwrap_or(false)
     }