Skip to content

Commit 4c453ce

Browse files
authored
fix: restart router when there is no nodes (openobserve#728)
fix openobserve#720 When router not found other nodes, just reload nodes from etcd maybe won't fix the problem. maybe the router disconnect from etcd. we need reconnect it and load everything again. The better solution is, just let it `exit` and then `k8s` will restart it.
1 parent 7d76243 commit 4c453ce

File tree

1 file changed

+9
-22
lines changed

1 file changed

+9
-22
lines changed

src/service/router/mod.rs

+9-22
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ use awc::Client;
1818
use rand::seq::SliceRandom;
1919
use rand::thread_rng;
2020

21-
use crate::infra::cluster::{self, is_ingester, is_querier, list_nodes, Node, NodeStatus};
21+
use crate::infra::cluster;
2222
use crate::infra::config::CONFIG;
2323

2424
const QUERIER_ROUTES: [&str; 9] = [
@@ -89,33 +89,20 @@ async fn dispatch(
8989
return Ok(HttpResponse::ServiceUnavailable().body("No online nodes"));
9090
}
9191

92-
// random nodes
92+
// checking nodes
9393
let mut nodes = nodes.unwrap();
94-
let mut rng = thread_rng();
95-
nodes.shuffle(&mut rng);
96-
9794
if nodes.is_empty() {
98-
nodes = list_nodes().await.unwrap_or_else(|_| vec![]);
99-
let filter_fn = if check_querier_route(path) {
100-
|node: &Node| node.status == NodeStatus::Online && is_querier(&node.role)
101-
} else {
102-
|node: &Node| node.status == NodeStatus::Online && is_ingester(&node.role)
103-
};
104-
105-
nodes.retain(filter_fn);
106-
107-
if nodes.is_empty() {
108-
log::error!("No online querier or ingestor nodes");
109-
return Ok(
110-
HttpResponse::ServiceUnavailable().body("Please wait for other nodes to be online")
111-
);
112-
}
95+
log::error!("Not found online nodes, restaring...");
96+
std::process::exit(1);
11397
}
11498

115-
let node = nodes.first().unwrap();
116-
let new_url = format!("{}{}", node.http_addr, path);
99+
// random nodes
100+
let mut rng = thread_rng();
101+
nodes.shuffle(&mut rng);
117102

118103
// send query
104+
let node = nodes.first().unwrap();
105+
let new_url = format!("{}{}", node.http_addr, path);
119106
let resp = client
120107
.request_from(new_url.clone(), req.head())
121108
.send_stream(payload)

0 commit comments

Comments
 (0)