[server] Add start_time in server description

src/kudu/common/wire_protocol.proto

@@ -93,6 +93,9 @@ message ServerRegistrationPB {
   // In this case, https:// URLs should be generated for the above
   // 'http_addresses' field.
   optional bool https_enabled = 4;
+
+  // Seconds since the epoch.
+  optional int64 start_time = 5;
 }
 
 message ServerEntryPB {

src/kudu/integration-tests/registration-test.cc

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <stdint.h>
+
 #include <algorithm>
 #include <memory>
 #include <ostream>
@@ -32,6 +34,7 @@
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/ref_counted.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/walltime.h"
 #include "kudu/master/catalog_manager.h"
 #include "kudu/master/master-test-util.h"
 #include "kudu/master/master.h"
@@ -143,6 +146,7 @@ class RegistrationTest : public KuduTest {
   void SetUp() override {
     // Make heartbeats faster to speed test runtime.
     FLAGS_heartbeat_interval_ms = 10;
+    setup_time_ = WallTime_Now();
 
     KuduTest::SetUp();
 
@@ -209,6 +213,7 @@ class RegistrationTest : public KuduTest {
  protected:
   gscoped_ptr<InternalMiniCluster> cluster_;
   Schema schema_;
+  int64_t setup_time_;
 };
 
 TEST_F(RegistrationTest, TestTSRegisters) {
@@ -250,6 +255,21 @@ TEST_F(RegistrationTest, TestMasterSoftwareVersion) {
     ASSERT_TRUE(reg.has_software_version());
     ASSERT_STR_CONTAINS(reg.software_version(),
                         VersionInfo::GetVersionInfo());
+    ASSERT_LE(setup_time_, reg.start_time());
+    ASSERT_LE(reg.start_time(), WallTime_Now());
+  }
+}
+
+TEST_F(RegistrationTest, TestServerStartTime) {
+  ServerRegistrationPB reg;
+  cluster_->mini_master()->master()->GetMasterRegistration(&reg);
+  ASSERT_LE(setup_time_, reg.start_time());
+  ASSERT_LE(reg.start_time(), WallTime_Now());
+
+  for (int i = 0; i < cluster_->num_tablet_servers(); ++i) {
+    auto start_time = cluster_->mini_tablet_server(i)->server()->start_time();
+    ASSERT_LE(setup_time_, start_time);
+    ASSERT_LE(start_time, WallTime_Now());
   }
 }
 

src/kudu/master/master-test.cc

@@ -17,6 +17,8 @@
 
 #include "kudu/master/master.h"
 
+#include <time.h>
+
 #include <algorithm>
 #include <cstdint>
 #include <map>
@@ -54,6 +56,7 @@
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/gutil/strings/util.h"
+#include "kudu/gutil/walltime.h"
 #include "kudu/master/catalog_manager.h"
 #include "kudu/master/master.pb.h"
 #include "kudu/master/master.proxy.h"
@@ -209,6 +212,7 @@ TEST_F(MasterTest, TestRegisterAndHeartbeat) {
   MakeHostPortPB("localhost", 1000, fake_reg.add_rpc_addresses());
   MakeHostPortPB("localhost", 2000, fake_reg.add_http_addresses());
   fake_reg.set_software_version(VersionInfo::GetVersionInfo());
+  fake_reg.set_start_time(10000);
 
   // Information on replica management scheme.
   ReplicaManagementInfoPB rmi;
@@ -377,6 +381,9 @@ TEST_F(MasterTest, TestRegisterAndHeartbeat) {
     ASSERT_EQ(true, tablet_server["live"].GetBool());
     ASSERT_STREQ(VersionInfo::GetVersionInfo().c_str(),
         tablet_server["version"].GetString());
+    string start_time;
+    StringAppendStrftime(&start_time, "%Y-%m-%d %H:%M:%S %Z", (time_t)10000, true);
+    ASSERT_STREQ(start_time.c_str(), tablet_server["start_time"].GetString());
   }
 
   // Ensure that trying to re-register with a different version is OK.
@@ -391,6 +398,22 @@ TEST_F(MasterTest, TestRegisterAndHeartbeat) {
     // the numeric portion will match.
     req.mutable_registration()->set_software_version(Substitute("kudu $0 (rev SOME_NON_GIT_HASH)",
                                                                 KUDU_VERSION_STRING));
+
+    ASSERT_OK(proxy_->TSHeartbeat(req, &resp, &rpc));
+    ASSERT_FALSE(resp.has_error());
+  }
+
+  // Ensure that trying to re-register with a different start_time is OK.
+  {
+    TSHeartbeatRequestPB req;
+    TSHeartbeatResponsePB resp;
+    RpcController rpc;
+    req.mutable_common()->CopyFrom(common);
+    req.mutable_registration()->CopyFrom(fake_reg);
+    req.mutable_replica_management_info()->CopyFrom(rmi);
+    // 10 minutes later.
+    req.mutable_registration()->set_start_time(10600);
+
     ASSERT_OK(proxy_->TSHeartbeat(req, &resp, &rpc));
     ASSERT_FALSE(resp.has_error());
   }

src/kudu/master/master.cc

@@ -293,6 +293,7 @@ Status Master::InitMasterRegistration() {
     reg.set_https_enabled(web_server()->IsSecure());
   }
   reg.set_software_version(VersionInfo::GetVersionInfo());
+  reg.set_start_time(start_time_);
 
   registration_.Swap(&reg);
   registration_initialized_.store(true);

src/kudu/master/master_path_handlers.cc

@@ -17,6 +17,8 @@
 
 #include "kudu/master/master_path_handlers.h"
 
+#include <time.h>
+
 #include <algorithm>
 #include <array>
 #include <cstdint>
@@ -50,6 +52,7 @@
 #include "kudu/gutil/strings/join.h"
 #include "kudu/gutil/strings/numbers.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/walltime.h"
 #include "kudu/master/catalog_manager.h"
 #include "kudu/master/master.h"
 #include "kudu/master/master.pb.h"
@@ -129,6 +132,8 @@ void MasterPathHandlers::HandleTabletServers(const Webserver::WebRequest& /*req*
                                      reg.http_addresses(0).port());
     }
     ts_json["time_since_hb"] = StringPrintf("%.1fs", desc->TimeSinceHeartbeat().ToSeconds());
+    ts_json["start_time"] = pb_util::ParseStartTime(reg);
+    reg.clear_start_time();
     ts_json["registration"] = pb_util::SecureShortDebugString(reg);
     ts_json["location"] = desc->location().get_value_or("<none>");
     version_counts[reg.software_version()][desc->PresumedDead() ? 1 : 0]++;
@@ -458,7 +463,7 @@ void MasterPathHandlers::HandleMasters(const Webserver::WebRequest& /*req*/,
       continue;
     }
     EasyJson master_json = (*output)["masters"].PushBack(EasyJson::kObject);
-    const ServerRegistrationPB& reg = master.registration();
+    ServerRegistrationPB reg = master.registration();
     master_json["uuid"] = master.instance_id().permanent_uuid();
     if (!reg.http_addresses().empty()) {
       master_json["target"] = Substitute("$0://$1:$2/",
@@ -467,7 +472,9 @@ void MasterPathHandlers::HandleMasters(const Webserver::WebRequest& /*req*/,
                                          reg.http_addresses(0).port());
     }
     master_json["role"] = master.has_role() ? RaftPeerPB_Role_Name(master.role()) : "N/A";
-    master_json["registration"] = pb_util::SecureShortDebugString(master.registration());
+    master_json["start_time"] = pb_util::ParseStartTime(reg);
+    reg.clear_start_time();
+    master_json["registration"] = pb_util::SecureShortDebugString(reg);
   }
 }
 
@@ -645,6 +652,9 @@ void MasterPathHandlers::HandleDumpEntities(const Webserver::WebRequest& /*req*/
     jw.String("version");
     jw.String(reg.software_version());
 
+    jw.String("start_time");
+    jw.String(pb_util::ParseStartTime(reg));
+
     jw.EndObject();
   }
   jw.EndArray();

src/kudu/master/ts_descriptor-test.cc

@@ -67,6 +67,7 @@ void SetupBasicRegistrationInfo(const string& uuid,
   http_hostport->set_port(54321);
   registration->set_software_version("1.0.0");
   registration->set_https_enabled(false);
+  registration->set_start_time(10000);
 }
 
 TEST(TSDescriptorTest, TestRegistration) {

src/kudu/server/server_base.cc

@@ -43,6 +43,7 @@
 #include "kudu/gutil/port.h"
 #include "kudu/gutil/strings/strcat.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/walltime.h"
 #include "kudu/rpc/messenger.h"
 #include "kudu/rpc/remote_user.h"
 #include "kudu/rpc/result_tracker.h"
@@ -679,6 +680,8 @@ Status ServerBase::Start() {
                           "Failed to dump server info to " + options_.dump_info_path);
   }
 
+  start_time_ = WallTime_Now();
+
   return Status::OK();
 }
 

src/kudu/server/server_base.h

@@ -107,6 +107,10 @@ class ServerBase {
   // Return a PB describing the status of the server (version info, bound ports, etc)
   Status GetStatusPB(ServerStatusPB* status) const;
 
+  int64_t start_time() const {
+    return start_time_;
+  }
+
   enum {
     SUPER_USER = 1,
     USER = 1 << 1,
@@ -161,6 +165,7 @@ class ServerBase {
   void LogUnauthorizedAccess(rpc::RpcContext* rpc) const;
 
   const std::string name_;
+  int64_t start_time_;
 
   std::unique_ptr<MinidumpExceptionHandler> minidump_handler_;
   std::shared_ptr<MemTracker> mem_tracker_;

src/kudu/tools/tool_action_master.cc

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "kudu/tools/tool_action.h"
+#include <time.h>
 
 #include <algorithm>
 #include <iostream>
@@ -37,11 +37,14 @@
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/stringpiece.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/walltime.h"
 #include "kudu/master/master.h"
 #include "kudu/master/master.pb.h"
 #include "kudu/master/master.proxy.h"
+#include "kudu/tools/tool_action.h"
 #include "kudu/tools/tool_action_common.h"
 #include "kudu/util/status.h"
+#include "kudu/util/pb_util.h"
 
 DECLARE_string(columns);
 
@@ -144,6 +147,10 @@ Status ListMasters(const RunnerContext& context) {
       for (const auto& master : masters) {
         values.push_back(master.registration().software_version());
       }
+    } else if (boost::iequals(column, "start_time")) {
+      for (const auto& master : masters) {
+        values.emplace_back(std::move(pb_util::ParseStartTime(master.registration())));
+      }
     } else {
       return Status::InvalidArgument("unknown column (--columns)", column);
     }
@@ -207,7 +214,8 @@ unique_ptr<Mode> BuildMasterMode() {
       .AddOptionalParameter("columns", string("uuid,rpc-addresses"),
                             string("Comma-separated list of master info fields to "
                                    "include in output.\nPossible values: uuid, "
-                                   "rpc-addresses, http-addresses, version, and seqno"))
+                                   "rpc-addresses, http-addresses, version, seqno "
+                                   "and start_time"))
       .AddOptionalParameter("format")
       .AddOptionalParameter("timeout_ms")
       .Build();

src/kudu/tools/tool_action_tserver.cc

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "kudu/tools/tool_action.h"
+#include <time.h>
 
 #include <iostream>
 #include <memory>
@@ -35,10 +35,13 @@
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/stringpiece.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/walltime.h"
 #include "kudu/master/master.pb.h"
 #include "kudu/master/master.proxy.h"
+#include "kudu/tools/tool_action.h"
 #include "kudu/tools/tool_action_common.h"
 #include "kudu/tserver/tablet_server.h"
+#include "kudu/util/pb_util.h"
 #include "kudu/util/status.h"
 
 DECLARE_string(columns);
@@ -143,6 +146,10 @@ Status ListTServers(const RunnerContext& context) {
         string loc = server.location();
         values.emplace_back(loc.empty() ? "<none>" : std::move(loc));
       }
+    } else if (boost::iequals(column, "start_time")) {
+      for (const auto& server : servers) {
+        values.emplace_back(std::move(pb_util::ParseStartTime(server.registration())));
+      }
     } else {
       return Status::InvalidArgument("unknown column (--columns)", column);
     }
@@ -207,7 +214,7 @@ unique_ptr<Mode> BuildTServerMode() {
                             string("Comma-separated list of tserver info fields to "
                                    "include in output.\nPossible values: uuid, "
                                    "rpc-addresses, http-addresses, version, seqno, "
-                                   "and heartbeat"))
+                                   "heartbeat and start_time"))
       .AddOptionalParameter("format")
       .AddOptionalParameter("timeout_ms")
       .Build();

src/kudu/tserver/heartbeater.cc

@@ -369,6 +369,7 @@ Status Heartbeater::Thread::SetupRegistration(ServerRegistrationPB* reg) {
     reg->set_https_enabled(server_->web_server()->IsSecure());
   }
   reg->set_software_version(VersionInfo::GetVersionInfo());
+  reg->set_start_time(server_->start_time());
 
   return Status::OK();
 }

src/kudu/util/pb_util.cc

@@ -649,6 +649,19 @@ string SecureShortDebugString(const Message& msg) {
   return debug_string;
 }
 
+std::string ParseStartTime(const ServerRegistrationPB& reg) {
+  string start_time;
+  if (reg.has_start_time()) {
+    // Convert epoch time to localtime.
+    StringAppendStrftime(&start_time, "%Y-%m-%d %H:%M:%S %Z",
+                         static_cast<time_t>(reg.start_time()), true);
+  } else {
+    start_time = "<unknown>";
+  }
+
+  return start_time;
+}
+
 
 WritablePBContainerFile::WritablePBContainerFile(shared_ptr<RWFile> writer)
   : state_(FileState::NOT_INITIALIZED),

src/kudu/util/pb_util.h

@@ -30,6 +30,7 @@
 #include <google/protobuf/message.h>
 #include <gtest/gtest_prod.h>
 
+#include "kudu/common/wire_protocol.pb.h"
 #include "kudu/gutil/ref_counted.h"
 #include "kudu/util/mutex.h"
 #include "kudu/util/debug/trace_event_impl.h"
@@ -116,6 +117,9 @@ std::string SecureDebugString(const google::protobuf::Message& msg);
 // Same as SecureDebugString() above, but equivalent to Message::ShortDebugString.
 std::string SecureShortDebugString(const google::protobuf::Message& msg);
 
+// Parse 'start_time' in ServerRegistrationPB, return localtime or '<unknown>'.
+std::string ParseStartTime(const ServerRegistrationPB& reg);
+
 // A protobuf "container" has the following format (all integers in
 // little-endian byte order).
 //

www/masters.mustache

@@ -32,13 +32,15 @@ under the License.
     <thead><tr>
       <th>UUID</th>
       <th>Role</th>
+      <th>Start time</th>
       <th>Registration</th>
     </tr></thead>
     <tbody>
     {{#masters}}
       <tr>
           <td>{{#target}}<a href="{{.}}">{{/target}}{{uuid}}{{#target}}</a>{{/target}}</td>
           <td>{{role}}</td>
+          <td>{{start_time}}</td>
           <td><pre><code>{{registration}}</code></pre></td>
       </tr>
     {{/masters}}

www/tablet-servers.mustache

@@ -47,6 +47,7 @@ under the License.
       <th>UUID</th>
       <th>Location</th>
       <th>Time since heartbeat</th>
+      <th>Start time</th>
       <th>Registration</th>
     </tr></thead>
     <tbody>
@@ -55,6 +56,7 @@ under the License.
         <td>{{#target}}<a href="{{.}}">{{/target}}{{uuid}}{{#target}}</a>{{/target}}</td>
         <td>{{location}}</td>
         <td>{{time_since_hb}}</td>
+        <td>{{start_time}}</td>
         <td><pre><code>{{registration}}</code></pre></td>
       </tr>
     {{/live_tservers}}