Ship failure logs to Scuba

Summary:
Added Scuba logging for Platform Manager exploration errors

Build considerations:
* OSS build is unaffected by using a no-op implementation in `oss/ScubaLogger.cpp`

Memory usage:
* Previous issues were encountered because including a logging library ended up causing OOMs. I ran a quick analysis of two PM runs one with Scuba and one without - saw minimal difference:
```
[root@fboss332610281.ash7 ~]# cat memory_a.log
Peak RSS: 424584 KB
[root@fboss332610281.ash7 ~]# cat memory_b.log
Peak RSS: 428384 KB
```

* Log calls are wrapped in  try/catch to prevent any failures from Scuba propagating back up and causing problems

- Logs are sent to the `perfpipe_fboss_platform_manager_errors` Scuba category
- Each error log includes:
  - Device hostname (from NetworkUtil::getLocalHost)
  - Platform name
  - Device path
  - Error type
  - Error message
  - Exploration status
  - Timestamp

Reviewed By: RomanChoporov-Meta

Differential Revision: D85386916

fbshipit-source-id: 3b67dd7563178f1ac35555621daf2432128f50f6

Author: Scott8440

cmake/PlatformPlatformManager.cmake

@@ -140,6 +140,14 @@ target_link_libraries(platform_manager_device_path_resolver
   weutil_eeprom_contents_cpp2
 )
 
+add_library(scuba_logger
+  fboss/platform/platform_manager/oss/ScubaLogger.cpp
+)
+
+target_link_libraries(scuba_logger
+  fmt::fmt
+)
+
 add_library(platform_manager_platform_explorer
   fboss/platform/platform_manager/PlatformExplorer.cpp
   fboss/platform/platform_manager/ExplorationSummary.cpp
@@ -159,6 +167,7 @@ target_link_libraries(platform_manager_platform_explorer
   weutil_fboss_eeprom_interface
   ioctl_smbus_eeprom_reader
   Folly::folly
+  scuba_logger
 )
 
 add_library(platform_manager_config_validator
@@ -222,6 +231,7 @@ target_link_libraries(platform_manager
   ${SYSTEMD}
   gpiod_line
   range-v3
+  scuba_logger
 )
 
 install(TARGETS platform_manager)

fboss/platform/platform_manager/BUCK

@@ -168,6 +168,7 @@ cpp_library(
         ":platform_manager_config-cpp2-types",
         ":platform_manager_service-cpp2-types",
         ":presence_checker",
+        ":scuba_logger",
         ":utils",
         "//fb303:service_data",
         "//fboss/platform/helpers:platform_fs_utils",
@@ -179,6 +180,19 @@ cpp_library(
     ],
 )
 
+cpp_library(
+    name = "scuba_logger",
+    srcs = [
+        "ScubaLogger.cpp",
+    ],
+    exported_deps = [
+        "//common/network:util",
+        "//folly/json:dynamic",
+        "//folly/logging:logging",
+        "//scribe/client:scribe_client_no_sr",
+    ],
+)
+
 cpp_library(
     name = "utils",
     srcs = [
@@ -253,6 +267,7 @@ cpp_binary(
         ":config_utils",
         ":pkg_manager",
         ":platform_manager_handler",
+        ":scuba_logger",
         "//fb303:logging",
         "//fb303:service_data",
         "//fboss/platform/helpers:init",

fboss/platform/platform_manager/ExplorationSummary.cpp

@@ -6,6 +6,7 @@
 #include <folly/logging/xlog.h>
 #include <re2/re2.h>
 
+#include "fboss/platform/platform_manager/ScubaLogger.h"
 #include "fboss/platform/platform_manager/Utils.h"
 
 namespace facebook::fboss::platform::platform_manager {
@@ -92,6 +93,25 @@ void ExplorationSummary::publishCounters(ExplorationStatus finalStatus) {
   }
 }
 
+void ExplorationSummary::publishToScuba(ExplorationStatus finalStatus) {
+  // Log individual errors
+  for (const auto& [devicePath, explorationErrors] : devicePathToErrors_) {
+    for (const auto& error : explorationErrors) {
+      std::unordered_map<std::string, std::string> normals;
+
+      normals["platform"] = *platformConfig_.platformName();
+      normals["device_path"] = devicePath;
+      normals["event"] = *error.errorType();
+      normals["error_message"] = *error.message();
+      normals["exploration_status"] =
+          apache::thrift::util::enumNameSafe(finalStatus);
+
+      ScubaLogger::log(normals);
+      XLOG(INFO) << "Logged Platform Manager error to Scuba: " << devicePath;
+    }
+  }
+}
+
 ExplorationStatus ExplorationSummary::summarize() {
   ExplorationStatus finalStatus = ExplorationStatus::FAILED;
   if (devicePathToErrors_.empty() && devicePathToExpectedErrors_.empty()) {
@@ -102,6 +122,7 @@ ExplorationStatus ExplorationSummary::summarize() {
   // Exploration summary reporting.
   print(finalStatus);
   publishCounters(finalStatus);
+  publishToScuba(finalStatus);
   return finalStatus;
 }
 

fboss/platform/platform_manager/ExplorationSummary.h

@@ -124,5 +124,6 @@ class ExplorationSummary {
 
   void print(ExplorationStatus finalStatus);
   void publishCounters(ExplorationStatus finalStatus);
+  void publishToScuba(ExplorationStatus finalStatus);
 };
 } // namespace facebook::fboss::platform::platform_manager

fboss/platform/platform_manager/Main.cpp

@@ -12,6 +12,7 @@
 #include "fboss/platform/platform_manager/ConfigUtils.h"
 #include "fboss/platform/platform_manager/PkgManager.h"
 #include "fboss/platform/platform_manager/PlatformManagerHandler.h"
+#include "fboss/platform/platform_manager/ScubaLogger.h"
 
 using namespace facebook;
 using namespace facebook::fboss::platform;
@@ -49,54 +50,67 @@ int main(int argc, char** argv) {
   fb303::registerFollyLoggingOptionHandlers();
   helpers::init(&argc, &argv);
 
-  auto config = ConfigUtils().getConfig();
+  try {
+    auto config = ConfigUtils().getConfig();
+
+    PkgManager pkgManager(config);
+    pkgManager.processAll();
+    PlatformExplorer platformExplorer(config);
+    platformExplorer.explore();
+    if (FLAGS_run_once) {
+      XLOG(INFO) << fmt::format(
+          "Ran with --run_once={}. Skipping running as a daemon...",
+          FLAGS_run_once);
+      return 0;
+    }
 
-  PkgManager pkgManager(config);
-  pkgManager.processAll();
-  PlatformExplorer platformExplorer(config);
-  platformExplorer.explore();
-  if (FLAGS_run_once) {
-    XLOG(INFO) << fmt::format(
-        "Ran with --run_once={}. Skipping running as a daemon...",
-        FLAGS_run_once);
-    return 0;
-  }
+    // When systemd starts PlatformManager, it sets the below env in PM
+    // environment. This is a path to Unix domain socket at /run/systemd/notify.
+    if (!FLAGS_run_in_netos) {
+      const auto notifySocketEnv{"NOTIFY_SOCKET"};
+      if (std::getenv(notifySocketEnv)) {
+        sdNotifyReady();
+      } else {
+        XLOG(WARNING) << fmt::format(
+            "Skipping sd_notify since ${} is not set which does not "
+            "imply systemd execution.",
+            notifySocketEnv);
+      }
+    }
 
-  // When systemd starts PlatformManager, it sets the below env in PM
-  // environment. This is a path to Unix domain socket at /run/systemd/notify.
-  if (!FLAGS_run_in_netos) {
-    const auto notifySocketEnv{"NOTIFY_SOCKET"};
-    if (std::getenv(notifySocketEnv)) {
-      sdNotifyReady();
-    } else {
-      XLOG(WARNING) << fmt::format(
-          "Skipping sd_notify since ${} is not set which does not "
-          "imply systemd execution.",
-          notifySocketEnv);
+    std::optional<DataStore> ds = platformExplorer.getDataStore();
+    if (!ds.has_value()) {
+      XLOG(ERR)
+          << "PlatformExplorer did not succeed. Not starting Thrift Service.";
+      return 1;
     }
-  }
+    XLOG(INFO) << "Running PlatformManager thrift service...";
 
-  std::optional<DataStore> ds = platformExplorer.getDataStore();
-  if (!ds.has_value()) {
-    XLOG(ERR)
-        << "PlatformExplorer did not succeed. Not starting Thrift Service.";
-    return 1;
-  }
-  XLOG(INFO) << "Running PlatformManager thrift service...";
+    auto server = std::make_shared<apache::thrift::ThriftServer>();
+    auto handler = std::make_shared<PlatformManagerHandler>(
+        platformExplorer, ds.value(), config);
+    server->setPort(FLAGS_thrift_port);
+    server->setInterface(handler);
+    server->setAllowPlaintextOnLoopback(true);
+
+    auto evb = server->getEventBaseManager()->getEventBase();
+    helpers::SignalHandler signalHandler(evb, server);
 
-  auto server = std::make_shared<apache::thrift::ThriftServer>();
-  auto handler = std::make_shared<PlatformManagerHandler>(
-      platformExplorer, ds.value(), config);
-  server->setPort(FLAGS_thrift_port);
-  server->setInterface(handler);
-  server->setAllowPlaintextOnLoopback(true);
+    helpers::runThriftService(
+        server, handler, "PlatformManagerService", FLAGS_thrift_port);
 
-  auto evb = server->getEventBaseManager()->getEventBase();
-  helpers::SignalHandler signalHandler(evb, server);
+    XLOG(INFO) << "================ STOPPED PLATFORM BINARY ================";
+    return 0;
+  } catch (const std::exception& ex) {
+    // Log unexpected crash to Scuba
+    std::unordered_map<std::string, std::string> normals;
 
-  helpers::runThriftService(
-      server, handler, "PlatformManagerService", FLAGS_thrift_port);
+    normals["event"] = "unexpected_crash";
+    normals["error_message"] = ex.what();
 
-  XLOG(INFO) << "================ STOPPED PLATFORM BINARY ================";
-  return 0;
+    ScubaLogger::log(normals);
+    XLOG(FATAL) << "Platform Manager crashed with unexpected exception: "
+                << ex.what();
+    return 1;
+  }
 }

fboss/platform/platform_manager/ScubaLogger.cpp

@@ -0,0 +1,50 @@
+// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
+
+#include "fboss/platform/platform_manager/ScubaLogger.h"
+
+#include <chrono>
+
+#include <folly/json/dynamic.h>
+#include <folly/json/json.h>
+#include <folly/logging/xlog.h>
+#include "common/network/NetworkUtil.h"
+#include "scribe/client/ScribeClient.h"
+
+namespace facebook::fboss::platform::platform_manager {
+namespace ScubaLogger {
+
+void log(
+    const std::unordered_map<std::string, std::string>& normals,
+    const std::unordered_map<std::string, int64_t>& ints,
+    const std::string& category) {
+  folly::dynamic normalsObj = folly::dynamic::object;
+  normalsObj["hostname"] = network::NetworkUtil::getLocalHost(true);
+  for (const auto& [key, value] : normals) {
+    normalsObj[key] = value;
+  }
+
+  folly::dynamic intsObj = folly::dynamic::object;
+  intsObj["time"] = std::chrono::duration_cast<std::chrono::seconds>(
+                        std::chrono::system_clock::now().time_since_epoch())
+                        .count();
+  for (const auto& [key, value] : ints) {
+    intsObj[key] = value;
+  }
+
+  folly::dynamic scubaObj = folly::dynamic::object;
+  scubaObj["normal"] = normalsObj;
+  scubaObj["int"] = intsObj;
+
+  try {
+    scribe::ScribeClient::getLite()->offer(category, folly::toJson(scubaObj));
+    XLOG(DBG2) << "Successfully logged to Scuba (" << category
+               << "): " << folly::toPrettyJson(scubaObj);
+  } catch (const std::exception& e) {
+    XLOG(ERR) << "Exception: " << folly::exceptionStr(e)
+              << " while logging to Scuba (" << category
+              << "): " << folly::toPrettyJson(scubaObj);
+  }
+}
+
+} // namespace ScubaLogger
+} // namespace facebook::fboss::platform::platform_manager

fboss/platform/platform_manager/ScubaLogger.h

@@ -0,0 +1,38 @@
+// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
+
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+namespace facebook::fboss::platform::platform_manager {
+
+/**
+ * ScubaLogger provides utilities for logging data to Scuba.
+ * In OSS builds, this is a no-op stub.
+ * In internal builds, this logs data via ScribeClient.
+ */
+namespace ScubaLogger {
+
+// Default Scuba category for Platform Manager logs
+constexpr const char* kDefaultCategory = "perfpipe_fboss_platform_manager";
+
+/**
+ * Log an event to Scuba with normal (string) and int fields.
+ *
+ * Note: The following fields are automatically added:
+ *   - "hostname" (string): The hostname from getLocalHost()
+ *   - "time" (int): Unix timestamp in seconds
+ *
+ * @param normals Map of string field names to string values
+ * @param ints Map of string field names to int64_t values
+ * @param category The Scuba category to log to
+ */
+void log(
+    const std::unordered_map<std::string, std::string>& normals,
+    const std::unordered_map<std::string, int64_t>& ints = {},
+    const std::string& category = kDefaultCategory);
+
+} // namespace ScubaLogger
+
+} // namespace facebook::fboss::platform::platform_manager

fboss/platform/platform_manager/oss/ScubaLogger.cpp

@@ -0,0 +1,19 @@
+// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
+
+#include "fboss/platform/platform_manager/ScubaLogger.h"
+
+#include <folly/logging/xlog.h>
+
+namespace facebook::fboss::platform::platform_manager {
+namespace ScubaLogger {
+
+void log(
+    const std::unordered_map<std::string, std::string>& normals,
+    const std::unordered_map<std::string, int64_t>& ints,
+    const std::string& category) {
+  // No-op: Scuba logging is not available in OSS builds
+  XLOG(DBG3) << "Scuba logging is not available in OSS builds. ";
+}
+
+} // namespace ScubaLogger
+} // namespace facebook::fboss::platform::platform_manager