7109928971

Committed 06 Dec 2023 03:50AM UTC coverage: 64.514% (+0.5%) from 63.997%

Build # 7109928971

Build Type

Pull #1253

github

Committed by

web-flow

Commit Message

Merge e773f89e8 into e2e83a099

Pull Request Pull Request #1253: Event drop metrics

Run Details

4411 of 8952 branches covered (0.0%)

Branch coverage included in aggregate %.

84 of 102 new or added lines in 3 files covered. (82.35%)

2 existing lines in 1 file now uncovered.

15407 of 21767 relevant lines covered (70.78%)

13474.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.65

/Source/santad/Metrics.mm

/// Copyright 2022 Google Inc. All rights reserved.
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///    http://www.apache.org/licenses/LICENSE-2.0
///
///    Unless required by applicable law or agreed to in writing, software
///    distributed under the License is distributed on an "AS IS" BASIS,
///    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
///    See the License for the specific language governing permissions and
///    limitations under the License.

#include "Source/santad/Metrics.h"
#include <EndpointSecurity/ESTypes.h>

#include <memory>

#include "Source/common/BranchPrediction.h"
#import "Source/common/SNTLogging.h"
#import "Source/common/SNTXPCMetricServiceInterface.h"
#import "Source/santad/SNTApplicationCoreMetrics.h"

static NSString *const kProcessorAuthorizer = @"Authorizer";
static NSString *const kProcessorDeviceManager = @"DeviceManager";
static NSString *const kProcessorRecorder = @"Recorder";
static NSString *const kProcessorTamperResistance = @"TamperResistance";
static NSString *const kProcessorFileAccessAuthorizer = @"FileAccessAuthorizer";

static NSString *const kEventTypeAuthClone = @"AuthClone";
static NSString *const kEventTypeAuthCopyfile = @"AuthCopyfile";
static NSString *const kEventTypeAuthCreate = @"AuthCreate";
static NSString *const kEventTypeAuthExchangedata = @"AuthExchangedata";
static NSString *const kEventTypeAuthExec = @"AuthExec";
static NSString *const kEventTypeAuthKextload = @"AuthKextload";
static NSString *const kEventTypeAuthLink = @"AuthLink";
static NSString *const kEventTypeAuthMount = @"AuthMount";
static NSString *const kEventTypeAuthOpen = @"AuthOpen";
static NSString *const kEventTypeAuthRemount = @"AuthRemount";
static NSString *const kEventTypeAuthRename = @"AuthRename";
static NSString *const kEventTypeAuthTruncate = @"AuthTruncate";
static NSString *const kEventTypeAuthUnlink = @"AuthUnlink";
static NSString *const kEventTypeNotifyClose = @"NotifyClose";
static NSString *const kEventTypeNotifyExchangedata = @"NotifyExchangedata";
static NSString *const kEventTypeNotifyExec = @"NotifyExec";
static NSString *const kEventTypeNotifyExit = @"NotifyExit";
static NSString *const kEventTypeNotifyFork = @"NotifyFork";
static NSString *const kEventTypeNotifyLink = @"NotifyLink";
static NSString *const kEventTypeNotifyRename = @"NotifyRename";
static NSString *const kEventTypeNotifyUnlink = @"NotifyUnlink";
static NSString *const kEventTypeNotifyUnmount = @"NotifyUnmount";
static NSString *const kPseudoEventTypeGlobal = @"Global";

static NSString *const kEventDispositionDropped = @"Dropped";
static NSString *const kEventDispositionProcessed = @"Processed";

// Compat values
static NSString *const kFileAccessMetricStatusOK = @"OK";
static NSString *const kFileAccessMetricStatusBlockedUser = @"BLOCKED_USER";

static NSString *const kFileAccessPolicyDecisionDenied = @"Denied";
static NSString *const kFileAccessPolicyDecisionDeniedInvalidSignature = @"Denied";
static NSString *const kFileAccessPolicyDecisionAllowedAuditOnly = @"AllowedAuditOnly";

static NSString *const kFileAccessMetricsAccessType = @"access";

namespace santa::santad {

NSString *const ProcessorToString(Processor processor) {
  switch (processor) {
    case Processor::kAuthorizer: return kProcessorAuthorizer;
    case Processor::kDeviceManager: return kProcessorDeviceManager;
    case Processor::kRecorder: return kProcessorRecorder;
    case Processor::kTamperResistance: return kProcessorTamperResistance;
    case Processor::kFileAccessAuthorizer: return kProcessorFileAccessAuthorizer;
    default:
      [NSException raise:@"Invalid processor"
                  format:@"Unknown processor value: %d", static_cast<int>(processor)];
      return nil;
  }
}

NSString *const EventTypeToString(es_event_type_t eventType) {
  switch (eventType) {
    case ES_EVENT_TYPE_AUTH_CLONE: return kEventTypeAuthClone;
    case ES_EVENT_TYPE_AUTH_COPYFILE: return kEventTypeAuthCopyfile;
    case ES_EVENT_TYPE_AUTH_CREATE: return kEventTypeAuthCreate;
    case ES_EVENT_TYPE_AUTH_EXCHANGEDATA: return kEventTypeAuthExchangedata;
    case ES_EVENT_TYPE_AUTH_EXEC: return kEventTypeAuthExec;
    case ES_EVENT_TYPE_AUTH_KEXTLOAD: return kEventTypeAuthKextload;
    case ES_EVENT_TYPE_AUTH_LINK: return kEventTypeAuthLink;
    case ES_EVENT_TYPE_AUTH_MOUNT: return kEventTypeAuthMount;
    case ES_EVENT_TYPE_AUTH_OPEN: return kEventTypeAuthOpen;
    case ES_EVENT_TYPE_AUTH_REMOUNT: return kEventTypeAuthRemount;
    case ES_EVENT_TYPE_AUTH_RENAME: return kEventTypeAuthRename;
    case ES_EVENT_TYPE_AUTH_TRUNCATE: return kEventTypeAuthTruncate;
    case ES_EVENT_TYPE_AUTH_UNLINK: return kEventTypeAuthUnlink;
    case ES_EVENT_TYPE_NOTIFY_CLOSE: return kEventTypeNotifyClose;
    case ES_EVENT_TYPE_NOTIFY_EXCHANGEDATA: return kEventTypeNotifyExchangedata;
    case ES_EVENT_TYPE_NOTIFY_EXEC: return kEventTypeNotifyExec;
    case ES_EVENT_TYPE_NOTIFY_EXIT: return kEventTypeNotifyExit;
    case ES_EVENT_TYPE_NOTIFY_FORK: return kEventTypeNotifyFork;
    case ES_EVENT_TYPE_NOTIFY_LINK: return kEventTypeNotifyLink;
    case ES_EVENT_TYPE_NOTIFY_RENAME: return kEventTypeNotifyRename;
    case ES_EVENT_TYPE_NOTIFY_UNLINK: return kEventTypeNotifyUnlink;
    case ES_EVENT_TYPE_NOTIFY_UNMOUNT: return kEventTypeNotifyUnmount;
    case ES_EVENT_TYPE_LAST: return kPseudoEventTypeGlobal;
    default:
      [NSException raise:@"Invalid event type" format:@"Invalid event type: %d", eventType];
      return nil;
  }
}

NSString *const EventDispositionToString(EventDisposition d) {
  switch (d) {
    case EventDisposition::kDropped: return kEventDispositionDropped;
    case EventDisposition::kProcessed: return kEventDispositionProcessed;
    default:
      [NSException raise:@"Invalid disposition"
                  format:@"Unknown disposition value: %d", static_cast<int>(d)];
      return nil;
  }
}

NSString *const FileAccessMetricStatusToString(FileAccessMetricStatus status) {
  switch (status) {
    case FileAccessMetricStatus::kOK: return kFileAccessMetricStatusOK;
    case FileAccessMetricStatus::kBlockedUser: return kFileAccessMetricStatusBlockedUser;
    default:
      [NSException raise:@"Invalid file access metric status"
                  format:@"Unknown file access metric status value: %d", static_cast<int>(status)];
      return nil;
  }
}

NSString *const FileAccessPolicyDecisionToString(FileAccessPolicyDecision decision) {
  switch (decision) {
    case FileAccessPolicyDecision::kDenied: return kFileAccessPolicyDecisionDenied;
    case FileAccessPolicyDecision::kDeniedInvalidSignature:
      return kFileAccessPolicyDecisionDeniedInvalidSignature;
    case FileAccessPolicyDecision::kAllowedAuditOnly:
      return kFileAccessPolicyDecisionAllowedAuditOnly;
    default:
      [NSException
         raise:@"Invalid file access policy decision"
        format:@"Unknown file access policy decision value: %d", static_cast<int>(decision)];
      return nil;
  }
}

std::shared_ptr<Metrics> Metrics::Create(SNTMetricSet *metric_set, uint64_t interval) {
  dispatch_queue_t q = dispatch_queue_create("com.google.santa.santametricsservice.q",
                                             DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL);

  dispatch_source_t timer_source = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, q);

  SNTMetricInt64Gauge *event_processing_times =
    [metric_set int64GaugeWithName:@"/santa/event_processing_time"
                        fieldNames:@[ @"Processor", @"Event" ]
                          helpText:@"Time to process various event types by each processor"];

  SNTMetricCounter *event_counts =
    [metric_set counterWithName:@"/santa/event_count"
                     fieldNames:@[ @"Processor", @"Event", @"Disposition" ]
                       helpText:@"Events received and processed by each processor"];

  SNTMetricCounter *rate_limit_counts =
    [metric_set counterWithName:@"/santa/rate_limit_count"
                     fieldNames:@[ @"Processor" ]
                       helpText:@"Events rate limited by each processor"];

  SNTMetricCounter *faa_event_counts = [metric_set
    counterWithName:@"/santa/file_access_authorizer/log/count"
         fieldNames:@[
           @"config_version", @"access_type", @"rule_id", @"status", @"operation", @"decision"
         ]
           helpText:@"Count of times a log is emitted from the File Access Authorizer client"];

  SNTMetricCounter *drop_counts =
    [metric_set counterWithName:@"/santa/event_drop_count"
                     fieldNames:@[ @"Processor", @"Event" ]
                       helpText:@"Count of the number of drops for each event"];

  std::shared_ptr<Metrics> metrics = std::make_shared<Metrics>(
    q, timer_source, interval, event_processing_times, event_counts, rate_limit_counts,
    faa_event_counts, drop_counts, metric_set, ^(Metrics *metrics) {
      SNTRegisterCoreMetrics();
      metrics->EstablishConnection();
    });

  std::weak_ptr<Metrics> weak_metrics(metrics);
  dispatch_source_set_event_handler(metrics->timer_source_, ^{
    std::shared_ptr<Metrics> shared_metrics = weak_metrics.lock();
    if (!shared_metrics) {
      return;
    }

    shared_metrics->ExportLocked(metric_set);
  });

  return metrics;
}

Metrics::Metrics(dispatch_queue_t q, dispatch_source_t timer_source, uint64_t interval,
                 SNTMetricInt64Gauge *event_processing_times, SNTMetricCounter *event_counts,
                 SNTMetricCounter *rate_limit_counts, SNTMetricCounter *faa_event_counts,
                 SNTMetricCounter *drop_counts, SNTMetricSet *metric_set,
                 void (^run_on_first_start)(Metrics *))
    : q_(q),
      timer_source_(timer_source),
      interval_(interval),
      event_processing_times_(event_processing_times),
      event_counts_(event_counts),
      rate_limit_counts_(rate_limit_counts),
      faa_event_counts_(faa_event_counts),
      drop_counts_(drop_counts),
      metric_set_(metric_set),
      run_on_first_start_(run_on_first_start) {
  SetInterval(interval_);

  events_q_ = dispatch_queue_create("com.google.santa.santametricsservice.events_q",
                                    DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL);
}

Metrics::~Metrics() {
  if (!running_) {
    // The timer_source_ must be resumed to ensure it has a proper retain count before being
    // destroyed. Additionally, it should first be cancelled to ensure the timer isn't ever fired
    // (see man page for `dispatch_source_cancel(3)`).
    dispatch_source_cancel(timer_source_);
    dispatch_resume(timer_source_);
  }
}

void Metrics::EstablishConnection() {
  MOLXPCConnection *metrics_connection = [SNTXPCMetricServiceInterface configuredConnection];
  metrics_connection.invalidationHandler = ^{
    dispatch_sync(dispatch_get_main_queue(), ^{
      LOGW(@"Metrics service connection invalidated. Reconnecting...");
      EstablishConnection();
    });
  };
  [metrics_connection resume];
  metrics_connection_ = metrics_connection;
}

void Metrics::Export() {
  dispatch_sync(q_, ^{
    ExportLocked(metric_set_);
  });
}

void Metrics::ExportLocked(SNTMetricSet *metric_set) {
  FlushMetrics();
  [[metrics_connection_ remoteObjectProxy] exportForMonitoring:[metric_set export]];
}

void Metrics::FlushMetrics() {
  dispatch_sync(events_q_, ^{
    for (const auto &kv : event_counts_cache_) {
      NSString *processorName = ProcessorToString(std::get<Processor>(kv.first));
      NSString *eventName = EventTypeToString(std::get<es_event_type_t>(kv.first));
      NSString *dispositionName = EventDispositionToString(std::get<EventDisposition>(kv.first));

      [event_counts_ incrementBy:kv.second
                  forFieldValues:@[ processorName, eventName, dispositionName ]];
    }

    for (const auto &kv : event_times_cache_) {
      NSString *processorName = ProcessorToString(std::get<Processor>(kv.first));
      NSString *eventName = EventTypeToString(std::get<es_event_type_t>(kv.first));

      [event_processing_times_ set:kv.second forFieldValues:@[ processorName, eventName ]];
    }

    for (const auto &kv : rate_limit_counts_cache_) {
      NSString *processorName = ProcessorToString(kv.first);

      [rate_limit_counts_ incrementBy:kv.second forFieldValues:@[ processorName ]];
    }

    for (const auto &kv : faa_event_counts_cache_) {
      NSString *policyVersion = @(std::get<0>(kv.first).c_str());  // FileAccessMetricsPolicyVersion
      NSString *policyName = @(std::get<1>(kv.first).c_str());     // FileAccessMetricsPolicyName
      NSString *eventName = EventTypeToString(std::get<es_event_type_t>(kv.first));
      NSString *status = FileAccessMetricStatusToString(std::get<FileAccessMetricStatus>(kv.first));
      NSString *decision =
        FileAccessPolicyDecisionToString(std::get<FileAccessPolicyDecision>(kv.first));

      [faa_event_counts_
           incrementBy:kv.second
        forFieldValues:@[
          policyVersion, kFileAccessMetricsAccessType, policyName, status, eventName, decision
        ]];
    }

    for (const auto &kv : drop_cache_) {
      if (kv.second.drops > 0) {
        NSString *processorName = ProcessorToString(std::get<Processor>(kv.first));
        NSString *eventName = EventTypeToString(std::get<es_event_type_t>(kv.first));

        [drop_counts_ incrementBy:kv.second.drops forFieldValues:@[ processorName, eventName ]];
      }
    }

    // Reset the maps so the next cycle begins with a clean state
    event_counts_cache_ = {};
    event_times_cache_ = {};
    rate_limit_counts_cache_ = {};
    faa_event_counts_cache_ = {};
    drop_cache_ = {};
  });
}

void Metrics::SetInterval(uint64_t interval) {
  dispatch_sync(q_, ^{
    LOGI(@"Setting metrics interval to %llu (exporting? %s)", interval, running_ ? "YES" : "NO");
    interval_ = interval;
    dispatch_source_set_timer(timer_source_, dispatch_time(DISPATCH_TIME_NOW, 0),
                              interval_ * NSEC_PER_SEC, 250 * NSEC_PER_MSEC);
  });
}

void Metrics::StartPoll() {
  static dispatch_once_t once_token;
  dispatch_once(&once_token, ^{
    run_on_first_start_(this);
  });

  dispatch_sync(q_, ^{
    if (!running_) {
      LOGI(@"Starting to export metrics every %llu seconds", interval_);
      running_ = true;
      dispatch_resume(timer_source_);
    } else {
      LOGW(@"Attempted to start metrics poll while already started");
    }
  });
}

void Metrics::StopPoll() {
  dispatch_sync(q_, ^{
    if (running_) {
      LOGI(@"Stopping metrics export");
      dispatch_suspend(timer_source_);
      running_ = false;
    } else {
      LOGW(@"Attempted to stop metrics poll while already stopped");
    }
  });
}

void Metrics::SetEventMetrics(Processor processor, es_event_type_t event_type,
                              EventDisposition event_disposition, int64_t nanos) {
  dispatch_sync(events_q_, ^{
    event_counts_cache_[EventCountTuple{processor, event_type, event_disposition}]++;
    event_times_cache_[EventTimesTuple{processor, event_type}] = nanos;
  });
}

void Metrics::UpdateEventStats(Processor processor, const es_message_t *msg) {
  dispatch_sync(events_q_, ^{
    EventStatsTuple event_stats_key{processor, msg->event_type};
    // NB: Using the invalid event type ES_EVENT_TYPE_LAST to store drop counts
    // based on the global sequence number.
    EventStatsTuple global_stats_key{processor, ES_EVENT_TYPE_LAST};

    SequenceStats old_event_stats = drop_cache_[event_stats_key];
    SequenceStats old_global_stats = drop_cache_[global_stats_key];

    // Sequence number should always increment by 1.
    // Drops detected if the difference between sequence numbers is ever more than 1.
    int64_t new_event_drops =
      unlikely(msg->seq_num == 0) ? 0 : (msg->seq_num - old_event_stats.seq_num - 1);
    int64_t new_global_drops =
      unlikely(msg->global_seq_num == 0) ? 0 : (msg->global_seq_num - old_global_stats.seq_num - 1);

    // Only log one or the other, prefer the event specific log.
    // For higher volume event types, we'll normally see the event-specific log eventually
    // For lower volume event types, we may only see the global drop message
    if (new_event_drops > 0) {
      LOGD(@"Drops detected for client: %@, event: %@, drops: %llu", ProcessorToString(processor),
           EventTypeToString(msg->event_type), new_event_drops);
    } else if (new_global_drops > 0) {
      LOGD(@"Drops detected globally for client: %@, drops: %llu", ProcessorToString(processor),
           new_global_drops);
    }

    drop_cache_[event_stats_key] =
      SequenceStats{.seq_num = msg->seq_num, .drops = old_event_stats.drops + new_event_drops};

    drop_cache_[global_stats_key] = SequenceStats{
      .seq_num = msg->global_seq_num, .drops = old_global_stats.drops + new_global_drops};
  });
}

void Metrics::SetRateLimitingMetrics(Processor processor, int64_t events_rate_limited_count) {
  dispatch_sync(events_q_, ^{
    rate_limit_counts_cache_[processor] += events_rate_limited_count;
  });
}

void Metrics::SetFileAccessEventMetrics(std::string policy_version, std::string rule_name,
                                        FileAccessMetricStatus status, es_event_type_t event_type,
                                        FileAccessPolicyDecision decision) {
  dispatch_sync(events_q_, ^{
    faa_event_counts_cache_[FileAccessEventCountTuple{
      std::move(policy_version), std::move(rule_name), status, event_type, decision}]++;
  });
}

}  // namespace santa::santad

1	/// Copyright 2022 Google Inc. All rights reserved.
2	///
3	/// Licensed under the Apache License, Version 2.0 (the "License");
4	/// you may not use this file except in compliance with the License.
5	/// You may obtain a copy of the License at
6	///
7	/// http://www.apache.org/licenses/LICENSE-2.0
8	///
9	/// Unless required by applicable law or agreed to in writing, software
10	/// distributed under the License is distributed on an "AS IS" BASIS,
11	/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	/// See the License for the specific language governing permissions and
13	/// limitations under the License.
14
15	#include "Source/santad/Metrics.h"
16	#include <EndpointSecurity/ESTypes.h>
17
18	#include <memory>
19
20	#include "Source/common/BranchPrediction.h"
21	#import "Source/common/SNTLogging.h"
22	#import "Source/common/SNTXPCMetricServiceInterface.h"
23	#import "Source/santad/SNTApplicationCoreMetrics.h"
24
25	static NSString *const kProcessorAuthorizer = @"Authorizer";
26	static NSString *const kProcessorDeviceManager = @"DeviceManager";
27	static NSString *const kProcessorRecorder = @"Recorder";
28	static NSString *const kProcessorTamperResistance = @"TamperResistance";
29	static NSString *const kProcessorFileAccessAuthorizer = @"FileAccessAuthorizer";
30
31	static NSString *const kEventTypeAuthClone = @"AuthClone";
32	static NSString *const kEventTypeAuthCopyfile = @"AuthCopyfile";
33	static NSString *const kEventTypeAuthCreate = @"AuthCreate";
34	static NSString *const kEventTypeAuthExchangedata = @"AuthExchangedata";
35	static NSString *const kEventTypeAuthExec = @"AuthExec";
36	static NSString *const kEventTypeAuthKextload = @"AuthKextload";
37	static NSString *const kEventTypeAuthLink = @"AuthLink";
38	static NSString *const kEventTypeAuthMount = @"AuthMount";
39	static NSString *const kEventTypeAuthOpen = @"AuthOpen";
40	static NSString *const kEventTypeAuthRemount = @"AuthRemount";
41	static NSString *const kEventTypeAuthRename = @"AuthRename";
42	static NSString *const kEventTypeAuthTruncate = @"AuthTruncate";
43	static NSString *const kEventTypeAuthUnlink = @"AuthUnlink";
44	static NSString *const kEventTypeNotifyClose = @"NotifyClose";
45	static NSString *const kEventTypeNotifyExchangedata = @"NotifyExchangedata";
46	static NSString *const kEventTypeNotifyExec = @"NotifyExec";
47	static NSString *const kEventTypeNotifyExit = @"NotifyExit";
48	static NSString *const kEventTypeNotifyFork = @"NotifyFork";
49	static NSString *const kEventTypeNotifyLink = @"NotifyLink";
50	static NSString *const kEventTypeNotifyRename = @"NotifyRename";
51	static NSString *const kEventTypeNotifyUnlink = @"NotifyUnlink";
52	static NSString *const kEventTypeNotifyUnmount = @"NotifyUnmount";
53	static NSString *const kPseudoEventTypeGlobal = @"Global";
54
55	static NSString *const kEventDispositionDropped = @"Dropped";
56	static NSString *const kEventDispositionProcessed = @"Processed";
57
58	// Compat values
59	static NSString *const kFileAccessMetricStatusOK = @"OK";
60	static NSString *const kFileAccessMetricStatusBlockedUser = @"BLOCKED_USER";
61
62	static NSString *const kFileAccessPolicyDecisionDenied = @"Denied";
63	static NSString *const kFileAccessPolicyDecisionDeniedInvalidSignature = @"Denied";
64	static NSString *const kFileAccessPolicyDecisionAllowedAuditOnly = @"AllowedAuditOnly";
65
66	static NSString *const kFileAccessMetricsAccessType = @"access";
67
68	namespace santa::santad {
69
70	NSString *const ProcessorToString(Processor processor) {	14✔
71	switch (processor) {	14✔
72	case Processor::kAuthorizer: return kProcessorAuthorizer;	5✔
73	case Processor::kDeviceManager: return kProcessorDeviceManager;	1✔
74	case Processor::kRecorder: return kProcessorRecorder;	5✔
75	case Processor::kTamperResistance: return kProcessorTamperResistance;	1✔
76	case Processor::kFileAccessAuthorizer: return kProcessorFileAccessAuthorizer;	1✔
77	default:	1✔
78	[NSException raise:@"Invalid processor"	1✔
79	format:@"Unknown processor value: %d", static_cast<int>(processor)];	1✔
80	return nil;	1✔
81	}	14✔
82	}	14✔
83
84	NSString *const EventTypeToString(es_event_type_t eventType) {	32✔
85	switch (eventType) {	32✔
86	case ES_EVENT_TYPE_AUTH_CLONE: return kEventTypeAuthClone;	1✔
87	case ES_EVENT_TYPE_AUTH_COPYFILE: return kEventTypeAuthCopyfile;	1✔
88	case ES_EVENT_TYPE_AUTH_CREATE: return kEventTypeAuthCreate;	1✔
89	case ES_EVENT_TYPE_AUTH_EXCHANGEDATA: return kEventTypeAuthExchangedata;	1✔
90	case ES_EVENT_TYPE_AUTH_EXEC: return kEventTypeAuthExec;	3✔
91	case ES_EVENT_TYPE_AUTH_KEXTLOAD: return kEventTypeAuthKextload;	1✔
92	case ES_EVENT_TYPE_AUTH_LINK: return kEventTypeAuthLink;	1✔
93	case ES_EVENT_TYPE_AUTH_MOUNT: return kEventTypeAuthMount;	1✔
94	case ES_EVENT_TYPE_AUTH_OPEN: return kEventTypeAuthOpen;	3✔
95	case ES_EVENT_TYPE_AUTH_REMOUNT: return kEventTypeAuthRemount;	1✔
96	case ES_EVENT_TYPE_AUTH_RENAME: return kEventTypeAuthRename;	1✔
97	case ES_EVENT_TYPE_AUTH_TRUNCATE: return kEventTypeAuthTruncate;	1✔
98	case ES_EVENT_TYPE_AUTH_UNLINK: return kEventTypeAuthUnlink;	1✔
99	case ES_EVENT_TYPE_NOTIFY_CLOSE: return kEventTypeNotifyClose;	1✔
100	case ES_EVENT_TYPE_NOTIFY_EXCHANGEDATA: return kEventTypeNotifyExchangedata;	1✔
101	case ES_EVENT_TYPE_NOTIFY_EXEC: return kEventTypeNotifyExec;	4✔
102	case ES_EVENT_TYPE_NOTIFY_EXIT: return kEventTypeNotifyExit;	1✔
103	case ES_EVENT_TYPE_NOTIFY_FORK: return kEventTypeNotifyFork;	1✔
104	case ES_EVENT_TYPE_NOTIFY_LINK: return kEventTypeNotifyLink;	1✔
105	case ES_EVENT_TYPE_NOTIFY_RENAME: return kEventTypeNotifyRename;	1✔
106	case ES_EVENT_TYPE_NOTIFY_UNLINK: return kEventTypeNotifyUnlink;	1✔
107	case ES_EVENT_TYPE_NOTIFY_UNMOUNT: return kEventTypeNotifyUnmount;	1✔
108	case ES_EVENT_TYPE_LAST: return kPseudoEventTypeGlobal;	2✔
109	default:	1✔
110	[NSException raise:@"Invalid event type" format:@"Invalid event type: %d", eventType];	1✔
111	return nil;	1✔
112	}	32✔
113	}	32✔
114
115	NSString *const EventDispositionToString(EventDisposition d) {	5✔
116	switch (d) {	5✔
117	case EventDisposition::kDropped: return kEventDispositionDropped;	1✔
118	case EventDisposition::kProcessed: return kEventDispositionProcessed;	3✔
119	default:	1✔
120	[NSException raise:@"Invalid disposition"	1✔
121	format:@"Unknown disposition value: %d", static_cast<int>(d)];	1✔
122	return nil;	1✔
123	}	5✔
124	}	5✔
125
126	NSString *const FileAccessMetricStatusToString(FileAccessMetricStatus status) {	4✔
127	switch (status) {	4✔
128	case FileAccessMetricStatus::kOK: return kFileAccessMetricStatusOK;	2✔
129	case FileAccessMetricStatus::kBlockedUser: return kFileAccessMetricStatusBlockedUser;	1✔
130	default:	1✔
131	[NSException raise:@"Invalid file access metric status"	1✔
132	format:@"Unknown file access metric status value: %d", static_cast<int>(status)];	1✔
133	return nil;	1✔
134	}	4✔
135	}	4✔
136
137	NSString *const FileAccessPolicyDecisionToString(FileAccessPolicyDecision decision) {	7✔
138	switch (decision) {	7✔
139	case FileAccessPolicyDecision::kDenied: return kFileAccessPolicyDecisionDenied;	2✔
140	case FileAccessPolicyDecision::kDeniedInvalidSignature:	1✔
141	return kFileAccessPolicyDecisionDeniedInvalidSignature;	1✔
142	case FileAccessPolicyDecision::kAllowedAuditOnly:	×
143	return kFileAccessPolicyDecisionAllowedAuditOnly;	×
144	default:	4✔
145	[NSException	4✔
146	raise:@"Invalid file access policy decision"	4✔
147	format:@"Unknown file access policy decision value: %d", static_cast<int>(decision)];	4✔
148	return nil;	4✔
149	}	7✔
150	}	7✔
151
152	std::shared_ptr<Metrics> Metrics::Create(SNTMetricSet *metric_set, uint64_t interval) {	20✔
153	dispatch_queue_t q = dispatch_queue_create("com.google.santa.santametricsservice.q",	20✔
154	DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL);	20✔
155
156	dispatch_source_t timer_source = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, q);	20✔
157
158	SNTMetricInt64Gauge *event_processing_times =	20✔
159	[metric_set int64GaugeWithName:@"/santa/event_processing_time"	20✔
160	fieldNames:@[ @"Processor", @"Event" ]	20✔
161	helpText:@"Time to process various event types by each processor"];	20✔
162
163	SNTMetricCounter *event_counts =	20✔
164	[metric_set counterWithName:@"/santa/event_count"	20✔
165	fieldNames:@[ @"Processor", @"Event", @"Disposition" ]	20✔
166	helpText:@"Events received and processed by each processor"];	20✔
167
168	SNTMetricCounter *rate_limit_counts =	20✔
169	[metric_set counterWithName:@"/santa/rate_limit_count"	20✔
170	fieldNames:@[ @"Processor" ]	20✔
171	helpText:@"Events rate limited by each processor"];	20✔
172
173	SNTMetricCounter *faa_event_counts = [metric_set	20✔
174	counterWithName:@"/santa/file_access_authorizer/log/count"	20✔
175	fieldNames:@[	20✔
176	@"config_version", @"access_type", @"rule_id", @"status", @"operation", @"decision"	20✔
177	]	20✔
178	helpText:@"Count of times a log is emitted from the File Access Authorizer client"];	20✔
179
180	SNTMetricCounter *drop_counts =	20✔
181	[metric_set counterWithName:@"/santa/event_drop_count"	20✔
182	fieldNames:@[ @"Processor", @"Event" ]	20✔
183	helpText:@"Count of the number of drops for each event"];	20✔
184
185	std::shared_ptr<Metrics> metrics = std::make_shared<Metrics>(	20✔
186	q, timer_source, interval, event_processing_times, event_counts, rate_limit_counts,	20✔
NEW 187	faa_event_counts, drop_counts, metric_set, ^(Metrics *metrics) {	×
NEW 188	SNTRegisterCoreMetrics();	×
NEW 189	metrics->EstablishConnection();	×
NEW 190	});	×
191
192	std::weak_ptr<Metrics> weak_metrics(metrics);	20✔
193	dispatch_source_set_event_handler(metrics->timer_source_, ^{	×
194	std::shared_ptr<Metrics> shared_metrics = weak_metrics.lock();	×
195	if (!shared_metrics) {	×
196	return;	×
197	}	×
198
199	shared_metrics->ExportLocked(metric_set);	×
200	});	×
201
202	return metrics;	20✔
203	}	20✔
204
205	Metrics::Metrics(dispatch_queue_t q, dispatch_source_t timer_source, uint64_t interval,
206	SNTMetricInt64Gauge event_processing_times, SNTMetricCounter event_counts,
207	SNTMetricCounter rate_limit_counts, SNTMetricCounter faa_event_counts,
208	SNTMetricCounter drop_counts, SNTMetricSet metric_set,
209	void (^run_on_first_start)(Metrics *))
210	: q_(q),
211	timer_source_(timer_source),
212	interval_(interval),
213	event_processing_times_(event_processing_times),
214	event_counts_(event_counts),
215	rate_limit_counts_(rate_limit_counts),
216	faa_event_counts_(faa_event_counts),
217	drop_counts_(drop_counts),
218	metric_set_(metric_set),
219	run_on_first_start_(run_on_first_start) {	27✔
220	SetInterval(interval_);	27✔
221
222	events_q_ = dispatch_queue_create("com.google.santa.santametricsservice.events_q",	27✔
223	DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL);	27✔
224	}	27✔
225
226	Metrics::~Metrics() {	27✔
227	if (!running_) {	27!
228	// The timer_source_ must be resumed to ensure it has a proper retain count before being
229	// destroyed. Additionally, it should first be cancelled to ensure the timer isn't ever fired
230	// (see man page for `dispatch_source_cancel(3)`).
231	dispatch_source_cancel(timer_source_);	27✔
232	dispatch_resume(timer_source_);	27✔
233	}	27✔
234	}	27✔
235
236	void Metrics::EstablishConnection() {	×
237	MOLXPCConnection *metrics_connection = [SNTXPCMetricServiceInterface configuredConnection];	×
238	metrics_connection.invalidationHandler = ^{	×
239	dispatch_sync(dispatch_get_main_queue(), ^{	×
240	LOGW(@"Metrics service connection invalidated. Reconnecting...");	×
241	EstablishConnection();	×
242	});	×
243	};	×
244	[metrics_connection resume];	×
245	metrics_connection_ = metrics_connection;	×
246	}	×
247
248	void Metrics::Export() {	×
249	dispatch_sync(q_, ^{	×
250	ExportLocked(metric_set_);	×
251	});	×
252	}	×
253
254	void Metrics::ExportLocked(SNTMetricSet *metric_set) {	×
255	FlushMetrics();	×
256	[[metrics_connection_ remoteObjectProxy] exportForMonitoring:[metric_set export]];	×
257	}	×
258
259	void Metrics::FlushMetrics() {	1✔
260	dispatch_sync(events_q_, ^{	1✔
261	for (const auto &kv : event_counts_cache_) {	2✔
262	NSString *processorName = ProcessorToString(std::get<Processor>(kv.first));	2✔
263	NSString *eventName = EventTypeToString(std::get<es_event_type_t>(kv.first));	2✔
264	NSString *dispositionName = EventDispositionToString(std::get<EventDisposition>(kv.first));	2✔
265
266	[event_counts_ incrementBy:kv.second	2✔
267	forFieldValues:@[ processorName, eventName, dispositionName ]];	2✔
268	}	2✔
269
270	for (const auto &kv : event_times_cache_) {	2✔
271	NSString *processorName = ProcessorToString(std::get<Processor>(kv.first));	2✔
272	NSString *eventName = EventTypeToString(std::get<es_event_type_t>(kv.first));	2✔
273
274	[event_processing_times_ set:kv.second forFieldValues:@[ processorName, eventName ]];	2✔
275	}	2✔
276
277	for (const auto &kv : rate_limit_counts_cache_) {	1✔
278	NSString *processorName = ProcessorToString(kv.first);	1✔
279
280	[rate_limit_counts_ incrementBy:kv.second forFieldValues:@[ processorName ]];	1✔
281	}	1✔
282
283	for (const auto &kv : faa_event_counts_cache_) {	1✔
284	NSString *policyVersion = @(std::get<0>(kv.first).c_str()); // FileAccessMetricsPolicyVersion	1✔
285	NSString *policyName = @(std::get<1>(kv.first).c_str()); // FileAccessMetricsPolicyName	1✔
286	NSString *eventName = EventTypeToString(std::get<es_event_type_t>(kv.first));	1✔
287	NSString *status = FileAccessMetricStatusToString(std::get<FileAccessMetricStatus>(kv.first));	1✔
288	NSString *decision =	1✔
289	FileAccessPolicyDecisionToString(std::get<FileAccessPolicyDecision>(kv.first));	1✔
290
291	[faa_event_counts_	1✔
292	incrementBy:kv.second	1✔
293	forFieldValues:@[	1✔
294	policyVersion, kFileAccessMetricsAccessType, policyName, status, eventName, decision	1✔
295	]];	1✔
296	}	1✔
297
298	for (const auto &kv : drop_cache_) {	2✔
299	if (kv.second.drops > 0) {	2!
300	NSString *processorName = ProcessorToString(std::get<Processor>(kv.first));	2✔
301	NSString *eventName = EventTypeToString(std::get<es_event_type_t>(kv.first));	2✔
302
303	[drop_counts_ incrementBy:kv.second.drops forFieldValues:@[ processorName, eventName ]];	2✔
304	}	2✔
305	}	2✔
306
307	// Reset the maps so the next cycle begins with a clean state
308	event_counts_cache_ = {};	1✔
309	event_times_cache_ = {};	1✔
310	rate_limit_counts_cache_ = {};	1✔
311	faa_event_counts_cache_ = {};	1✔
312	drop_cache_ = {};	1✔
313	});	1✔
314	}	1✔
315
316	void Metrics::SetInterval(uint64_t interval) {	28✔
317	dispatch_sync(q_, ^{	28✔
318	LOGI(@"Setting metrics interval to %llu (exporting? %s)", interval, running_ ? "YES" : "NO");	28!
319	interval_ = interval;	28✔
320	dispatch_source_set_timer(timer_source_, dispatch_time(DISPATCH_TIME_NOW, 0),	28✔
321	interval_ * NSEC_PER_SEC, 250 * NSEC_PER_MSEC);	28✔
322	});	28✔
323	}	28✔
324
325	void Metrics::StartPoll() {	2✔
326	static dispatch_once_t once_token;	2✔
327	dispatch_once(&once_token, ^{	1✔
328	run_on_first_start_(this);	1✔
329	});	1✔
330
331	dispatch_sync(q_, ^{	2✔
332	if (!running_) {	2✔
333	LOGI(@"Starting to export metrics every %llu seconds", interval_);	1✔
334	running_ = true;	1✔
335	dispatch_resume(timer_source_);	1✔
336	} else {	1✔
337	LOGW(@"Attempted to start metrics poll while already started");	1✔
338	}	1✔
339	});	2✔
340	}	2✔
341
342	void Metrics::StopPoll() {	2✔
343	dispatch_sync(q_, ^{	2✔
344	if (running_) {	2✔
345	LOGI(@"Stopping metrics export");	1✔
346	dispatch_suspend(timer_source_);	1✔
347	running_ = false;	1✔
348	} else {	1✔
349	LOGW(@"Attempted to stop metrics poll while already stopped");	1✔
350	}	1✔
351	});	2✔
352	}	2✔
353
354	void Metrics::SetEventMetrics(Processor processor, es_event_type_t event_type,
355	EventDisposition event_disposition, int64_t nanos) {	5✔
356	dispatch_sync(events_q_, ^{	5✔
357	event_counts_cache_[EventCountTuple{processor, event_type, event_disposition}]++;	5✔
358	event_times_cache_[EventTimesTuple{processor, event_type}] = nanos;	5✔
359	});	5✔
360	}	5✔
361
362	void Metrics::UpdateEventStats(Processor processor, const es_message_t *msg) {	4✔
363	dispatch_sync(events_q_, ^{	4✔
364	EventStatsTuple event_stats_key{processor, msg->event_type};	4✔
365	// NB: Using the invalid event type ES_EVENT_TYPE_LAST to store drop counts
366	// based on the global sequence number.
367	EventStatsTuple global_stats_key{processor, ES_EVENT_TYPE_LAST};	4✔
368
369	SequenceStats old_event_stats = drop_cache_[event_stats_key];	4✔
370	SequenceStats old_global_stats = drop_cache_[global_stats_key];	4✔
371
372	// Sequence number should always increment by 1.
373	// Drops detected if the difference between sequence numbers is ever more than 1.
374	int64_t new_event_drops =	4✔
375	unlikely(msg->seq_num == 0) ? 0 : (msg->seq_num - old_event_stats.seq_num - 1);	4✔
376	int64_t new_global_drops =	4✔
377	unlikely(msg->global_seq_num == 0) ? 0 : (msg->global_seq_num - old_global_stats.seq_num - 1);	4✔
378
379	// Only log one or the other, prefer the event specific log.
380	// For higher volume event types, we'll normally see the event-specific log eventually
381	// For lower volume event types, we may only see the global drop message
382	if (new_event_drops > 0) {	4✔
383	LOGD(@"Drops detected for client: %@, event: %@, drops: %llu", ProcessorToString(processor),	2✔
384	EventTypeToString(msg->event_type), new_event_drops);	2✔
385	} else if (new_global_drops > 0) {	2!
NEW 386	LOGD(@"Drops detected globally for client: %@, drops: %llu", ProcessorToString(processor),	×
NEW 387	new_global_drops);	×
NEW 388	}	×
389
390	drop_cache_[event_stats_key] =	4✔
391	SequenceStats{.seq_num = msg->seq_num, .drops = old_event_stats.drops + new_event_drops};	4✔
392
393	drop_cache_[global_stats_key] = SequenceStats{	4✔
394	.seq_num = msg->global_seq_num, .drops = old_global_stats.drops + new_global_drops};	4✔
395	});	4✔
396	}	4✔
397
398	void Metrics::SetRateLimitingMetrics(Processor processor, int64_t events_rate_limited_count) {	4✔
399	dispatch_sync(events_q_, ^{	4✔
400	rate_limit_counts_cache_[processor] += events_rate_limited_count;	4✔
401	});	4✔
402	}	4✔
403
404	void Metrics::SetFileAccessEventMetrics(std::string policy_version, std::string rule_name,
405	FileAccessMetricStatus status, es_event_type_t event_type,
406	FileAccessPolicyDecision decision) {	4✔
407	dispatch_sync(events_q_, ^{	4✔
408	faa_event_counts_cache_[FileAccessEventCountTuple{	4✔
409	std::move(policy_version), std::move(rule_name), status, event_type, decision}]++;	4✔
410	});	4✔
411	}	4✔
412
413	} // namespace santa::santad

google / santa / 7109928971

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous