diff --git a/daemon/instance.go b/daemon/instance.go index 4ed741822..9f950c643 100644 --- a/daemon/instance.go +++ b/daemon/instance.go @@ -87,12 +87,17 @@ func (s *StartedService) newInstance(profileContent string, overrideOptions *Ove } } } - if s.oomKiller && C.IsIos { + if s.oomKillerEnabled { if !common.Any(options.Services, func(it option.Service) bool { return it.Type == C.TypeOOMKiller }) { + oomOptions := &option.OOMKillerServiceOptions{ + KillerDisabled: s.oomKillerDisabled, + MemoryLimitOverride: s.oomMemoryLimit, + } options.Services = append(options.Services, option.Service{ - Type: C.TypeOOMKiller, + Type: C.TypeOOMKiller, + Options: oomOptions, }) } } diff --git a/daemon/started_service.go b/daemon/started_service.go index facbc66c7..5fa248a30 100644 --- a/daemon/started_service.go +++ b/daemon/started_service.go @@ -14,6 +14,7 @@ import ( "github.com/sagernet/sing-box/experimental/deprecated" "github.com/sagernet/sing-box/log" "github.com/sagernet/sing-box/protocol/group" + "github.com/sagernet/sing-box/service/oomkiller" "github.com/sagernet/sing/common" "github.com/sagernet/sing/common/batch" E "github.com/sagernet/sing/common/exceptions" @@ -34,10 +35,12 @@ var _ StartedServiceServer = (*StartedService)(nil) type StartedService struct { ctx context.Context // platform adapter.PlatformInterface - handler PlatformHandler - debug bool - logMaxLines int - oomKiller bool + handler PlatformHandler + debug bool + logMaxLines int + oomKillerEnabled bool + oomKillerDisabled bool + oomMemoryLimit uint64 // workingDirectory string // tempDirectory string // userID int @@ -66,10 +69,12 @@ type StartedService struct { type ServiceOptions struct { Context context.Context // Platform adapter.PlatformInterface - Handler PlatformHandler - Debug bool - LogMaxLines int - OOMKiller bool + Handler PlatformHandler + Debug bool + LogMaxLines int + OOMKillerEnabled bool + OOMKillerDisabled bool + OOMMemoryLimit uint64 // WorkingDirectory string // TempDirectory string // UserID int @@ -81,10 +86,12 @@ func NewStartedService(options ServiceOptions) *StartedService { s := &StartedService{ ctx: options.Context, // platform: options.Platform, - handler: options.Handler, - debug: options.Debug, - logMaxLines: options.LogMaxLines, - oomKiller: options.OOMKiller, + handler: options.Handler, + debug: options.Debug, + logMaxLines: options.LogMaxLines, + oomKillerEnabled: options.OOMKillerEnabled, + oomKillerDisabled: options.OOMKillerDisabled, + oomMemoryLimit: options.OOMMemoryLimit, // workingDirectory: options.WorkingDirectory, // tempDirectory: options.TempDirectory, // userID: options.UserID, @@ -710,6 +717,21 @@ func (s *StartedService) TriggerDebugCrash(ctx context.Context, request *DebugCr return &emptypb.Empty{}, nil } +func (s *StartedService) TriggerOOMReport(ctx context.Context, _ *emptypb.Empty) (*emptypb.Empty, error) { + if !s.debug { + return nil, status.Error(codes.PermissionDenied, "OOM report trigger unavailable") + } + instance := s.Instance() + if instance == nil { + return nil, status.Error(codes.FailedPrecondition, "service not started") + } + reporter := service.FromContext[oomkiller.OOMReporter](instance.ctx) + if reporter == nil { + return nil, status.Error(codes.Unavailable, "OOM reporter not available") + } + return &emptypb.Empty{}, reporter.WriteReport(memory.Total()) +} + func (s *StartedService) SubscribeConnections(request *SubscribeConnectionsRequest, server grpc.ServerStreamingServer[ConnectionEvents]) error { err := s.waitForStarted(server.Context()) if err != nil { diff --git a/daemon/started_service.pb.go b/daemon/started_service.pb.go index d7103b3ba..403ba6605 100644 --- a/daemon/started_service.pb.go +++ b/daemon/started_service.pb.go @@ -2008,7 +2008,7 @@ const file_daemon_started_service_proto_rawDesc = "" + "\x13ConnectionEventType\x12\x18\n" + "\x14CONNECTION_EVENT_NEW\x10\x00\x12\x1b\n" + "\x17CONNECTION_EVENT_UPDATE\x10\x01\x12\x1b\n" + - "\x17CONNECTION_EVENT_CLOSED\x10\x022\xaf\f\n" + + "\x17CONNECTION_EVENT_CLOSED\x10\x022\xf5\f\n" + "\x0eStartedService\x12=\n" + "\vStopService\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\x12?\n" + "\rReloadService\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\x12K\n" + @@ -2026,7 +2026,8 @@ const file_daemon_started_service_proto_rawDesc = "" + "\x0eSetGroupExpand\x12\x1d.daemon.SetGroupExpandRequest\x1a\x16.google.protobuf.Empty\"\x00\x12K\n" + "\x14GetSystemProxyStatus\x12\x16.google.protobuf.Empty\x1a\x19.daemon.SystemProxyStatus\"\x00\x12W\n" + "\x15SetSystemProxyEnabled\x12$.daemon.SetSystemProxyEnabledRequest\x1a\x16.google.protobuf.Empty\"\x00\x12H\n" + - "\x11TriggerDebugCrash\x12\x19.daemon.DebugCrashRequest\x1a\x16.google.protobuf.Empty\"\x00\x12Y\n" + + "\x11TriggerDebugCrash\x12\x19.daemon.DebugCrashRequest\x1a\x16.google.protobuf.Empty\"\x00\x12D\n" + + "\x10TriggerOOMReport\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12Y\n" + "\x14SubscribeConnections\x12#.daemon.SubscribeConnectionsRequest\x1a\x18.daemon.ConnectionEvents\"\x000\x01\x12K\n" + "\x0fCloseConnection\x12\x1e.daemon.CloseConnectionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12G\n" + "\x13CloseAllConnections\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12M\n" + @@ -2114,35 +2115,37 @@ var file_daemon_started_service_proto_depIdxs = []int32{ 31, // 26: daemon.StartedService.GetSystemProxyStatus:input_type -> google.protobuf.Empty 19, // 27: daemon.StartedService.SetSystemProxyEnabled:input_type -> daemon.SetSystemProxyEnabledRequest 20, // 28: daemon.StartedService.TriggerDebugCrash:input_type -> daemon.DebugCrashRequest - 21, // 29: daemon.StartedService.SubscribeConnections:input_type -> daemon.SubscribeConnectionsRequest - 26, // 30: daemon.StartedService.CloseConnection:input_type -> daemon.CloseConnectionRequest - 31, // 31: daemon.StartedService.CloseAllConnections:input_type -> google.protobuf.Empty - 31, // 32: daemon.StartedService.GetDeprecatedWarnings:input_type -> google.protobuf.Empty - 31, // 33: daemon.StartedService.GetStartedAt:input_type -> google.protobuf.Empty - 31, // 34: daemon.StartedService.StopService:output_type -> google.protobuf.Empty - 31, // 35: daemon.StartedService.ReloadService:output_type -> google.protobuf.Empty - 4, // 36: daemon.StartedService.SubscribeServiceStatus:output_type -> daemon.ServiceStatus - 7, // 37: daemon.StartedService.SubscribeLog:output_type -> daemon.Log - 8, // 38: daemon.StartedService.GetDefaultLogLevel:output_type -> daemon.DefaultLogLevel - 31, // 39: daemon.StartedService.ClearLogs:output_type -> google.protobuf.Empty - 9, // 40: daemon.StartedService.SubscribeStatus:output_type -> daemon.Status - 10, // 41: daemon.StartedService.SubscribeGroups:output_type -> daemon.Groups - 17, // 42: daemon.StartedService.GetClashModeStatus:output_type -> daemon.ClashModeStatus - 16, // 43: daemon.StartedService.SubscribeClashMode:output_type -> daemon.ClashMode - 31, // 44: daemon.StartedService.SetClashMode:output_type -> google.protobuf.Empty - 31, // 45: daemon.StartedService.URLTest:output_type -> google.protobuf.Empty - 31, // 46: daemon.StartedService.SelectOutbound:output_type -> google.protobuf.Empty - 31, // 47: daemon.StartedService.SetGroupExpand:output_type -> google.protobuf.Empty - 18, // 48: daemon.StartedService.GetSystemProxyStatus:output_type -> daemon.SystemProxyStatus - 31, // 49: daemon.StartedService.SetSystemProxyEnabled:output_type -> google.protobuf.Empty - 31, // 50: daemon.StartedService.TriggerDebugCrash:output_type -> google.protobuf.Empty - 23, // 51: daemon.StartedService.SubscribeConnections:output_type -> daemon.ConnectionEvents - 31, // 52: daemon.StartedService.CloseConnection:output_type -> google.protobuf.Empty - 31, // 53: daemon.StartedService.CloseAllConnections:output_type -> google.protobuf.Empty - 27, // 54: daemon.StartedService.GetDeprecatedWarnings:output_type -> daemon.DeprecatedWarnings - 29, // 55: daemon.StartedService.GetStartedAt:output_type -> daemon.StartedAt - 34, // [34:56] is the sub-list for method output_type - 12, // [12:34] is the sub-list for method input_type + 31, // 29: daemon.StartedService.TriggerOOMReport:input_type -> google.protobuf.Empty + 21, // 30: daemon.StartedService.SubscribeConnections:input_type -> daemon.SubscribeConnectionsRequest + 26, // 31: daemon.StartedService.CloseConnection:input_type -> daemon.CloseConnectionRequest + 31, // 32: daemon.StartedService.CloseAllConnections:input_type -> google.protobuf.Empty + 31, // 33: daemon.StartedService.GetDeprecatedWarnings:input_type -> google.protobuf.Empty + 31, // 34: daemon.StartedService.GetStartedAt:input_type -> google.protobuf.Empty + 31, // 35: daemon.StartedService.StopService:output_type -> google.protobuf.Empty + 31, // 36: daemon.StartedService.ReloadService:output_type -> google.protobuf.Empty + 4, // 37: daemon.StartedService.SubscribeServiceStatus:output_type -> daemon.ServiceStatus + 7, // 38: daemon.StartedService.SubscribeLog:output_type -> daemon.Log + 8, // 39: daemon.StartedService.GetDefaultLogLevel:output_type -> daemon.DefaultLogLevel + 31, // 40: daemon.StartedService.ClearLogs:output_type -> google.protobuf.Empty + 9, // 41: daemon.StartedService.SubscribeStatus:output_type -> daemon.Status + 10, // 42: daemon.StartedService.SubscribeGroups:output_type -> daemon.Groups + 17, // 43: daemon.StartedService.GetClashModeStatus:output_type -> daemon.ClashModeStatus + 16, // 44: daemon.StartedService.SubscribeClashMode:output_type -> daemon.ClashMode + 31, // 45: daemon.StartedService.SetClashMode:output_type -> google.protobuf.Empty + 31, // 46: daemon.StartedService.URLTest:output_type -> google.protobuf.Empty + 31, // 47: daemon.StartedService.SelectOutbound:output_type -> google.protobuf.Empty + 31, // 48: daemon.StartedService.SetGroupExpand:output_type -> google.protobuf.Empty + 18, // 49: daemon.StartedService.GetSystemProxyStatus:output_type -> daemon.SystemProxyStatus + 31, // 50: daemon.StartedService.SetSystemProxyEnabled:output_type -> google.protobuf.Empty + 31, // 51: daemon.StartedService.TriggerDebugCrash:output_type -> google.protobuf.Empty + 31, // 52: daemon.StartedService.TriggerOOMReport:output_type -> google.protobuf.Empty + 23, // 53: daemon.StartedService.SubscribeConnections:output_type -> daemon.ConnectionEvents + 31, // 54: daemon.StartedService.CloseConnection:output_type -> google.protobuf.Empty + 31, // 55: daemon.StartedService.CloseAllConnections:output_type -> google.protobuf.Empty + 27, // 56: daemon.StartedService.GetDeprecatedWarnings:output_type -> daemon.DeprecatedWarnings + 29, // 57: daemon.StartedService.GetStartedAt:output_type -> daemon.StartedAt + 35, // [35:58] is the sub-list for method output_type + 12, // [12:35] is the sub-list for method input_type 12, // [12:12] is the sub-list for extension type_name 12, // [12:12] is the sub-list for extension extendee 0, // [0:12] is the sub-list for field type_name diff --git a/daemon/started_service.proto b/daemon/started_service.proto index 890ffdb54..3434c3f19 100644 --- a/daemon/started_service.proto +++ b/daemon/started_service.proto @@ -27,6 +27,7 @@ service StartedService { rpc GetSystemProxyStatus(google.protobuf.Empty) returns(SystemProxyStatus) {} rpc SetSystemProxyEnabled(SetSystemProxyEnabledRequest) returns(google.protobuf.Empty) {} rpc TriggerDebugCrash(DebugCrashRequest) returns(google.protobuf.Empty) {} + rpc TriggerOOMReport(google.protobuf.Empty) returns(google.protobuf.Empty) {} rpc SubscribeConnections(SubscribeConnectionsRequest) returns(stream ConnectionEvents) {} rpc CloseConnection(CloseConnectionRequest) returns(google.protobuf.Empty) {} diff --git a/daemon/started_service_grpc.pb.go b/daemon/started_service_grpc.pb.go index e61e634b7..bdf81e4a6 100644 --- a/daemon/started_service_grpc.pb.go +++ b/daemon/started_service_grpc.pb.go @@ -32,6 +32,7 @@ const ( StartedService_GetSystemProxyStatus_FullMethodName = "/daemon.StartedService/GetSystemProxyStatus" StartedService_SetSystemProxyEnabled_FullMethodName = "/daemon.StartedService/SetSystemProxyEnabled" StartedService_TriggerDebugCrash_FullMethodName = "/daemon.StartedService/TriggerDebugCrash" + StartedService_TriggerOOMReport_FullMethodName = "/daemon.StartedService/TriggerOOMReport" StartedService_SubscribeConnections_FullMethodName = "/daemon.StartedService/SubscribeConnections" StartedService_CloseConnection_FullMethodName = "/daemon.StartedService/CloseConnection" StartedService_CloseAllConnections_FullMethodName = "/daemon.StartedService/CloseAllConnections" @@ -60,6 +61,7 @@ type StartedServiceClient interface { GetSystemProxyStatus(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*SystemProxyStatus, error) SetSystemProxyEnabled(ctx context.Context, in *SetSystemProxyEnabledRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) TriggerDebugCrash(ctx context.Context, in *DebugCrashRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + TriggerOOMReport(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*emptypb.Empty, error) SubscribeConnections(ctx context.Context, in *SubscribeConnectionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ConnectionEvents], error) CloseConnection(ctx context.Context, in *CloseConnectionRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) CloseAllConnections(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*emptypb.Empty, error) @@ -290,6 +292,16 @@ func (c *startedServiceClient) TriggerDebugCrash(ctx context.Context, in *DebugC return out, nil } +func (c *startedServiceClient) TriggerOOMReport(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, StartedService_TriggerOOMReport_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *startedServiceClient) SubscribeConnections(ctx context.Context, in *SubscribeConnectionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ConnectionEvents], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &StartedService_ServiceDesc.Streams[5], StartedService_SubscribeConnections_FullMethodName, cOpts...) @@ -370,6 +382,7 @@ type StartedServiceServer interface { GetSystemProxyStatus(context.Context, *emptypb.Empty) (*SystemProxyStatus, error) SetSystemProxyEnabled(context.Context, *SetSystemProxyEnabledRequest) (*emptypb.Empty, error) TriggerDebugCrash(context.Context, *DebugCrashRequest) (*emptypb.Empty, error) + TriggerOOMReport(context.Context, *emptypb.Empty) (*emptypb.Empty, error) SubscribeConnections(*SubscribeConnectionsRequest, grpc.ServerStreamingServer[ConnectionEvents]) error CloseConnection(context.Context, *CloseConnectionRequest) (*emptypb.Empty, error) CloseAllConnections(context.Context, *emptypb.Empty) (*emptypb.Empty, error) @@ -453,6 +466,10 @@ func (UnimplementedStartedServiceServer) TriggerDebugCrash(context.Context, *Deb return nil, status.Error(codes.Unimplemented, "method TriggerDebugCrash not implemented") } +func (UnimplementedStartedServiceServer) TriggerOOMReport(context.Context, *emptypb.Empty) (*emptypb.Empty, error) { + return nil, status.Error(codes.Unimplemented, "method TriggerOOMReport not implemented") +} + func (UnimplementedStartedServiceServer) SubscribeConnections(*SubscribeConnectionsRequest, grpc.ServerStreamingServer[ConnectionEvents]) error { return status.Error(codes.Unimplemented, "method SubscribeConnections not implemented") } @@ -764,6 +781,24 @@ func _StartedService_TriggerDebugCrash_Handler(srv interface{}, ctx context.Cont return interceptor(ctx, in, info, handler) } +func _StartedService_TriggerOOMReport_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(emptypb.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StartedServiceServer).TriggerOOMReport(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StartedService_TriggerOOMReport_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StartedServiceServer).TriggerOOMReport(ctx, req.(*emptypb.Empty)) + } + return interceptor(ctx, in, info, handler) +} + func _StartedService_SubscribeConnections_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(SubscribeConnectionsRequest) if err := stream.RecvMsg(m); err != nil { @@ -902,6 +937,10 @@ var StartedService_ServiceDesc = grpc.ServiceDesc{ MethodName: "TriggerDebugCrash", Handler: _StartedService_TriggerDebugCrash_Handler, }, + { + MethodName: "TriggerOOMReport", + Handler: _StartedService_TriggerOOMReport_Handler, + }, { MethodName: "CloseConnection", Handler: _StartedService_CloseConnection_Handler, diff --git a/experimental/libbox/command_client.go b/experimental/libbox/command_client.go index 1f26acaa9..114198a14 100644 --- a/experimental/libbox/command_client.go +++ b/experimental/libbox/command_client.go @@ -558,6 +558,13 @@ func (c *CommandClient) TriggerNativeCrash() error { return err } +func (c *CommandClient) TriggerOOMReport() error { + _, err := callWithResult(c, func(client daemon.StartedServiceClient) (*emptypb.Empty, error) { + return client.TriggerOOMReport(context.Background(), &emptypb.Empty{}) + }) + return err +} + func (c *CommandClient) GetDeprecatedNotes() (DeprecatedNoteIterator, error) { return callWithResult(c, func(client daemon.StartedServiceClient) (DeprecatedNoteIterator, error) { warnings, err := client.GetDeprecatedWarnings(context.Background(), &emptypb.Empty{}) diff --git a/experimental/libbox/command_server.go b/experimental/libbox/command_server.go index 03060ace8..56ff6d976 100644 --- a/experimental/libbox/command_server.go +++ b/experimental/libbox/command_server.go @@ -58,10 +58,12 @@ func NewCommandServer(handler CommandServerHandler, platformInterface PlatformIn server.StartedService = daemon.NewStartedService(daemon.ServiceOptions{ Context: ctx, // Platform: platformWrapper, - Handler: (*platformHandler)(server), - Debug: sDebug, - LogMaxLines: sLogMaxLines, - OOMKiller: memoryLimitEnabled, + Handler: (*platformHandler)(server), + Debug: sDebug, + LogMaxLines: sLogMaxLines, + OOMKillerEnabled: sOOMKillerEnabled, + OOMKillerDisabled: sOOMKillerDisabled, + OOMMemoryLimit: uint64(sOOMMemoryLimit), // WorkingDirectory: sWorkingPath, // TempDirectory: sTempPath, // UserID: sUserID, diff --git a/experimental/libbox/config.go b/experimental/libbox/config.go index eca3fdf94..4b21e5051 100644 --- a/experimental/libbox/config.go +++ b/experimental/libbox/config.go @@ -12,6 +12,7 @@ import ( "github.com/sagernet/sing-box/include" "github.com/sagernet/sing-box/log" "github.com/sagernet/sing-box/option" + "github.com/sagernet/sing-box/service/oomkiller" tun "github.com/sagernet/sing-tun" "github.com/sagernet/sing/common/control" E "github.com/sagernet/sing/common/exceptions" @@ -22,6 +23,8 @@ import ( "github.com/sagernet/sing/service/filemanager" ) +var sOOMReporter oomkiller.OOMReporter + func baseContext(platformInterface PlatformInterface) context.Context { dnsRegistry := include.DNSTransportRegistry() if platformInterface != nil { @@ -33,6 +36,9 @@ func baseContext(platformInterface PlatformInterface) context.Context { } ctx := context.Background() ctx = filemanager.WithDefault(ctx, sWorkingPath, sTempPath, sUserID, sGroupID) + if sOOMReporter != nil { + ctx = service.ContextWith[oomkiller.OOMReporter](ctx, sOOMReporter) + } return box.Context(ctx, include.InboundRegistry(), include.OutboundRegistry(), include.EndpointRegistry(), dnsRegistry, include.ServiceRegistry(), include.CertificateProviderRegistry()) } diff --git a/experimental/libbox/log.go b/experimental/libbox/log.go index 012965e72..c4d3f8355 100644 --- a/experimental/libbox/log.go +++ b/experimental/libbox/log.go @@ -4,43 +4,24 @@ package libbox import ( "archive/zip" - "bytes" - "encoding/json" "io" "io/fs" "os" "path/filepath" "runtime" "runtime/debug" - "strconv" "time" - - C "github.com/sagernet/sing-box/constant" -) - -const ( - crashReportMetadataFileName = "metadata.json" - crashReportGoLogFileName = "go.log" - crashReportConfigFileName = "configuration.json" ) var crashOutputFile *os.File type crashReportMetadata struct { - Source string `json:"source"` - BundleIdentifier string `json:"bundleIdentifier,omitempty"` - ProcessName string `json:"processName,omitempty"` - ProcessPath string `json:"processPath,omitempty"` - StartedAt string `json:"startedAt,omitempty"` - CrashedAt string `json:"crashedAt,omitempty"` - AppVersion string `json:"appVersion,omitempty"` - AppMarketingVersion string `json:"appMarketingVersion,omitempty"` - CoreVersion string `json:"coreVersion,omitempty"` - GoVersion string `json:"goVersion,omitempty"` - SignalName string `json:"signalName,omitempty"` - SignalCode string `json:"signalCode,omitempty"` - ExceptionName string `json:"exceptionName,omitempty"` - ExceptionReason string `json:"exceptionReason,omitempty"` + reportMetadata + CrashedAt string `json:"crashedAt,omitempty"` + SignalName string `json:"signalName,omitempty"` + SignalCode string `json:"signalCode,omitempty"` + ExceptionName string `json:"exceptionName,omitempty"` + ExceptionReason string `json:"exceptionReason,omitempty"` } func archiveCrashReport(path string, crashReportsDir string) { @@ -55,59 +36,29 @@ func archiveCrashReport(path string, crashReportsDir string) { crashTime = info.ModTime().UTC() } - metadata := currentCrashReportMetadata(crashTime) - if len(bytes.TrimSpace(content)) == 0 { - os.Remove(path) - return - } + initReportDir(crashReportsDir) + destPath := nextAvailableReportPath(crashReportsDir, crashTime) + initReportDir(destPath) - os.MkdirAll(crashReportsDir, 0o777) - destName := crashTime.Format("2006-01-02T15-04-05") - destPath := filepath.Join(crashReportsDir, destName) - for i := 1; ; i++ { - if _, err := os.Stat(destPath); os.IsNotExist(err) { - break - } - destPath = filepath.Join(crashReportsDir, - crashTime.Format("2006-01-02T15-04-05")+"-"+strconv.Itoa(i)) + writeReportFile(destPath, "go.log", content) + metadata := crashReportMetadata{ + reportMetadata: baseReportMetadata(), + CrashedAt: crashTime.Format(time.RFC3339), } - - os.MkdirAll(destPath, 0o777) - logPath := filepath.Join(destPath, crashReportGoLogFileName) - os.WriteFile(logPath, content, 0o666) - if runtime.GOOS != "android" { - os.Chown(destPath, sUserID, sGroupID) - os.Chown(logPath, sUserID, sGroupID) - } - writeCrashReportMetadata(destPath, metadata) + writeReportMetadata(destPath, metadata) os.Remove(path) - archiveConfigSnapshot(destPath) + copyConfigSnapshot(destPath) + os.Remove(configSnapshotPath()) } func configSnapshotPath() string { - return filepath.Join(sTempPath, crashReportConfigFileName) + return filepath.Join(sTempPath, "configuration.json") } func saveConfigSnapshot(configContent string) { snapshotPath := configSnapshotPath() os.WriteFile(snapshotPath, []byte(configContent), 0o666) - if runtime.GOOS != "android" { - os.Chown(snapshotPath, sUserID, sGroupID) - } -} - -func archiveConfigSnapshot(destPath string) { - snapshotPath := configSnapshotPath() - content, err := os.ReadFile(snapshotPath) - if err != nil || len(bytes.TrimSpace(content)) == 0 { - return - } - configPath := filepath.Join(destPath, crashReportConfigFileName) - os.WriteFile(configPath, content, 0o666) - if runtime.GOOS != "android" { - os.Chown(configPath, sUserID, sGroupID) - } - os.Remove(snapshotPath) + chownReport(snapshotPath) } func redirectStderr(path string) error { @@ -138,35 +89,6 @@ func redirectStderr(path string) error { return nil } -func currentCrashReportMetadata(crashTime time.Time) crashReportMetadata { - processPath, _ := os.Executable() - processName := filepath.Base(processPath) - if processName == "." { - processName = "" - } - return crashReportMetadata{ - Source: sCrashReportSource, - ProcessName: processName, - ProcessPath: processPath, - CrashedAt: crashTime.Format(time.RFC3339), - CoreVersion: C.Version, - GoVersion: GoVersion(), - } -} - -func writeCrashReportMetadata(reportPath string, metadata crashReportMetadata) { - data, err := json.Marshal(metadata) - if err != nil { - return - } - - metaPath := filepath.Join(reportPath, crashReportMetadataFileName) - os.WriteFile(metaPath, data, 0o666) - if runtime.GOOS != "android" { - os.Chown(metaPath, sUserID, sGroupID) - } -} - func CreateZipArchive(sourcePath string, destinationPath string) error { sourceInfo, err := os.Stat(sourcePath) if err != nil { diff --git a/experimental/libbox/memory.go b/experimental/libbox/memory.go deleted file mode 100644 index b0b87f73f..000000000 --- a/experimental/libbox/memory.go +++ /dev/null @@ -1,26 +0,0 @@ -package libbox - -import ( - "math" - runtimeDebug "runtime/debug" - - C "github.com/sagernet/sing-box/constant" -) - -var memoryLimitEnabled bool - -func SetMemoryLimit(enabled bool) { - memoryLimitEnabled = enabled - const memoryLimitGo = 45 * 1024 * 1024 - if enabled { - runtimeDebug.SetGCPercent(10) - if C.IsIos { - runtimeDebug.SetMemoryLimit(memoryLimitGo) - } - } else { - runtimeDebug.SetGCPercent(100) - if C.IsIos { - runtimeDebug.SetMemoryLimit(math.MaxInt64) - } - } -} diff --git a/experimental/libbox/oom_report.go b/experimental/libbox/oom_report.go new file mode 100644 index 000000000..33d6dbf8c --- /dev/null +++ b/experimental/libbox/oom_report.go @@ -0,0 +1,93 @@ +//go:build darwin || linux + +package libbox + +import ( + "compress/gzip" + "os" + "path/filepath" + "runtime/pprof" + "strings" + "time" + + "github.com/sagernet/sing-box/service/oomkiller" + "github.com/sagernet/sing/common/byteformats" + "github.com/sagernet/sing/common/memory" +) + +func init() { + sOOMReporter = &oomReporter{} +} + +var oomReportProfiles = []string{ + "allocs", + "block", + "goroutine", + "heap", + "mutex", + "threadcreate", +} + +type oomReportMetadata struct { + reportMetadata + RecordedAt string `json:"recordedAt"` + MemoryUsage string `json:"memoryUsage"` + AvailableMemory string `json:"availableMemory,omitempty"` +} + +type oomReporter struct{} + +var _ oomkiller.OOMReporter = (*oomReporter)(nil) + +func (r *oomReporter) WriteReport(memoryUsage uint64) error { + now := time.Now().UTC() + reportsDir := filepath.Join(sWorkingPath, "oom_reports") + err := os.MkdirAll(reportsDir, 0o777) + if err != nil { + return err + } + chownReport(reportsDir) + + destPath := nextAvailableReportPath(reportsDir, now) + err = os.MkdirAll(destPath, 0o777) + if err != nil { + return err + } + chownReport(destPath) + + for _, name := range oomReportProfiles { + writeOOMProfile(destPath, name) + } + + writeReportFile(destPath, "cmdline", []byte(strings.Join(os.Args, "\000"))) + metadata := oomReportMetadata{ + reportMetadata: baseReportMetadata(), + RecordedAt: now.Format(time.RFC3339), + MemoryUsage: byteformats.FormatMemoryBytes(memoryUsage), + } + availableMemory := memory.Available() + if availableMemory > 0 { + metadata.AvailableMemory = byteformats.FormatMemoryBytes(availableMemory) + } + writeReportMetadata(destPath, metadata) + copyConfigSnapshot(destPath) + + return nil +} + +func writeOOMProfile(destPath string, name string) { + profile := pprof.Lookup(name) + if profile == nil { + return + } + filePath := filepath.Join(destPath, name+".pb.gz") + file, err := os.Create(filePath) + if err != nil { + return + } + defer file.Close() + gzipWriter := gzip.NewWriter(file) + defer gzipWriter.Close() + profile.WriteTo(gzipWriter, 0) + chownReport(filePath) +} diff --git a/experimental/libbox/report.go b/experimental/libbox/report.go new file mode 100644 index 000000000..1c4506702 --- /dev/null +++ b/experimental/libbox/report.go @@ -0,0 +1,89 @@ +//go:build darwin || linux + +package libbox + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "runtime" + "strconv" + "time" + + C "github.com/sagernet/sing-box/constant" +) + +type reportMetadata struct { + Source string `json:"source,omitempty"` + BundleIdentifier string `json:"bundleIdentifier,omitempty"` + ProcessName string `json:"processName,omitempty"` + ProcessPath string `json:"processPath,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + AppVersion string `json:"appVersion,omitempty"` + AppMarketingVersion string `json:"appMarketingVersion,omitempty"` + CoreVersion string `json:"coreVersion,omitempty"` + GoVersion string `json:"goVersion,omitempty"` +} + +func baseReportMetadata() reportMetadata { + processPath, _ := os.Executable() + processName := filepath.Base(processPath) + if processName == "." { + processName = "" + } + return reportMetadata{ + Source: sCrashReportSource, + ProcessName: processName, + ProcessPath: processPath, + CoreVersion: C.Version, + GoVersion: GoVersion(), + } +} + +func writeReportFile(destPath string, name string, content []byte) { + filePath := filepath.Join(destPath, name) + os.WriteFile(filePath, content, 0o666) + chownReport(filePath) +} + +func writeReportMetadata(destPath string, metadata any) { + data, err := json.Marshal(metadata) + if err != nil { + return + } + writeReportFile(destPath, "metadata.json", data) +} + +func copyConfigSnapshot(destPath string) { + snapshotPath := configSnapshotPath() + content, err := os.ReadFile(snapshotPath) + if err != nil || len(bytes.TrimSpace(content)) == 0 { + return + } + writeReportFile(destPath, "configuration.json", content) +} + +func initReportDir(path string) { + os.MkdirAll(path, 0o777) + chownReport(path) +} + +func chownReport(path string) { + if runtime.GOOS != "android" { + os.Chown(path, sUserID, sGroupID) + } +} + +func nextAvailableReportPath(reportsDir string, timestamp time.Time) string { + destName := timestamp.Format("2006-01-02T15-04-05") + destPath := filepath.Join(reportsDir, destName) + for i := 1; i <= 1000; i++ { + _, err := os.Stat(destPath) + if os.IsNotExist(err) { + break + } + destPath = filepath.Join(reportsDir, destName+"-"+strconv.Itoa(i)) + } + return destPath +} diff --git a/experimental/libbox/setup.go b/experimental/libbox/setup.go index 460388215..e5a5f4f7a 100644 --- a/experimental/libbox/setup.go +++ b/experimental/libbox/setup.go @@ -1,6 +1,7 @@ package libbox import ( + "math" "os" "path/filepath" "runtime" @@ -25,6 +26,9 @@ var ( sLogMaxLines int sDebug bool sCrashReportSource string + sOOMKillerEnabled bool + sOOMKillerDisabled bool + sOOMMemoryLimit int64 ) func init() { @@ -42,6 +46,9 @@ type SetupOptions struct { LogMaxLines int Debug bool CrashReportSource string + OOMKillerEnabled bool + OOMKillerDisabled bool + OOMMemoryLimit int64 } func Setup(options *SetupOptions) error { @@ -61,6 +68,14 @@ func Setup(options *SetupOptions) error { sLogMaxLines = options.LogMaxLines sDebug = options.Debug sCrashReportSource = options.CrashReportSource + sOOMKillerEnabled = options.OOMKillerEnabled + sOOMKillerDisabled = options.OOMKillerDisabled + sOOMMemoryLimit = options.OOMMemoryLimit + if sOOMKillerEnabled && sOOMMemoryLimit > 0 { + debug.SetMemoryLimit(sOOMMemoryLimit) + } else { + debug.SetMemoryLimit(math.MaxInt64) + } os.MkdirAll(sWorkingPath, 0o777) os.MkdirAll(sTempPath, 0o777) diff --git a/option/oom_killer.go b/option/oom_killer.go index 2032ed09a..8f46f9573 100644 --- a/option/oom_killer.go +++ b/option/oom_killer.go @@ -6,9 +6,11 @@ import ( ) type OOMKillerServiceOptions struct { - MemoryLimit *byteformats.MemoryBytes `json:"memory_limit,omitempty"` - SafetyMargin *byteformats.MemoryBytes `json:"safety_margin,omitempty"` - MinInterval badoption.Duration `json:"min_interval,omitempty"` - MaxInterval badoption.Duration `json:"max_interval,omitempty"` - ChecksBeforeLimit int `json:"checks_before_limit,omitempty"` + MemoryLimit *byteformats.MemoryBytes `json:"memory_limit,omitempty"` + SafetyMargin *byteformats.MemoryBytes `json:"safety_margin,omitempty"` + MinInterval badoption.Duration `json:"min_interval,omitempty"` + MaxInterval badoption.Duration `json:"max_interval,omitempty"` + ChecksBeforeLimit int `json:"checks_before_limit,omitempty"` + KillerDisabled bool `json:"-"` + MemoryLimitOverride uint64 `json:"-"` } diff --git a/service/oomkiller/config.go b/service/oomkiller/config.go index 693ced995..a25a36ff9 100644 --- a/service/oomkiller/config.go +++ b/service/oomkiller/config.go @@ -7,7 +7,7 @@ import ( E "github.com/sagernet/sing/common/exceptions" ) -func buildTimerConfig(options option.OOMKillerServiceOptions, memoryLimit uint64, useAvailable bool) (timerConfig, error) { +func buildTimerConfig(options option.OOMKillerServiceOptions, memoryLimit uint64, useAvailable bool, killerDisabled bool) (timerConfig, error) { safetyMargin := uint64(defaultSafetyMargin) if options.SafetyMargin != nil && options.SafetyMargin.Value() > 0 { safetyMargin = options.SafetyMargin.Value() @@ -47,5 +47,6 @@ func buildTimerConfig(options option.OOMKillerServiceOptions, memoryLimit uint64 maxInterval: maxInterval, checksBeforeLimit: checksBeforeLimit, useAvailable: useAvailable, + killerDisabled: killerDisabled, }, nil } diff --git a/service/oomkiller/report.go b/service/oomkiller/report.go new file mode 100644 index 000000000..c17bf1716 --- /dev/null +++ b/service/oomkiller/report.go @@ -0,0 +1,30 @@ +package oomkiller + +import ( + "time" + + "github.com/sagernet/sing/service" +) + +func (s *Service) writeOOMReport(memoryUsage uint64) { + now := time.Now().Unix() + for { + lastReport := s.lastReportTime.Load() + if now-lastReport < 3600 { + return + } + if s.lastReportTime.CompareAndSwap(lastReport, now) { + break + } + } + reporter := service.FromContext[OOMReporter](s.ctx) + if reporter == nil { + return + } + err := reporter.WriteReport(memoryUsage) + if err != nil { + s.logger.Warn("failed to write OOM report: ", err) + } else { + s.logger.Info("OOM report saved") + } +} diff --git a/service/oomkiller/reporter.go b/service/oomkiller/reporter.go new file mode 100644 index 000000000..47ef69d43 --- /dev/null +++ b/service/oomkiller/reporter.go @@ -0,0 +1,5 @@ +package oomkiller + +type OOMReporter interface { + WriteReport(memoryUsage uint64) error +} diff --git a/service/oomkiller/service.go b/service/oomkiller/service.go index c3612d926..d9c8b3079 100644 --- a/service/oomkiller/service.go +++ b/service/oomkiller/service.go @@ -36,12 +36,14 @@ import ( "context" runtimeDebug "runtime/debug" "sync" + "sync/atomic" "github.com/sagernet/sing-box/adapter" boxService "github.com/sagernet/sing-box/adapter/service" boxConstant "github.com/sagernet/sing-box/constant" "github.com/sagernet/sing-box/log" "github.com/sagernet/sing-box/option" + "github.com/sagernet/sing/common/byteformats" "github.com/sagernet/sing/common/memory" "github.com/sagernet/sing/service" ) @@ -57,30 +59,38 @@ var ( type Service struct { boxService.Adapter - logger log.ContextLogger - router adapter.Router - memoryLimit uint64 - hasTimerMode bool - useAvailable bool - timerConfig timerConfig - adaptiveTimer *adaptiveTimer + ctx context.Context + logger log.ContextLogger + router adapter.Router + memoryLimit uint64 + hasTimerMode bool + useAvailable bool + killerDisabled bool + timerConfig timerConfig + adaptiveTimer *adaptiveTimer + lastReportTime atomic.Int64 } func NewService(ctx context.Context, logger log.ContextLogger, tag string, options option.OOMKillerServiceOptions) (adapter.Service, error) { s := &Service{ - Adapter: boxService.NewAdapter(boxConstant.TypeOOMKiller, tag), - logger: logger, - router: service.FromContext[adapter.Router](ctx), + Adapter: boxService.NewAdapter(boxConstant.TypeOOMKiller, tag), + ctx: ctx, + logger: logger, + router: service.FromContext[adapter.Router](ctx), + killerDisabled: options.KillerDisabled, } - if options.MemoryLimit != nil { + if options.MemoryLimitOverride > 0 { + s.memoryLimit = options.MemoryLimitOverride + s.hasTimerMode = true + } else if options.MemoryLimit != nil { s.memoryLimit = options.MemoryLimit.Value() if s.memoryLimit > 0 { s.hasTimerMode = true } } - config, err := buildTimerConfig(options, s.memoryLimit, s.useAvailable) + config, err := buildTimerConfig(options, s.memoryLimit, s.useAvailable, s.killerDisabled) if err != nil { return nil, err } @@ -95,9 +105,12 @@ func (s *Service) Start(stage adapter.StartStage) error { } if s.hasTimerMode { - s.adaptiveTimer = newAdaptiveTimer(s.logger, s.router, s.timerConfig) + s.adaptiveTimer = newAdaptiveTimer(s.logger, s.router, s.timerConfig, + func(usage uint64) { s.writeOOMReport(usage) }, + nil, + ) if s.memoryLimit > 0 { - s.logger.Info("started memory monitor with limit: ", s.memoryLimit/(1024*1024), " MiB") + s.logger.Info("started memory monitor with limit: ", byteformats.FormatMemoryBytes(s.memoryLimit)) } else { s.logger.Info("started memory monitor with available memory detection") } @@ -162,27 +175,32 @@ func goMemoryPressureCallback(status C.ulong) { usage := memory.Total() if s.hasTimerMode { if isCritical { - s.logger.Warn("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB") + s.logger.Warn("memory pressure: ", level, ", usage: ", byteformats.FormatMemoryBytes(usage)) if s.adaptiveTimer != nil { s.adaptiveTimer.startNow() } } else if isWarning { - s.logger.Warn("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB") + s.logger.Warn("memory pressure: ", level, ", usage: ", byteformats.FormatMemoryBytes(usage)) } else { - s.logger.Debug("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB") + s.logger.Debug("memory pressure: ", level, ", usage: ", byteformats.FormatMemoryBytes(usage)) if s.adaptiveTimer != nil { s.adaptiveTimer.stop() } } } else { if isCritical { - s.logger.Error("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB, resetting network") - s.router.ResetNetwork() + s.writeOOMReport(usage) + if s.killerDisabled { + s.logger.Warn("memory pressure: ", level, " (report only), usage: ", byteformats.FormatMemoryBytes(usage)) + } else { + s.logger.Error("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB, resetting network") + s.router.ResetNetwork() + } freeOSMemory = true } else if isWarning { - s.logger.Warn("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB") + s.logger.Warn("memory pressure: ", level, ", usage: ", byteformats.FormatMemoryBytes(usage)) } else { - s.logger.Debug("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB") + s.logger.Debug("memory pressure: ", level, ", usage: ", byteformats.FormatMemoryBytes(usage)) } } } diff --git a/service/oomkiller/service_stub.go b/service/oomkiller/service_stub.go index 13348bac1..9b2ca79fa 100644 --- a/service/oomkiller/service_stub.go +++ b/service/oomkiller/service_stub.go @@ -4,12 +4,14 @@ package oomkiller import ( "context" + "sync/atomic" "github.com/sagernet/sing-box/adapter" boxService "github.com/sagernet/sing-box/adapter/service" boxConstant "github.com/sagernet/sing-box/constant" "github.com/sagernet/sing-box/log" "github.com/sagernet/sing-box/option" + "github.com/sagernet/sing/common/byteformats" E "github.com/sagernet/sing/common/exceptions" "github.com/sagernet/sing/common/memory" "github.com/sagernet/sing/service" @@ -21,33 +23,42 @@ func RegisterService(registry *boxService.Registry) { type Service struct { boxService.Adapter - logger log.ContextLogger - router adapter.Router - adaptiveTimer *adaptiveTimer - timerConfig timerConfig - hasTimerMode bool - useAvailable bool - memoryLimit uint64 + ctx context.Context + logger log.ContextLogger + router adapter.Router + memoryLimit uint64 + hasTimerMode bool + useAvailable bool + killerDisabled bool + timerConfig timerConfig + adaptiveTimer *adaptiveTimer + lastReportTime atomic.Int64 } func NewService(ctx context.Context, logger log.ContextLogger, tag string, options option.OOMKillerServiceOptions) (adapter.Service, error) { s := &Service{ - Adapter: boxService.NewAdapter(boxConstant.TypeOOMKiller, tag), - logger: logger, - router: service.FromContext[adapter.Router](ctx), + Adapter: boxService.NewAdapter(boxConstant.TypeOOMKiller, tag), + ctx: ctx, + logger: logger, + router: service.FromContext[adapter.Router](ctx), + killerDisabled: options.KillerDisabled, } - if options.MemoryLimit != nil { - s.memoryLimit = options.MemoryLimit.Value() - } - if s.memoryLimit > 0 { + if options.MemoryLimitOverride > 0 { + s.memoryLimit = options.MemoryLimitOverride s.hasTimerMode = true - } else if memory.AvailableSupported() { + } else if options.MemoryLimit != nil { + s.memoryLimit = options.MemoryLimit.Value() + if s.memoryLimit > 0 { + s.hasTimerMode = true + } + } + if !s.hasTimerMode && memory.AvailableSupported() { s.useAvailable = true s.hasTimerMode = true } - config, err := buildTimerConfig(options, s.memoryLimit, s.useAvailable) + config, err := buildTimerConfig(options, s.memoryLimit, s.useAvailable, s.killerDisabled) if err != nil { return nil, err } @@ -63,12 +74,15 @@ func (s *Service) Start(stage adapter.StartStage) error { if !s.hasTimerMode { return E.New("memory pressure monitoring is not available on this platform without memory_limit") } - s.adaptiveTimer = newAdaptiveTimer(s.logger, s.router, s.timerConfig) + s.adaptiveTimer = newAdaptiveTimer(s.logger, s.router, s.timerConfig, + func(usage uint64) { s.writeOOMReport(usage) }, + nil, + ) s.adaptiveTimer.start(0) if s.useAvailable { s.logger.Info("started memory monitor with available memory detection") } else { - s.logger.Info("started memory monitor with limit: ", s.memoryLimit/(1024*1024), " MiB") + s.logger.Info("started memory monitor with limit: ", byteformats.FormatMemoryBytes(s.memoryLimit)) } return nil } diff --git a/service/oomkiller/service_timer.go b/service/oomkiller/service_timer.go index 315e17156..593dd988e 100644 --- a/service/oomkiller/service_timer.go +++ b/service/oomkiller/service_timer.go @@ -7,6 +7,7 @@ import ( "github.com/sagernet/sing-box/adapter" "github.com/sagernet/sing-box/log" + "github.com/sagernet/sing/common/byteformats" "github.com/sagernet/sing/common/memory" ) @@ -26,11 +27,15 @@ type adaptiveTimer struct { maxInterval time.Duration checksBeforeLimit int useAvailable bool + killerDisabled bool + onTriggered func(uint64) + onRecovered func() - access sync.Mutex - timer *time.Timer - previousUsage uint64 - lastInterval time.Duration + access sync.Mutex + timer *time.Timer + previousUsage uint64 + lastInterval time.Duration + previouslyTriggered bool } type timerConfig struct { @@ -40,9 +45,10 @@ type timerConfig struct { maxInterval time.Duration checksBeforeLimit int useAvailable bool + killerDisabled bool } -func newAdaptiveTimer(logger log.ContextLogger, router adapter.Router, config timerConfig) *adaptiveTimer { +func newAdaptiveTimer(logger log.ContextLogger, router adapter.Router, config timerConfig, onTriggered func(uint64), onRecovered func()) *adaptiveTimer { return &adaptiveTimer{ logger: logger, router: router, @@ -52,6 +58,9 @@ func newAdaptiveTimer(logger log.ContextLogger, router adapter.Router, config ti maxInterval: config.maxInterval, checksBeforeLimit: config.checksBeforeLimit, useAvailable: config.useAvailable, + killerDisabled: config.killerDisabled, + onTriggered: onTriggered, + onRecovered: onRecovered, } } @@ -130,9 +139,24 @@ func (t *adaptiveTimer) poll() { } if triggered { - t.logger.Error("memory threshold reached, usage: ", usage/(1024*1024), " MiB, resetting network") - t.router.ResetNetwork() + if !t.previouslyTriggered { + t.previouslyTriggered = true + if t.onTriggered != nil { + t.onTriggered(usage) + } + } + if t.killerDisabled { + t.logger.Warn("memory threshold reached (report only), usage: ", byteformats.FormatMemoryBytes(usage)) + } else { + t.logger.Error("memory threshold reached, usage: ", usage/(1024*1024), " MiB, resetting network") + t.router.ResetNetwork() + } runtimeDebug.FreeOSMemory() + } else if t.previouslyTriggered { + t.previouslyTriggered = false + if t.onRecovered != nil { + t.onRecovered() + } } var interval time.Duration