diff --git a/Source/Engine/Graphics/GPUDevice.cpp b/Source/Engine/Graphics/GPUDevice.cpp index 9a1935228..684e73f3a 100644 --- a/Source/Engine/Graphics/GPUDevice.cpp +++ b/Source/Engine/Graphics/GPUDevice.cpp @@ -361,8 +361,7 @@ void GPUDevice::OnRequestingExit() Engine::FatalError != FatalErrorType::GPUHang && Engine::FatalError != FatalErrorType::GPUOutOfMemory) return; - // TODO: get and log actual GPU memory used by the engine (API-specific) - DumpResourcesToLog(); + OnCrash(); } GPUDevice::GPUDevice(RendererType type, ShaderProfile profile) @@ -751,6 +750,11 @@ void GPUDevice::RenderEnd() #endif } +void GPUDevice::OnCrash() +{ + DumpResourcesToLog(); +} + GPUTasksContext* GPUDevice::CreateTasksContext() { return New(this); @@ -822,6 +826,16 @@ uint64 GPUDevice::GetMemoryUsage() const return result; } +GPUMemoryStats GPUDevice::GetMemoryStats() +{ + GPUMemoryStats stats; + stats.UsedDedicatedMemory = GetMemoryUsage(); + stats.TotalDedicatedMemory = TotalGraphicsMemory; + stats.UsedSystemMemory = 0; + stats.TotalSystemMemory = 0; + return stats; +} + Array GPUDevice::GetResources() const { _resourcesLock.Lock(); diff --git a/Source/Engine/Graphics/GPUDevice.h b/Source/Engine/Graphics/GPUDevice.h index e6df74886..fbef26a6d 100644 --- a/Source/Engine/Graphics/GPUDevice.h +++ b/Source/Engine/Graphics/GPUDevice.h @@ -33,6 +33,34 @@ class Model; class Material; class MaterialBase; +/// +/// Contains information about current GPU memory usage and budget. +/// +API_STRUCT(NoDefault) struct GPUMemoryStats +{ + DECLARE_SCRIPTING_TYPE_MINIMAL(GPUMemoryStats); + + /// + /// Amount of used dedicated video memory in bytes. Memory local to the device, and represents the fastest available memory to the GPU. + /// + API_FIELD() uint64 UsedDedicatedMemory = 0; + + /// + /// Total amount of dedicated memory budget in bytes. Memory local to the device, and represents the fastest available memory to the GPU. + /// + API_FIELD() uint64 TotalDedicatedMemory = 0; + + /// + /// Amount of used system video memory in bytes. Memory non-local to the device, and may have slower performance than the dedicated/local. + /// + API_FIELD() uint64 UsedSystemMemory = 0; + + /// + /// Total amount of system memory budget in bytes. Memory non-local to the device, and may have slower performance than the dedicated/local. + /// + API_FIELD() uint64 TotalSystemMemory = 0; +}; + /// /// Graphics device object for rendering on GPU. /// @@ -272,10 +300,15 @@ public: API_PROPERTY() virtual void* GetNativePtr() const = 0; /// - /// Gets the amount of memory usage by all the GPU resources (in bytes). + /// Gets the amount of memory usage by all the GPU resources (in bytes). Returned value is estimated based on resources created by the engine and might not be accurate. Use GPUMemoryStats for more detailed memory budget usage. /// API_PROPERTY() uint64 GetMemoryUsage() const; + /// + /// Gets the current GPU memory stats. + /// + API_PROPERTY() virtual GPUMemoryStats GetMemoryStats(); + /// /// Gets the list with all active GPU resources. /// @@ -417,6 +450,11 @@ protected: /// virtual void RenderEnd(); + /// + /// Called when program crashed due to GPU error (out of memory, hang, error - see Engine::FatalError). By default, it logs all GPU resources to the log. Can be used to update platform-specific stats or extract crash-info. + /// + virtual void OnCrash(); + public: /// /// Creates the texture. diff --git a/Source/Engine/GraphicsDevice/DirectX/DX12/GPUDeviceDX12.cpp b/Source/Engine/GraphicsDevice/DirectX/DX12/GPUDeviceDX12.cpp index 5be4b7706..3d525b276 100644 --- a/Source/Engine/GraphicsDevice/DirectX/DX12/GPUDeviceDX12.cpp +++ b/Source/Engine/GraphicsDevice/DirectX/DX12/GPUDeviceDX12.cpp @@ -578,6 +578,28 @@ GPUDeviceDX12::GPUDeviceDX12(IDXGIFactory4* dxgiFactory, GPUAdapterDX* adapter) { } +GPUMemoryStats GPUDeviceDX12::GetMemoryStats() +{ + GPUMemoryStats stats; + D3D12MA::Budget localBudget, nonLocalBudget; + Allocator->GetBudget(&localBudget, &nonLocalBudget); + if (Allocator->IsUMA()) + { + // UMA (Unified Memory Architecture) means no dedicated video memory and the system memory is shared between CPU and GPU + stats.UsedSystemMemory = localBudget.UsageBytes; + stats.TotalSystemMemory = localBudget.BudgetBytes; + } + else + { + // Discrete GPU + stats.UsedDedicatedMemory = localBudget.UsageBytes; + stats.TotalDedicatedMemory = localBudget.BudgetBytes; + stats.UsedSystemMemory = nonLocalBudget.UsageBytes; + stats.TotalSystemMemory = nonLocalBudget.BudgetBytes; + } + return stats; +} + bool GPUDeviceDX12::Init() { #if PLATFORM_XBOX_SCARLETT || PLATFORM_XBOX_ONE @@ -972,6 +994,19 @@ void GPUDeviceDX12::RenderEnd() heap->EndQueryBatchAndResolveQueryData(_mainContext); } +void GPUDeviceDX12::OnCrash() +{ + // Dump allocator stats + //D3D12MA::TotalStatistics statistics; + //Allocator->CalculateStatistics(&statistics); + D3D12MA::Budget localBudget, nonLocalBudget; + Allocator->GetBudget(&localBudget, &nonLocalBudget); + LOG(Info, "[D3D12 Memory] Local budget: {} / {} bytes (blocks: {}, allocs: {})", localBudget.UsageBytes, localBudget.BudgetBytes, localBudget.Stats.BlockCount, localBudget.Stats.AllocationCount); + LOG(Info, "[D3D12 Memory] Non-local budget: {} / {} bytes (blocks: {}, allocs: {})", nonLocalBudget.UsageBytes, nonLocalBudget.BudgetBytes, nonLocalBudget.Stats.BlockCount, nonLocalBudget.Stats.AllocationCount); + + GPUDeviceDX::OnCrash(); +} + GPUDeviceDX12::~GPUDeviceDX12() { // Ensure to be disposed diff --git a/Source/Engine/GraphicsDevice/DirectX/DX12/GPUDeviceDX12.h b/Source/Engine/GraphicsDevice/DirectX/DX12/GPUDeviceDX12.h index ab5b6c1be..cc69fd0b8 100644 --- a/Source/Engine/GraphicsDevice/DirectX/DX12/GPUDeviceDX12.h +++ b/Source/Engine/GraphicsDevice/DirectX/DX12/GPUDeviceDX12.h @@ -33,6 +33,7 @@ namespace D3D12MA }; #define _D3D12MA_JSON_WRITER 0 #define _D3D12MA_STRING_BUILDER 0 +#define D3D12MA_NO_HELPERS 1 #if !BUILD_DEBUG #define D3D12MA_ASSERT(cond) #endif @@ -191,7 +192,6 @@ private: void updateRes2Dispose(); public: - // [GPUDeviceDX] GPUContext* GetMainContext() override { @@ -201,9 +201,11 @@ public: { return _device; } + GPUMemoryStats GetMemoryStats() override; bool Init() override; void DrawBegin() override; void RenderEnd() override; + void OnCrash() override; void Dispose() final override; void WaitForGPU() override; bool GetQueryResult(uint64 queryID, uint64& result, bool wait = false) override; diff --git a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp index de04158f0..0e6244271 100644 --- a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp +++ b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp @@ -1537,6 +1537,31 @@ void* GPUDeviceVulkan::GetNativePtr() const return _nativePtr; } +GPUMemoryStats GPUDeviceVulkan::GetMemoryStats() +{ + GPUMemoryStats stats; + VmaBudget budgets[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(Allocator, budgets); + const VkPhysicalDeviceMemoryProperties* memoryProperties; + vmaGetMemoryProperties(Allocator, &memoryProperties); + for (uint32 i = 0; i < memoryProperties->memoryHeapCount; i++) + { + VkMemoryHeap heap = memoryProperties->memoryHeaps[i]; + VmaBudget& budget = budgets[i]; + if (heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT && Adapter->GpuProps.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) + { + stats.UsedDedicatedMemory += budget.usage; + stats.TotalDedicatedMemory += budget.budget; + } + else + { + stats.UsedSystemMemory += budget.usage; + stats.TotalSystemMemory += budget.budget; + } + } + return stats; +} + static int32 GetMaxSampleCount(VkSampleCountFlags counts) { if (counts & VK_SAMPLE_COUNT_64_BIT) diff --git a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h index 953ee79ee..eec67a18a 100644 --- a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h +++ b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h @@ -569,6 +569,7 @@ public: GPUContext* GetMainContext() override; GPUAdapter* GetAdapter() const override; void* GetNativePtr() const override; + GPUMemoryStats GetMemoryStats() override; bool Init() override; void DrawBegin() override; void Dispose() override; diff --git a/Source/Engine/GraphicsDevice/Vulkan/IncludeVulkanHeaders.h b/Source/Engine/GraphicsDevice/Vulkan/IncludeVulkanHeaders.h index 619757e10..ed595f8d8 100644 --- a/Source/Engine/GraphicsDevice/Vulkan/IncludeVulkanHeaders.h +++ b/Source/Engine/GraphicsDevice/Vulkan/IncludeVulkanHeaders.h @@ -41,6 +41,7 @@ #define VMA_SYSTEM_ALIGNED_FREE(ptr) Platform::Free(ptr) #define VMA_NULLABLE #define VMA_NOT_NULL +#define VMA_STATS_STRING_ENABLED 0 #include #if PLATFORM_APPLE_FAMILY diff --git a/Source/Engine/Profiler/ProfilerMemory.cpp b/Source/Engine/Profiler/ProfilerMemory.cpp index d56d524fd..00f71e380 100644 --- a/Source/Engine/Profiler/ProfilerMemory.cpp +++ b/Source/Engine/Profiler/ProfilerMemory.cpp @@ -15,6 +15,7 @@ #include "Engine/Scripting/ManagedCLR/MCore.h" #include "Engine/Threading/ThreadLocal.h" #include "Engine/Utilities/StringConverter.h" +#include "Engine/Graphics/GPUDevice.h" #include #define GROUPS_COUNT (int32)ProfilerMemory::Groups::MAX @@ -338,11 +339,17 @@ void TickProfilerMemory() memory.UsedPhysicalMemory -= GroupMemory[(int32)ProfilerMemory::Groups::Profiler]; GroupMemory[(int32)ProfilerMemory::Groups::Total] = memory.UsedPhysicalMemory; GroupMemory[(int32)ProfilerMemory::Groups::TotalUntracked] = Math::Max(memory.UsedPhysicalMemory - GroupMemory[(int32)ProfilerMemory::Groups::TotalTracked], 0); + if (GPUDevice::Instance) + { + auto memoryGPU = GPUDevice::Instance->GetMemoryStats(); + GroupMemory[(int32)ProfilerMemory::Groups::TotalGPU] = memoryGPU.UsedDedicatedMemory + memoryGPU.UsedSystemMemory; + } // Update peeks UPDATE_PEEK(ProfilerMemory::Groups::Profiler); UPDATE_PEEK(ProfilerMemory::Groups::Total); UPDATE_PEEK(ProfilerMemory::Groups::TotalUntracked); + UPDATE_PEEK(ProfilerMemory::Groups::TotalGPU); GroupMemoryPeek[(int32)ProfilerMemory::Groups::Total] = Math::Max(GroupMemoryPeek[(int32)ProfilerMemory::Groups::Total], GroupMemoryPeek[(int32)ProfilerMemory::Groups::TotalTracked]); } diff --git a/Source/Engine/Profiler/ProfilerMemory.h b/Source/Engine/Profiler/ProfilerMemory.h index f3612c308..41117e7ae 100644 --- a/Source/Engine/Profiler/ProfilerMemory.h +++ b/Source/Engine/Profiler/ProfilerMemory.h @@ -28,6 +28,8 @@ public: TotalTracked, // Total amount of untracked memory (gap between total system memory usage and tracked memory size). TotalUntracked, + // Total amount of used GPU video memory (sum of dedicated and system memory). + TotalGPU, // Initial memory used by program upon startup (eg. executable size, static variables). ProgramSize, // Profiling tool memory overhead.