Last active
June 2, 2025 09:05
-
-
Save Matt54/8b0375cf84ba3125c673c5073c428dd8 to your computer and use it in GitHub Desktop.
RealityKit HeightMap Image to Terrain with LowLevelMesh and LowLevelTexture
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#ifndef HeightMapParams_h | |
#define HeightMapParams_h | |
struct HeightMapParams { | |
simd_float2 size; | |
simd_uint2 dimensions; | |
}; | |
#endif /* HeightMapParams_h */ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import SwiftUI | |
import RealityKit | |
import Metal | |
struct HeightMapToColoredMeshView: View { | |
@State var heightMap: ExampleHeightMapImage | |
@State var isTransitioning: Bool = false | |
@State var colorMode: HeightMapColorMode = .terrain | |
@State var meshData: HeightMapMeshData? | |
@State var mesh: LowLevelMesh? | |
// holds the rgb values of the original image | |
@State var originalImageTexture: LowLevelTexture? | |
// used to define the positions/normals of the mesh | |
@State var heightMapTexture: LowLevelTexture? | |
// the colors you see on the mesh | |
@State var materialTexture: LowLevelTexture? | |
let device: MTLDevice | |
let commandQueue: MTLCommandQueue | |
let meshComputePipeline: MTLComputePipelineState | |
let imageToGreyscalePipeline: MTLComputePipelineState | |
let deriveNormalsPipeline: MTLComputePipelineState | |
let convertHeightMapToColorPipeline: MTLComputePipelineState | |
@State var timer: Timer? | |
@State var heightProgress: Float = 0.0 | |
let heightProgressRate: Float = 0.01 | |
let timerUpdateDuration: TimeInterval = 1/120.0 | |
init(heightMap: ExampleHeightMapImage = ExampleHeightMapImage.mountEverest) { | |
self.device = MTLCreateSystemDefaultDevice()! | |
self.commandQueue = device.makeCommandQueue()! | |
let library = device.makeDefaultLibrary()! | |
let updateFunction = library.makeFunction(name: "updateHeightMapMesh")! | |
let heightFunction = library.makeFunction(name: "convertGreyscaleToAlpha")! | |
let normalsFunction = library.makeFunction(name: "deriveNormalsFromHeightMap")! | |
let colorFunction = library.makeFunction(name: "convertHeightMapToColors")! | |
self.meshComputePipeline = try! device.makeComputePipelineState(function: updateFunction) | |
self.imageToGreyscalePipeline = try! device.makeComputePipelineState(function: heightFunction) | |
self.deriveNormalsPipeline = try! device.makeComputePipelineState(function: normalsFunction) | |
self.convertHeightMapToColorPipeline = try! device.makeComputePipelineState(function: colorFunction) | |
self.heightMap = heightMap | |
} | |
var body: some View { | |
RealityView { content in | |
let textureResource = try! await TextureResource.loadOnlineImage(heightMap.url) | |
let originalImageTexture = try! copyTextureResourceToLowLevelTexture(from: textureResource) | |
self.originalImageTexture = originalImageTexture | |
let width = UInt32(textureResource.width) | |
let height = UInt32(textureResource.height) | |
let dimensions: SIMD2<UInt32> = SIMD2(x: width, y: height) | |
let meshData = HeightMapMeshData(dimensions: SIMD2(x: width, y: height)) | |
self.meshData = meshData | |
let textureDescriptor = LowLevelTexture.Descriptor(pixelFormat: .rgba32Float, | |
width: Int(dimensions.x), | |
height: Int(dimensions.y), | |
textureUsage: [.shaderRead, .shaderWrite]) | |
self.heightMapTexture = try! LowLevelTexture(descriptor: textureDescriptor) | |
let meshMaterialTexture = try! LowLevelTexture(descriptor: textureDescriptor) | |
self.materialTexture = meshMaterialTexture | |
let mesh = try! VertexData.initializeMesh(vertexCapacity: meshData.vertexCount, | |
indexCapacity: meshData.indexCount) | |
let resource = try! await MeshResource(from: mesh) | |
var material = PhysicallyBasedMaterial() | |
let textureResourceForMaterial = try! await TextureResource(from: meshMaterialTexture) | |
material.baseColor = .init(texture: .init(textureResourceForMaterial)) | |
material.metallic = .init(floatLiteral: 0.5) // Make it more metallic | |
material.roughness = .init(floatLiteral: 0.25) // Make it smoother | |
let modelComponent = ModelComponent(mesh: resource, materials: [material]) | |
let entity = Entity() | |
entity.components.set(modelComponent) | |
content.add(entity) | |
entity.transform.rotation = .init(angle: -.pi / 2, axis: .init(x: 1, y: 0, z: 0)) | |
entity.position = .init(x: 0, y: -0.2, z: 0.0) | |
self.mesh = mesh | |
updateMeshAndTexture() | |
startTimer() | |
} | |
.ornament(attachmentAnchor: .scene(.topBack), contentAlignment: .top) { | |
VStack(spacing: 16) { | |
Text("Height Map Visualization") | |
.font(.headline) | |
Picker("Mountain", selection: $heightMap) { | |
ForEach(ExampleHeightMapImage.allCases, id: \.self) { mountain in | |
Text(mountain.displayName).tag(mountain) | |
} | |
} | |
.pickerStyle(.segmented) | |
.onChange(of: heightMap) { _, newMountain in | |
changeMountain(to: newMountain) | |
} | |
Picker("Color Mode", selection: $colorMode) { | |
ForEach(HeightMapColorMode.allCases, id: \.self) { mode in | |
Text(mode.name).tag(mode) | |
} | |
} | |
.pickerStyle(.segmented) | |
.onChange(of: colorMode) { _, newColorMode in | |
updateMaterialTextureColors() | |
} | |
} | |
.padding() | |
.background(Color.brown, in: RoundedRectangle(cornerRadius: 12)) | |
.padding(.top, 300) | |
} | |
} | |
func changeMountain(to newMountain: ExampleHeightMapImage) { | |
guard !isTransitioning else { return } | |
isTransitioning = true | |
// Stop current timer and start transition | |
stopTimer() | |
startTimer() | |
} | |
func loadNewMountainTexture() async throws { | |
let textureResource = try await TextureResource.loadOnlineImage(heightMap.url) | |
let newOriginalImageTexture = try copyTextureResourceToLowLevelTexture(from: textureResource) | |
// Update the texture | |
self.originalImageTexture = newOriginalImageTexture | |
// Mark transition as complete so height can start rising | |
isTransitioning = false | |
startTimer() | |
} | |
func startTimer() { | |
timer = Timer.scheduledTimer(withTimeInterval: timerUpdateDuration, repeats: true) { timer in | |
if isTransitioning && heightProgress > 0 { | |
// lower the height to 0 | |
heightProgress -= heightProgressRate * 2 | |
if heightProgress <= 0 { | |
heightProgress = 0 | |
// Load new texture when height reaches 0 | |
stopTimer() | |
Task { | |
try! await loadNewMountainTexture() | |
} | |
} | |
} else { | |
// increase height to 1 | |
heightProgress += heightProgressRate | |
if heightProgress >= 1 { | |
heightProgress = 1 | |
isTransitioning = false // Clear transition flag if it was set | |
stopTimer() | |
} | |
} | |
updateMeshAndTexture() | |
} | |
} | |
func updateMeshAndTexture() { | |
updateHeightMapFromImageTexture() | |
updateTextureNormals() | |
updateMaterialTextureColors() | |
updateMeshGeometry() | |
} | |
func stopTimer() { | |
timer?.invalidate() | |
timer = nil | |
} | |
func updateHeightMapFromImageTexture() { | |
guard let imageTexture = originalImageTexture, | |
let heightMapTexture, | |
let commandBuffer = commandQueue.makeCommandBuffer(), | |
let computeEncoder = commandBuffer.makeComputeCommandEncoder() else { return } | |
computeEncoder.setComputePipelineState(imageToGreyscalePipeline) | |
// input: rbg texture | |
computeEncoder.setTexture(imageTexture.read(), index: 0) | |
// output: alpha texture | |
computeEncoder.setTexture(heightMapTexture.replace(using: commandBuffer), index: 1) | |
var progress = heightProgress | |
computeEncoder.setBytes(&progress, length: MemoryLayout<Float>.size, index: 0) | |
let threadWidth = imageToGreyscalePipeline.threadExecutionWidth | |
let threadHeight = imageToGreyscalePipeline.maxTotalThreadsPerThreadgroup / threadWidth | |
let threadsPerThreadgroup = MTLSize(width: threadWidth, height: threadHeight, depth: 1) | |
let threadGroupSize = MTLSizeMake(8, 8, 1) | |
let threadGroups = MTLSizeMake( | |
(heightMapTexture.descriptor.width + threadGroupSize.width - 1) / threadGroupSize.width, | |
(heightMapTexture.descriptor.height + threadGroupSize.height - 1) / threadGroupSize.height, | |
1 | |
) | |
computeEncoder.dispatchThreadgroups(threadGroups, threadsPerThreadgroup: threadsPerThreadgroup) | |
computeEncoder.endEncoding() | |
commandBuffer.commit() | |
} | |
func updateTextureNormals() { | |
guard let heightMapTexture, | |
let meshData, | |
let commandBuffer = commandQueue.makeCommandBuffer(), | |
let computeEncoder = commandBuffer.makeComputeCommandEncoder() else { return } | |
computeEncoder.setComputePipelineState(deriveNormalsPipeline) | |
// Pass a readable version of the height map texture to the compute shader. | |
computeEncoder.setTexture(heightMapTexture.read(), index: 0) | |
// Pass a writable version of the height map texture to the compute shader. | |
computeEncoder.setTexture(heightMapTexture.replace(using: commandBuffer), index: 1) | |
// Pass the cell size to the compute shader. | |
var mutableCellSize = meshData.cellSize | |
computeEncoder.setBytes(&mutableCellSize, length: MemoryLayout<SIMD2<Float>>.size, index: 2) | |
let threadWidth = imageToGreyscalePipeline.threadExecutionWidth | |
let threadHeight = imageToGreyscalePipeline.maxTotalThreadsPerThreadgroup / threadWidth | |
let threadsPerThreadgroup = MTLSize(width: threadWidth, height: threadHeight, depth: 1) | |
let threadGroupSize = MTLSizeMake(8, 8, 1) | |
let threadGroups = MTLSizeMake( | |
(heightMapTexture.descriptor.width + threadGroupSize.width - 1) / threadGroupSize.width, | |
(heightMapTexture.descriptor.height + threadGroupSize.height - 1) / threadGroupSize.height, | |
1 | |
) | |
computeEncoder.dispatchThreadgroups(threadGroups, threadsPerThreadgroup: threadsPerThreadgroup) | |
computeEncoder.endEncoding() | |
commandBuffer.commit() | |
} | |
func updateMaterialTextureColors() { | |
guard let imageTexture = originalImageTexture, | |
let materialTexture, | |
let commandBuffer = commandQueue.makeCommandBuffer(), | |
let computeEncoder = commandBuffer.makeComputeCommandEncoder() else { return } | |
computeEncoder.setComputePipelineState(convertHeightMapToColorPipeline) | |
// input: rbg texture | |
computeEncoder.setTexture(imageTexture.read(), index: 0) | |
// output: rbg texture | |
computeEncoder.setTexture(materialTexture.replace(using: commandBuffer), index: 1) | |
var colorMode: Int32 = colorMode.value | |
computeEncoder.setBytes(&colorMode, length: MemoryLayout<Int32>.size, index: 0) | |
var progress = heightProgress | |
computeEncoder.setBytes(&progress, length: MemoryLayout<Float>.size, index: 1) | |
let threadWidth = convertHeightMapToColorPipeline.threadExecutionWidth | |
let threadHeight = convertHeightMapToColorPipeline.maxTotalThreadsPerThreadgroup / threadWidth | |
let threadsPerThreadgroup = MTLSize(width: threadWidth, height: threadHeight, depth: 1) | |
let threadGroupSize = MTLSizeMake(8, 8, 1) | |
let threadGroups = MTLSizeMake( | |
(materialTexture.descriptor.width + threadGroupSize.width - 1) / threadGroupSize.width, | |
(materialTexture.descriptor.height + threadGroupSize.height - 1) / threadGroupSize.height, | |
1 | |
) | |
computeEncoder.dispatchThreadgroups(threadGroups, threadsPerThreadgroup: threadsPerThreadgroup) | |
computeEncoder.endEncoding() | |
commandBuffer.commit() | |
} | |
func updateMeshGeometry() { | |
guard let mesh, | |
let heightMapTexture, | |
let meshData, | |
let commandBuffer = commandQueue.makeCommandBuffer(), | |
let computeEncoder = commandBuffer.makeComputeCommandEncoder() else { return } | |
var params = HeightMapParams(size: meshData.size, dimensions: meshData.dimensions) | |
let vertexBuffer = mesh.replace(bufferIndex: 0, using: commandBuffer) | |
let indexBuffer = mesh.replaceIndices(using: commandBuffer) | |
computeEncoder.setComputePipelineState(meshComputePipeline) | |
computeEncoder.setBuffer(vertexBuffer, offset: 0, index: 0) | |
computeEncoder.setBuffer(indexBuffer, offset: 0, index: 1) | |
computeEncoder.setBytes(¶ms, length: MemoryLayout<HeightMapParams>.stride, index: 2) | |
computeEncoder.setTexture(heightMapTexture.read(), index: 3) | |
let threadgroupSize = MTLSize(width: 8, height: 8, depth: 1) | |
let threadgroups = MTLSize( | |
width: (Int(meshData.dimensions.x) + threadgroupSize.width - 1) / threadgroupSize.width, | |
height: (Int(meshData.dimensions.y) + threadgroupSize.height - 1) / threadgroupSize.height, | |
depth: 1 | |
) | |
computeEncoder.dispatchThreadgroups(threadgroups, threadsPerThreadgroup: threadgroupSize) | |
computeEncoder.endEncoding() | |
commandBuffer.commit() | |
mesh.parts.replaceAll([ | |
LowLevelMesh.Part( | |
indexCount: meshData.indexCount, | |
topology: .triangle, | |
bounds: meshData.boundingBox | |
) | |
]) | |
} | |
func copyTextureResourceToLowLevelTexture(from textureResource: TextureResource) throws -> LowLevelTexture { | |
var descriptor = LowLevelTexture.Descriptor() | |
descriptor.textureType = .type2D | |
descriptor.pixelFormat = .rgba16Float | |
descriptor.width = textureResource.width | |
descriptor.height = textureResource.height | |
descriptor.mipmapLevelCount = 1 | |
descriptor.textureUsage = [.shaderRead, .shaderWrite] | |
let texture = try LowLevelTexture(descriptor: descriptor) | |
try textureResource.copy(to: texture.read()) | |
return texture | |
} | |
} | |
#Preview { | |
HeightMapToColoredMeshView() | |
} | |
enum ExampleHeightMapImage: CaseIterable { | |
case mountEverest | |
case k2 | |
case kangchenjunga | |
case lhotse | |
case makalu | |
static let baseURL = URL(string: "https://matt54.github.io/Resources/")! | |
var url: URL { | |
return ExampleHeightMapImage.baseURL.appendingPathComponent("\(filename)_2056.png") | |
} | |
var filename: String { | |
switch self { | |
case .mountEverest: | |
return "MT_EVEREST" | |
case .k2: | |
return "K2" | |
case .kangchenjunga: | |
return "Kangchenjunga" | |
case .lhotse: | |
return "Lhotse" | |
case .makalu: | |
return "Makalu" | |
} | |
} | |
var displayName: String { | |
switch self { | |
case .mountEverest: | |
return "Mount Everest" | |
case .k2: | |
return "K2" | |
case .kangchenjunga: | |
return "Kangchenjunga" | |
case .lhotse: | |
return "Lhotse" | |
case .makalu: | |
return "Makalu" | |
} | |
} | |
} | |
enum HeightMapColorMode: CaseIterable { | |
case terrain | |
case heatMap | |
var name: String { | |
switch self { | |
case .terrain: | |
"Terrain" | |
case .heatMap: | |
"Heat Map" | |
} | |
} | |
var value: Int32 { | |
switch self { | |
case .terrain: | |
0 | |
case .heatMap: | |
1 | |
} | |
} | |
} | |
struct HeightMapMeshData { | |
var size: SIMD2<Float> = [0.4, 0.4] | |
var dimensions: SIMD2<UInt32> = [2056, 2056] | |
var vertexCount: Int { | |
Int(dimensions.x * dimensions.y) | |
} | |
var indexCount: Int { | |
Int(6 * (dimensions.x - 1) * (dimensions.y - 1)) | |
} | |
var boundingBox: BoundingBox { | |
BoundingBox( | |
min: [-size.x/2, -size.y/2, 0], | |
max: [size.x/2, size.y/2, 0] | |
) | |
} | |
var cellSize: SIMD2<Float> { | |
SIMD2<Float>(size.x / Float(dimensions.x - 1), | |
size.y / Float(dimensions.y - 1)) | |
} | |
} | |
extension VertexData { | |
static var vertexAttributes: [LowLevelMesh.Attribute] = [ | |
.init(semantic: .position, format: .float3, offset: MemoryLayout<Self>.offset(of: \.position)!), | |
.init(semantic: .normal, format: .float3, offset: MemoryLayout<Self>.offset(of: \.normal)!), | |
.init(semantic: .uv0, format: .float2, offset: MemoryLayout<Self>.offset(of: \.uv)!) | |
] | |
static var vertexLayouts: [LowLevelMesh.Layout] = [ | |
.init(bufferIndex: 0, bufferStride: MemoryLayout<Self>.stride) | |
] | |
static var descriptor: LowLevelMesh.Descriptor { | |
var desc = LowLevelMesh.Descriptor() | |
desc.vertexAttributes = VertexData.vertexAttributes | |
desc.vertexLayouts = VertexData.vertexLayouts | |
desc.indexType = .uint32 | |
return desc | |
} | |
@MainActor static func initializeMesh(vertexCapacity: Int, | |
indexCapacity: Int) throws -> LowLevelMesh { | |
var desc = VertexData.descriptor | |
desc.vertexCapacity = vertexCapacity | |
desc.indexCapacity = indexCapacity | |
return try LowLevelMesh(descriptor: desc) | |
} | |
} | |
extension TextureResource { | |
static func loadOnlineImage(_ url: URL) async throws -> TextureResource { | |
let (data, _) = try await URLSession.shared.data(from: url) | |
let image = UIImage(data: data)! | |
let cgImage = image.cgImage! | |
return try await TextureResource(image: cgImage, options: .init(semantic: nil)) | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <simd/simd.h> | |
#ifndef VertexData_h | |
#define VertexData_h | |
struct VertexData { | |
simd_float3 position; | |
simd_float3 normal; | |
simd_float2 uv; | |
}; | |
#endif /* PlaneVertex_h */ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment