trying to steal an svo implementation since mine doesnt work well.
This commit is contained in:
Yggdrasil75
160
util/compression/zstd.cpp
Normal file
160
util/compression/zstd.cpp
Normal file
@@ -0,0 +1,160 @@
|
||||
#include "zstd.hpp"
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
|
||||
// Implementation of ZSTD_CompressStream methods
|
||||
size_t ZSTD_CompressStream::compress_continue(const void* src, void* dst, size_t srcSize) {
|
||||
// For compatibility with the original interface where dst size is inferred
|
||||
size_t dstCapacity = ZSTD_compressBound(srcSize);
|
||||
return this->compress(src, srcSize, dst, dstCapacity);
|
||||
}
|
||||
|
||||
// Implementation of ZSTD_DecompressStream methods
|
||||
size_t ZSTD_DecompressStream::decompress_continue(const void* src, void* dst, size_t srcSize) {
|
||||
// Note: srcSize parameter is actually the destination buffer size
|
||||
// This matches the confusing usage in the original VoxelOctree code
|
||||
return this->decompress(src, srcSize, dst, srcSize);
|
||||
}
|
||||
|
||||
// Helper functions for the compression system
|
||||
namespace {
|
||||
// Simple hash function for LZ77-style matching
|
||||
uint32_t computeHash(const uint8_t* data) {
|
||||
return (data[0] << 16) | (data[1] << 8) | data[2];
|
||||
}
|
||||
}
|
||||
|
||||
// More advanced compression implementation
|
||||
size_t ZSTD_CompressStream::compress(const void* src, size_t srcSize, void* dst, size_t dstCapacity) {
|
||||
if (srcSize == 0 || dstCapacity == 0) return 0;
|
||||
|
||||
const uint8_t* srcBytes = static_cast<const uint8_t*>(src);
|
||||
uint8_t* dstBytes = static_cast<uint8_t*>(dst);
|
||||
|
||||
size_t dstPos = 0;
|
||||
size_t srcPos = 0;
|
||||
|
||||
// Simple LZ77 compression
|
||||
while (srcPos < srcSize && dstPos + 3 < dstCapacity) {
|
||||
// Try to find a match in previous data
|
||||
size_t bestMatchPos = 0;
|
||||
size_t bestMatchLen = 0;
|
||||
|
||||
// Look for matches in recent data (simplified search window)
|
||||
size_t searchStart = (srcPos > 1024) ? srcPos - 1024 : 0;
|
||||
for (size_t i = searchStart; i < srcPos && i + 3 <= srcSize; i++) {
|
||||
if (srcBytes[i] == srcBytes[srcPos]) {
|
||||
size_t matchLen = 1;
|
||||
while (srcPos + matchLen < srcSize &&
|
||||
i + matchLen < srcPos &&
|
||||
srcBytes[i + matchLen] == srcBytes[srcPos + matchLen] &&
|
||||
matchLen < 255) {
|
||||
matchLen++;
|
||||
}
|
||||
|
||||
if (matchLen > bestMatchLen && matchLen >= 4) {
|
||||
bestMatchLen = matchLen;
|
||||
bestMatchPos = srcPos - i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bestMatchLen >= 4) {
|
||||
// Encode match
|
||||
dstBytes[dstPos++] = 0x80 | ((bestMatchLen - 4) & 0x7F);
|
||||
dstBytes[dstPos++] = (bestMatchPos >> 8) & 0xFF;
|
||||
dstBytes[dstPos++] = bestMatchPos & 0xFF;
|
||||
srcPos += bestMatchLen;
|
||||
} else {
|
||||
// Encode literals
|
||||
size_t literalStart = srcPos;
|
||||
size_t maxLiteral = std::min(srcSize - srcPos, size_t(127));
|
||||
|
||||
// Find run of non-compressible data
|
||||
while (srcPos - literalStart < maxLiteral) {
|
||||
// Check if next few bytes could be compressed
|
||||
bool canCompress = false;
|
||||
if (srcPos + 3 < srcSize) {
|
||||
// Simple heuristic: repeated bytes or patterns
|
||||
if (srcBytes[srcPos] == srcBytes[srcPos + 1] &&
|
||||
srcBytes[srcPos] == srcBytes[srcPos + 2]) {
|
||||
canCompress = true;
|
||||
}
|
||||
}
|
||||
if (canCompress) break;
|
||||
srcPos++;
|
||||
}
|
||||
|
||||
size_t literalLength = srcPos - literalStart;
|
||||
if (literalLength > 0) {
|
||||
if (dstPos + literalLength + 1 > dstCapacity) {
|
||||
// Not enough space
|
||||
break;
|
||||
}
|
||||
|
||||
dstBytes[dstPos++] = literalLength & 0x7F;
|
||||
memcpy(dstBytes + dstPos, srcBytes + literalStart, literalLength);
|
||||
dstPos += literalLength;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return dstPos;
|
||||
}
|
||||
|
||||
size_t ZSTD_DecompressStream::decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity) {
|
||||
if (srcSize == 0 || dstCapacity == 0) return 0;
|
||||
|
||||
const uint8_t* srcBytes = static_cast<const uint8_t*>(src);
|
||||
uint8_t* dstBytes = static_cast<uint8_t*>(dst);
|
||||
|
||||
size_t srcPos = 0;
|
||||
size_t dstPos = 0;
|
||||
|
||||
while (srcPos < srcSize && dstPos < dstCapacity) {
|
||||
uint8_t header = srcBytes[srcPos++];
|
||||
|
||||
if (header & 0x80) {
|
||||
// Match
|
||||
if (srcPos + 1 >= srcSize) break;
|
||||
|
||||
size_t matchLen = (header & 0x7F) + 4;
|
||||
uint16_t matchDist = (srcBytes[srcPos] << 8) | srcBytes[srcPos + 1];
|
||||
srcPos += 2;
|
||||
|
||||
if (matchDist == 0 || matchDist > dstPos) {
|
||||
// Invalid match distance
|
||||
break;
|
||||
}
|
||||
|
||||
size_t toCopy = std::min(matchLen, dstCapacity - dstPos);
|
||||
size_t matchPos = dstPos - matchDist;
|
||||
|
||||
for (size_t i = 0; i < toCopy; i++) {
|
||||
dstBytes[dstPos++] = dstBytes[matchPos + i];
|
||||
}
|
||||
|
||||
if (toCopy < matchLen) {
|
||||
break; // Output buffer full
|
||||
}
|
||||
} else {
|
||||
// Literals
|
||||
size_t literalLength = header;
|
||||
if (srcPos + literalLength > srcSize) break;
|
||||
|
||||
size_t toCopy = std::min(literalLength, dstCapacity - dstPos);
|
||||
memcpy(dstBytes + dstPos, srcBytes + srcPos, toCopy);
|
||||
srcPos += toCopy;
|
||||
dstPos += toCopy;
|
||||
|
||||
if (toCopy < literalLength) {
|
||||
break; // Output buffer full
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return dstPos;
|
||||
}
|
||||
207
util/compression/zstd.hpp
Normal file
207
util/compression/zstd.hpp
Normal file
@@ -0,0 +1,207 @@
|
||||
#ifndef ZSTD_HPP
|
||||
#define ZSTD_HPP
|
||||
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
|
||||
// Simple ZSTD compression implementation (simplified version)
|
||||
class ZSTD_Stream {
|
||||
public:
|
||||
virtual ~ZSTD_Stream() = default;
|
||||
};
|
||||
|
||||
class ZSTD_CompressStream : public ZSTD_Stream {
|
||||
private:
|
||||
std::vector<uint8_t> buffer;
|
||||
size_t position;
|
||||
|
||||
public:
|
||||
ZSTD_CompressStream() : position(0) {
|
||||
buffer.reserve(1024 * 1024); // 1MB initial capacity
|
||||
}
|
||||
|
||||
void reset() {
|
||||
buffer.clear();
|
||||
position = 0;
|
||||
}
|
||||
|
||||
size_t compress(const void* src, size_t srcSize, void* dst, size_t dstCapacity) {
|
||||
// Simplified compression - in reality this would use actual ZSTD algorithm
|
||||
// For this example, we'll just copy with simple RLE-like compression
|
||||
|
||||
if (dstCapacity < srcSize) {
|
||||
return 0; // Not enough space
|
||||
}
|
||||
|
||||
const uint8_t* srcBytes = static_cast<const uint8_t*>(src);
|
||||
uint8_t* dstBytes = static_cast<uint8_t*>(dst);
|
||||
|
||||
size_t dstPos = 0;
|
||||
size_t srcPos = 0;
|
||||
|
||||
while (srcPos < srcSize && dstPos + 2 < dstCapacity) {
|
||||
// Simple RLE compression for repeated bytes
|
||||
uint8_t current = srcBytes[srcPos];
|
||||
size_t runLength = 1;
|
||||
|
||||
while (srcPos + runLength < srcSize &&
|
||||
srcBytes[srcPos + runLength] == current &&
|
||||
runLength < 127) {
|
||||
runLength++;
|
||||
}
|
||||
|
||||
if (runLength > 3) {
|
||||
// Encode as RLE
|
||||
dstBytes[dstPos++] = 0x80 | (runLength & 0x7F);
|
||||
dstBytes[dstPos++] = current;
|
||||
srcPos += runLength;
|
||||
} else {
|
||||
// Encode as literal run
|
||||
size_t literalStart = srcPos;
|
||||
while (srcPos < srcSize &&
|
||||
(srcPos - literalStart < 127) &&
|
||||
(srcPos + 1 >= srcSize ||
|
||||
srcBytes[srcPos] != srcBytes[srcPos + 1] ||
|
||||
srcPos + 2 >= srcSize ||
|
||||
srcBytes[srcPos] != srcBytes[srcPos + 2])) {
|
||||
srcPos++;
|
||||
}
|
||||
|
||||
size_t literalLength = srcPos - literalStart;
|
||||
if (dstPos + literalLength + 1 > dstCapacity) {
|
||||
break;
|
||||
}
|
||||
|
||||
dstBytes[dstPos++] = literalLength & 0x7F;
|
||||
memcpy(dstBytes + dstPos, srcBytes + literalStart, literalLength);
|
||||
dstPos += literalLength;
|
||||
}
|
||||
}
|
||||
|
||||
return dstPos;
|
||||
}
|
||||
|
||||
size_t compress_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity) {
|
||||
return compress(src, srcSize, dst, dstCapacity);
|
||||
}
|
||||
|
||||
const std::vector<uint8_t>& getBuffer() const { return buffer; }
|
||||
};
|
||||
|
||||
class ZSTD_DecompressStream : public ZSTD_Stream {
|
||||
private:
|
||||
size_t position;
|
||||
|
||||
public:
|
||||
ZSTD_DecompressStream() : position(0) {}
|
||||
|
||||
void reset() {
|
||||
position = 0;
|
||||
}
|
||||
|
||||
size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity) {
|
||||
const uint8_t* srcBytes = static_cast<const uint8_t*>(src);
|
||||
uint8_t* dstBytes = static_cast<uint8_t*>(dst);
|
||||
|
||||
size_t srcPos = 0;
|
||||
size_t dstPos = 0;
|
||||
|
||||
while (srcPos < srcSize && dstPos < dstCapacity) {
|
||||
uint8_t header = srcBytes[srcPos++];
|
||||
|
||||
if (header & 0x80) {
|
||||
// RLE encoded
|
||||
size_t runLength = header & 0x7F;
|
||||
if (srcPos >= srcSize) break;
|
||||
|
||||
uint8_t value = srcBytes[srcPos++];
|
||||
|
||||
size_t toCopy = std::min(runLength, dstCapacity - dstPos);
|
||||
memset(dstBytes + dstPos, value, toCopy);
|
||||
dstPos += toCopy;
|
||||
|
||||
if (toCopy < runLength) {
|
||||
break; // Output buffer full
|
||||
}
|
||||
} else {
|
||||
// Literal data
|
||||
size_t literalLength = header;
|
||||
if (srcPos + literalLength > srcSize) break;
|
||||
|
||||
size_t toCopy = std::min(literalLength, dstCapacity - dstPos);
|
||||
memcpy(dstBytes + dstPos, srcBytes + srcPos, toCopy);
|
||||
srcPos += toCopy;
|
||||
dstPos += toCopy;
|
||||
|
||||
if (toCopy < literalLength) {
|
||||
break; // Output buffer full
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return dstPos;
|
||||
}
|
||||
|
||||
size_t decompress_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity) {
|
||||
return decompress(src, srcSize, dst, dstCapacity);
|
||||
}
|
||||
};
|
||||
|
||||
// Type definitions for compatibility with original code
|
||||
using ZSTD_stream_t = ZSTD_CompressStream;
|
||||
|
||||
// Stream creation functions
|
||||
inline ZSTD_CompressStream* ZSTD_createStream() {
|
||||
return new ZSTD_CompressStream();
|
||||
}
|
||||
|
||||
inline ZSTD_DecompressStream* ZSTD_createStreamDecode() {
|
||||
return new ZSTD_DecompressStream();
|
||||
}
|
||||
|
||||
// Stream free functions
|
||||
inline void ZSTD_freeStream(ZSTD_Stream* stream) {
|
||||
delete stream;
|
||||
}
|
||||
|
||||
inline void ZSTD_freeStreamDecode(ZSTD_Stream* stream) {
|
||||
delete stream;
|
||||
}
|
||||
|
||||
// Stream reset functions
|
||||
inline void ZSTD_resetStream(ZSTD_CompressStream* stream) {
|
||||
if (stream) stream->reset();
|
||||
}
|
||||
|
||||
inline void ZSTD_setStreamDecode(ZSTD_DecompressStream* stream, const void* dict, size_t dictSize) {
|
||||
if (stream) stream->reset();
|
||||
// Note: dict parameter is ignored in this simplified version
|
||||
(void)dict;
|
||||
(void)dictSize;
|
||||
}
|
||||
|
||||
// Compression functions
|
||||
inline size_t ZSTD_compressBound(size_t srcSize) {
|
||||
// Worst case: no compression + 1 byte header per 127 bytes
|
||||
return srcSize + (srcSize / 127) + 1;
|
||||
}
|
||||
|
||||
inline size_t ZSTD_Compress_continue(ZSTD_CompressStream* stream,
|
||||
const void* src, void* dst,
|
||||
size_t srcSize) {
|
||||
if (!stream) return 0;
|
||||
return stream->compress_continue(src, srcSize, dst, ZSTD_compressBound(srcSize));
|
||||
}
|
||||
|
||||
inline size_t ZSTD_Decompress_continue(ZSTD_DecompressStream* stream,
|
||||
const void* src, void* dst,
|
||||
size_t srcSize) {
|
||||
if (!stream) return 0;
|
||||
// Note: srcSize is actually the destination size in the original code
|
||||
// This is confusing but matches the usage in VoxelOctree
|
||||
return stream->decompress_continue(src, srcSize, dst, srcSize);
|
||||
}
|
||||
|
||||
#endif // ZSTD_HPP
|
||||
124
util/grid/svo.hpp
Normal file
124
util/grid/svo.hpp
Normal file
@@ -0,0 +1,124 @@
|
||||
#ifndef VOXELOCTREE_HPP
|
||||
#define VOXELOCTREE_HPP
|
||||
|
||||
#include "../vectorlogic/vec3.hpp"
|
||||
#include "../compression/zstd.hpp"
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <iostream>
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <cmath>
|
||||
#include <bit>
|
||||
|
||||
class VoxelData;
|
||||
|
||||
static const size_t CompressionBlockSize = 64*1024*1024;
|
||||
|
||||
class VoxelOctree {
|
||||
private:
|
||||
static const size_t MaxScale = 23;
|
||||
size_t _octSize;
|
||||
std::vector<uint32_t> _octree;
|
||||
VoxelData* _voxels;
|
||||
Vec3f _center;
|
||||
size_t buildOctree(ChunkedAllocator<uint32_t>& allocator, int x, int y, int z, int size, size_t descriptorIndex) {
|
||||
_voxels->prepateDataAccess(x, y, z, size);
|
||||
|
||||
int halfSize = size >> 1;
|
||||
const std::array<int, 8> posX = { x + halfSize, x, x+halfSize, x, x+halfSize, x, x+halfSize, x};
|
||||
const std::array<int, 8> posY = { y + halfSize, y+halfSize, y, y, y+halfSize, y+halfSize, y, y};
|
||||
const std::array<int, 8> posZ = { z + halfSize, z+halfSize, z+halfSize, z+halfSize, z, z, z, z};
|
||||
|
||||
}
|
||||
public:
|
||||
VoxelOctree(const std::string& path) : _voxels(nullptr) {
|
||||
std::ifstream file = std::ifstream(path, std::ios::binary);
|
||||
if (!file.isopen()) {
|
||||
throw std::runtime_error(std::string("failed to open: ") + path);
|
||||
}
|
||||
|
||||
float cd[3];
|
||||
file.read(reinterpret_cast<char*>(cd), sizeof(float) * 3);
|
||||
_center = Vec3f(cd);
|
||||
|
||||
uint64_t octreeSize;
|
||||
file.read(reinterpret_cast<char*>(&octreeSize), sizeof(uint64_t));
|
||||
_octSize = octreeSize;
|
||||
|
||||
_octree.resize(_octSize);
|
||||
|
||||
std::vector<uint8_t> compressionBuffer(zstd(static_cast<int>(CompressionBlockSize)));
|
||||
|
||||
std::unique_ptr<ZSTD_Stream, decltype(&ZSTD_freeStreamDecode)> stream(ZSTD_freeStreamDecode);
|
||||
ZSTD_setStreamDecode(stream.get(), reinterpret_cast<char*>(_octree.data()), 0);
|
||||
|
||||
uint64_t compressedSize = 0;
|
||||
const size_t elementSize = sizeof(uint32_t);
|
||||
for (uint64_t offset = 0; offset < _octSize * elementSize; offset += CompressionBlockSize) {
|
||||
uint64_t compsize;
|
||||
file.read(reinterpret_cast<char*>(&compsize), sizeof(uint64_t));
|
||||
|
||||
if (compsize > compressionBuffer.size()) compressionBuffer.resize(compsize);
|
||||
file.read(compressionBuffer.data(), static_cast<std::streamsize>(compsize));
|
||||
|
||||
int outsize = std::min(_octSize * elementSize - offset, CompressionBlockSize);
|
||||
ZSTD_Decompress_continue(stream.get(), compressionBuffer.data(), reinterpret_cast<char*>(_octree.data()) + offset, outsize);
|
||||
|
||||
compressedSize += compsize + sizeof(uint64_t);
|
||||
}
|
||||
}
|
||||
|
||||
VoxelOctree(VoxelData* voxels) : _voxels(voxels) {
|
||||
std::unique_ptr<ChunkedAllocator<uint32_t>> octreeAllocator = std::make_unique<ChunkedAllocator<uint32_t>>();
|
||||
|
||||
octreeAllocator->pushBack(0);
|
||||
buildOctree(*octreeAllocator, 0, 0, 0, _voxels->sideLength(), 0);
|
||||
(*octreeAllocator)[0] |= 1 << 18;
|
||||
|
||||
_octSize = octreeAllocator->size() + octreeAllocator-> insertionCount();
|
||||
_octree = octreeAllocator->finalize();
|
||||
_center = _voxels->getCenter();
|
||||
}
|
||||
|
||||
void save(const char* path) {
|
||||
std::ofstream file(path, std::iod::binary);
|
||||
if (!file.is_open()) {
|
||||
throw std::runtime_error(std::string("failed to write: ") + path);
|
||||
}
|
||||
|
||||
float cd[3] = _center;
|
||||
|
||||
file.write(reinterpret_cast<const char*>(cd), sizeof(float) * 3);
|
||||
|
||||
file.write(reinterpret_cast<const char*>(static_cast<uint64_t>(_octSize)), sizeof(uint64_t));
|
||||
std::vector<uint8_t> compressionBuffer(ZSTD_compressBound(static_cast<int>(CompressionBlockSize)));
|
||||
std::unique_ptr<ZSTD_stream_t, decltype(&ZSTD_freeStream)> stream(ZSTD_createStream(), ZSTD_freeStream);
|
||||
|
||||
ZSTD_resetStream(stream.get());
|
||||
|
||||
uint64_t compressedSize = 0;
|
||||
const size_t elementSize = sizeof(uint32_t);
|
||||
const char* src = reinterpret_cast<const char*>(_octree.data());
|
||||
|
||||
for (uint64_t offset = 0; offset < _octSize * elementSize; offset += CompressionBlockSize) {
|
||||
int outSize = _octSize * elementSize - offset, CompressionBlockSize;
|
||||
uint64_t compSize = ZSTD_Compress_continue(stream.get(), src+offset, compressionBuffer.data(), outSize);
|
||||
|
||||
file.write(reinterpret_cast<const char*>(&compSize), sizeof(uint64_t));
|
||||
file.write(compressionBuffer.data(), static_cast<std::streamsize>(compSize));
|
||||
|
||||
compressedSize += compSize + sizeof(uint64_t);
|
||||
}
|
||||
}
|
||||
|
||||
bool rayMarch(const Vec3f& origin, const Vec3f& dest, float rayScale, uint32_t& normal, float& t);
|
||||
|
||||
Vec3f center() const {
|
||||
return _center;
|
||||
}
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -9,11 +9,12 @@
|
||||
template<typename T>
|
||||
class Vec3 {
|
||||
public:
|
||||
T x, y, z;
|
||||
struct{ T x, y, z; };
|
||||
|
||||
Vec3() : x(0), y(0), z(0) {}
|
||||
Vec3(T x, T y, T z) : x(x), y(y), z(z) {}
|
||||
Vec3(T scalar) : x(scalar), y(scalar), z(scalar) {}
|
||||
Vec3(float[3] acd) : x(acd[0]), y(acd[1]), z(acd[2]) {}
|
||||
|
||||
Vec3(const class Vec2& vec2, T z = 0);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user