diff --git a/util/compression/zstd.cpp b/util/compression/zstd.cpp new file mode 100644 index 0000000..2c60f06 --- /dev/null +++ b/util/compression/zstd.cpp @@ -0,0 +1,160 @@ +#include "zstd.hpp" +#include +#include +#include +#include +#include + +// Implementation of ZSTD_CompressStream methods +size_t ZSTD_CompressStream::compress_continue(const void* src, void* dst, size_t srcSize) { + // For compatibility with the original interface where dst size is inferred + size_t dstCapacity = ZSTD_compressBound(srcSize); + return this->compress(src, srcSize, dst, dstCapacity); +} + +// Implementation of ZSTD_DecompressStream methods +size_t ZSTD_DecompressStream::decompress_continue(const void* src, void* dst, size_t srcSize) { + // Note: srcSize parameter is actually the destination buffer size + // This matches the confusing usage in the original VoxelOctree code + return this->decompress(src, srcSize, dst, srcSize); +} + +// Helper functions for the compression system +namespace { + // Simple hash function for LZ77-style matching + uint32_t computeHash(const uint8_t* data) { + return (data[0] << 16) | (data[1] << 8) | data[2]; + } +} + +// More advanced compression implementation +size_t ZSTD_CompressStream::compress(const void* src, size_t srcSize, void* dst, size_t dstCapacity) { + if (srcSize == 0 || dstCapacity == 0) return 0; + + const uint8_t* srcBytes = static_cast(src); + uint8_t* dstBytes = static_cast(dst); + + size_t dstPos = 0; + size_t srcPos = 0; + + // Simple LZ77 compression + while (srcPos < srcSize && dstPos + 3 < dstCapacity) { + // Try to find a match in previous data + size_t bestMatchPos = 0; + size_t bestMatchLen = 0; + + // Look for matches in recent data (simplified search window) + size_t searchStart = (srcPos > 1024) ? srcPos - 1024 : 0; + for (size_t i = searchStart; i < srcPos && i + 3 <= srcSize; i++) { + if (srcBytes[i] == srcBytes[srcPos]) { + size_t matchLen = 1; + while (srcPos + matchLen < srcSize && + i + matchLen < srcPos && + srcBytes[i + matchLen] == srcBytes[srcPos + matchLen] && + matchLen < 255) { + matchLen++; + } + + if (matchLen > bestMatchLen && matchLen >= 4) { + bestMatchLen = matchLen; + bestMatchPos = srcPos - i; + } + } + } + + if (bestMatchLen >= 4) { + // Encode match + dstBytes[dstPos++] = 0x80 | ((bestMatchLen - 4) & 0x7F); + dstBytes[dstPos++] = (bestMatchPos >> 8) & 0xFF; + dstBytes[dstPos++] = bestMatchPos & 0xFF; + srcPos += bestMatchLen; + } else { + // Encode literals + size_t literalStart = srcPos; + size_t maxLiteral = std::min(srcSize - srcPos, size_t(127)); + + // Find run of non-compressible data + while (srcPos - literalStart < maxLiteral) { + // Check if next few bytes could be compressed + bool canCompress = false; + if (srcPos + 3 < srcSize) { + // Simple heuristic: repeated bytes or patterns + if (srcBytes[srcPos] == srcBytes[srcPos + 1] && + srcBytes[srcPos] == srcBytes[srcPos + 2]) { + canCompress = true; + } + } + if (canCompress) break; + srcPos++; + } + + size_t literalLength = srcPos - literalStart; + if (literalLength > 0) { + if (dstPos + literalLength + 1 > dstCapacity) { + // Not enough space + break; + } + + dstBytes[dstPos++] = literalLength & 0x7F; + memcpy(dstBytes + dstPos, srcBytes + literalStart, literalLength); + dstPos += literalLength; + } + } + } + + return dstPos; +} + +size_t ZSTD_DecompressStream::decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity) { + if (srcSize == 0 || dstCapacity == 0) return 0; + + const uint8_t* srcBytes = static_cast(src); + uint8_t* dstBytes = static_cast(dst); + + size_t srcPos = 0; + size_t dstPos = 0; + + while (srcPos < srcSize && dstPos < dstCapacity) { + uint8_t header = srcBytes[srcPos++]; + + if (header & 0x80) { + // Match + if (srcPos + 1 >= srcSize) break; + + size_t matchLen = (header & 0x7F) + 4; + uint16_t matchDist = (srcBytes[srcPos] << 8) | srcBytes[srcPos + 1]; + srcPos += 2; + + if (matchDist == 0 || matchDist > dstPos) { + // Invalid match distance + break; + } + + size_t toCopy = std::min(matchLen, dstCapacity - dstPos); + size_t matchPos = dstPos - matchDist; + + for (size_t i = 0; i < toCopy; i++) { + dstBytes[dstPos++] = dstBytes[matchPos + i]; + } + + if (toCopy < matchLen) { + break; // Output buffer full + } + } else { + // Literals + size_t literalLength = header; + if (srcPos + literalLength > srcSize) break; + + size_t toCopy = std::min(literalLength, dstCapacity - dstPos); + memcpy(dstBytes + dstPos, srcBytes + srcPos, toCopy); + srcPos += toCopy; + dstPos += toCopy; + + if (toCopy < literalLength) { + break; // Output buffer full + } + } + } + + return dstPos; +} \ No newline at end of file diff --git a/util/compression/zstd.hpp b/util/compression/zstd.hpp new file mode 100644 index 0000000..876f592 --- /dev/null +++ b/util/compression/zstd.hpp @@ -0,0 +1,207 @@ +#ifndef ZSTD_HPP +#define ZSTD_HPP + +#include +#include +#include +#include + +// Simple ZSTD compression implementation (simplified version) +class ZSTD_Stream { +public: + virtual ~ZSTD_Stream() = default; +}; + +class ZSTD_CompressStream : public ZSTD_Stream { +private: + std::vector buffer; + size_t position; + +public: + ZSTD_CompressStream() : position(0) { + buffer.reserve(1024 * 1024); // 1MB initial capacity + } + + void reset() { + buffer.clear(); + position = 0; + } + + size_t compress(const void* src, size_t srcSize, void* dst, size_t dstCapacity) { + // Simplified compression - in reality this would use actual ZSTD algorithm + // For this example, we'll just copy with simple RLE-like compression + + if (dstCapacity < srcSize) { + return 0; // Not enough space + } + + const uint8_t* srcBytes = static_cast(src); + uint8_t* dstBytes = static_cast(dst); + + size_t dstPos = 0; + size_t srcPos = 0; + + while (srcPos < srcSize && dstPos + 2 < dstCapacity) { + // Simple RLE compression for repeated bytes + uint8_t current = srcBytes[srcPos]; + size_t runLength = 1; + + while (srcPos + runLength < srcSize && + srcBytes[srcPos + runLength] == current && + runLength < 127) { + runLength++; + } + + if (runLength > 3) { + // Encode as RLE + dstBytes[dstPos++] = 0x80 | (runLength & 0x7F); + dstBytes[dstPos++] = current; + srcPos += runLength; + } else { + // Encode as literal run + size_t literalStart = srcPos; + while (srcPos < srcSize && + (srcPos - literalStart < 127) && + (srcPos + 1 >= srcSize || + srcBytes[srcPos] != srcBytes[srcPos + 1] || + srcPos + 2 >= srcSize || + srcBytes[srcPos] != srcBytes[srcPos + 2])) { + srcPos++; + } + + size_t literalLength = srcPos - literalStart; + if (dstPos + literalLength + 1 > dstCapacity) { + break; + } + + dstBytes[dstPos++] = literalLength & 0x7F; + memcpy(dstBytes + dstPos, srcBytes + literalStart, literalLength); + dstPos += literalLength; + } + } + + return dstPos; + } + + size_t compress_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity) { + return compress(src, srcSize, dst, dstCapacity); + } + + const std::vector& getBuffer() const { return buffer; } +}; + +class ZSTD_DecompressStream : public ZSTD_Stream { +private: + size_t position; + +public: + ZSTD_DecompressStream() : position(0) {} + + void reset() { + position = 0; + } + + size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity) { + const uint8_t* srcBytes = static_cast(src); + uint8_t* dstBytes = static_cast(dst); + + size_t srcPos = 0; + size_t dstPos = 0; + + while (srcPos < srcSize && dstPos < dstCapacity) { + uint8_t header = srcBytes[srcPos++]; + + if (header & 0x80) { + // RLE encoded + size_t runLength = header & 0x7F; + if (srcPos >= srcSize) break; + + uint8_t value = srcBytes[srcPos++]; + + size_t toCopy = std::min(runLength, dstCapacity - dstPos); + memset(dstBytes + dstPos, value, toCopy); + dstPos += toCopy; + + if (toCopy < runLength) { + break; // Output buffer full + } + } else { + // Literal data + size_t literalLength = header; + if (srcPos + literalLength > srcSize) break; + + size_t toCopy = std::min(literalLength, dstCapacity - dstPos); + memcpy(dstBytes + dstPos, srcBytes + srcPos, toCopy); + srcPos += toCopy; + dstPos += toCopy; + + if (toCopy < literalLength) { + break; // Output buffer full + } + } + } + + return dstPos; + } + + size_t decompress_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity) { + return decompress(src, srcSize, dst, dstCapacity); + } +}; + +// Type definitions for compatibility with original code +using ZSTD_stream_t = ZSTD_CompressStream; + +// Stream creation functions +inline ZSTD_CompressStream* ZSTD_createStream() { + return new ZSTD_CompressStream(); +} + +inline ZSTD_DecompressStream* ZSTD_createStreamDecode() { + return new ZSTD_DecompressStream(); +} + +// Stream free functions +inline void ZSTD_freeStream(ZSTD_Stream* stream) { + delete stream; +} + +inline void ZSTD_freeStreamDecode(ZSTD_Stream* stream) { + delete stream; +} + +// Stream reset functions +inline void ZSTD_resetStream(ZSTD_CompressStream* stream) { + if (stream) stream->reset(); +} + +inline void ZSTD_setStreamDecode(ZSTD_DecompressStream* stream, const void* dict, size_t dictSize) { + if (stream) stream->reset(); + // Note: dict parameter is ignored in this simplified version + (void)dict; + (void)dictSize; +} + +// Compression functions +inline size_t ZSTD_compressBound(size_t srcSize) { + // Worst case: no compression + 1 byte header per 127 bytes + return srcSize + (srcSize / 127) + 1; +} + +inline size_t ZSTD_Compress_continue(ZSTD_CompressStream* stream, + const void* src, void* dst, + size_t srcSize) { + if (!stream) return 0; + return stream->compress_continue(src, srcSize, dst, ZSTD_compressBound(srcSize)); +} + +inline size_t ZSTD_Decompress_continue(ZSTD_DecompressStream* stream, + const void* src, void* dst, + size_t srcSize) { + if (!stream) return 0; + // Note: srcSize is actually the destination size in the original code + // This is confusing but matches the usage in VoxelOctree + return stream->decompress_continue(src, srcSize, dst, srcSize); +} + +#endif // ZSTD_HPP \ No newline at end of file diff --git a/util/grid/svo.hpp b/util/grid/svo.hpp new file mode 100644 index 0000000..0d45ccf --- /dev/null +++ b/util/grid/svo.hpp @@ -0,0 +1,124 @@ +#ifndef VOXELOCTREE_HPP +#define VOXELOCTREE_HPP + +#include "../vectorlogic/vec3.hpp" +#include "../compression/zstd.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class VoxelData; + +static const size_t CompressionBlockSize = 64*1024*1024; + +class VoxelOctree { +private: + static const size_t MaxScale = 23; + size_t _octSize; + std::vector _octree; + VoxelData* _voxels; + Vec3f _center; + size_t buildOctree(ChunkedAllocator& allocator, int x, int y, int z, int size, size_t descriptorIndex) { + _voxels->prepateDataAccess(x, y, z, size); + + int halfSize = size >> 1; + const std::array posX = { x + halfSize, x, x+halfSize, x, x+halfSize, x, x+halfSize, x}; + const std::array posY = { y + halfSize, y+halfSize, y, y, y+halfSize, y+halfSize, y, y}; + const std::array posZ = { z + halfSize, z+halfSize, z+halfSize, z+halfSize, z, z, z, z}; + + } +public: + VoxelOctree(const std::string& path) : _voxels(nullptr) { + std::ifstream file = std::ifstream(path, std::ios::binary); + if (!file.isopen()) { + throw std::runtime_error(std::string("failed to open: ") + path); + } + + float cd[3]; + file.read(reinterpret_cast(cd), sizeof(float) * 3); + _center = Vec3f(cd); + + uint64_t octreeSize; + file.read(reinterpret_cast(&octreeSize), sizeof(uint64_t)); + _octSize = octreeSize; + + _octree.resize(_octSize); + + std::vector compressionBuffer(zstd(static_cast(CompressionBlockSize))); + + std::unique_ptr stream(ZSTD_freeStreamDecode); + ZSTD_setStreamDecode(stream.get(), reinterpret_cast(_octree.data()), 0); + + uint64_t compressedSize = 0; + const size_t elementSize = sizeof(uint32_t); + for (uint64_t offset = 0; offset < _octSize * elementSize; offset += CompressionBlockSize) { + uint64_t compsize; + file.read(reinterpret_cast(&compsize), sizeof(uint64_t)); + + if (compsize > compressionBuffer.size()) compressionBuffer.resize(compsize); + file.read(compressionBuffer.data(), static_cast(compsize)); + + int outsize = std::min(_octSize * elementSize - offset, CompressionBlockSize); + ZSTD_Decompress_continue(stream.get(), compressionBuffer.data(), reinterpret_cast(_octree.data()) + offset, outsize); + + compressedSize += compsize + sizeof(uint64_t); + } + } + + VoxelOctree(VoxelData* voxels) : _voxels(voxels) { + std::unique_ptr> octreeAllocator = std::make_unique>(); + + octreeAllocator->pushBack(0); + buildOctree(*octreeAllocator, 0, 0, 0, _voxels->sideLength(), 0); + (*octreeAllocator)[0] |= 1 << 18; + + _octSize = octreeAllocator->size() + octreeAllocator-> insertionCount(); + _octree = octreeAllocator->finalize(); + _center = _voxels->getCenter(); + } + + void save(const char* path) { + std::ofstream file(path, std::iod::binary); + if (!file.is_open()) { + throw std::runtime_error(std::string("failed to write: ") + path); + } + + float cd[3] = _center; + + file.write(reinterpret_cast(cd), sizeof(float) * 3); + + file.write(reinterpret_cast(static_cast(_octSize)), sizeof(uint64_t)); + std::vector compressionBuffer(ZSTD_compressBound(static_cast(CompressionBlockSize))); + std::unique_ptr stream(ZSTD_createStream(), ZSTD_freeStream); + + ZSTD_resetStream(stream.get()); + + uint64_t compressedSize = 0; + const size_t elementSize = sizeof(uint32_t); + const char* src = reinterpret_cast(_octree.data()); + + for (uint64_t offset = 0; offset < _octSize * elementSize; offset += CompressionBlockSize) { + int outSize = _octSize * elementSize - offset, CompressionBlockSize; + uint64_t compSize = ZSTD_Compress_continue(stream.get(), src+offset, compressionBuffer.data(), outSize); + + file.write(reinterpret_cast(&compSize), sizeof(uint64_t)); + file.write(compressionBuffer.data(), static_cast(compSize)); + + compressedSize += compSize + sizeof(uint64_t); + } + } + + bool rayMarch(const Vec3f& origin, const Vec3f& dest, float rayScale, uint32_t& normal, float& t); + + Vec3f center() const { + return _center; + } +}; + +#endif \ No newline at end of file diff --git a/util/vectorlogic/vec3.hpp b/util/vectorlogic/vec3.hpp index 2af1f96..fce7e9a 100644 --- a/util/vectorlogic/vec3.hpp +++ b/util/vectorlogic/vec3.hpp @@ -9,14 +9,15 @@ template class Vec3 { public: - T x, y, z; + struct{ T x, y, z; }; Vec3() : x(0), y(0), z(0) {} Vec3(T x, T y, T z) : x(x), y(y), z(z) {} Vec3(T scalar) : x(scalar), y(scalar), z(scalar) {} + Vec3(float[3] acd) : x(acd[0]), y(acd[1]), z(acd[2]) {} Vec3(const class Vec2& vec2, T z = 0); - + Vec3& move(const Vec3& newpos) { x = newpos.x; y = newpos.y;