This repository has been archived on 2023-10-09. You can view files and clone it, but cannot push or open issues or pull requests.
Files
blender-archive/source/blender/blenlib/BLI_filereader.h

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

66 lines
2.2 KiB
C++
Raw Normal View History

/* SPDX-License-Identifier: GPL-2.0-or-later
* Copyright 2001-2002 NaN Holding BV. All rights reserved. */
/** \file
* \ingroup bli
* \brief Wrapper for reading from various sources (e.g. raw files, compressed files, memory...).
*/
#pragma once
#ifdef WIN32
# include "BLI_winstuff.h"
#else
# include <sys/types.h>
#endif
#include "BLI_compiler_attrs.h"
#include "BLI_utildefines.h"
#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
typedef int64_t off64_t;
#endif
#ifdef __cplusplus
extern "C" {
#endif
struct FileReader;
typedef ssize_t (*FileReaderReadFn)(struct FileReader *reader, void *buffer, size_t size);
typedef off64_t (*FileReaderSeekFn)(struct FileReader *reader, off64_t offset, int whence);
typedef void (*FileReaderCloseFn)(struct FileReader *reader);
/** General structure for all #FileReaders, implementations add custom fields at the end. */
typedef struct FileReader {
FileReaderReadFn read;
FileReaderSeekFn seek;
FileReaderCloseFn close;
off64_t offset;
} FileReader;
/* Functions for opening the various types of FileReader.
* They either succeed and return a valid FileReader, or fail and return NULL.
*
2021-10-18 11:16:24 +11:00
* If a FileReader is created, it has to be cleaned up and freed by calling its close()
* function unless another FileReader has taken ownership - for example, `Zstd` & `Gzip`
* take over the base FileReader and will clean it up when their clean() is called.
*/
/** Create #FileReader from raw file descriptor. */
FileReader *BLI_filereader_new_file(int filedes) ATTR_WARN_UNUSED_RESULT;
/** Create #FileReader from raw file descriptor using memory-mapped IO. */
FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
/** Create #FileReader from a region of memory. */
FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
ATTR_NONNULL();
/** Create #FileReader from applying `Zstd` decompression on an underlying file. */
Add support for Zstandard compression for .blend files Compressing blendfiles can help save a lot of disk space, but the slowdown while loading and saving is a major annoyance. Currently Blender uses Zlib (aka gzip aka Deflate) for compression, but there are now several more modern algorithms that outperform it in every way. In this patch, I decided for Zstandard aka Zstd for several reasons: - It is widely supported, both in other programs and libraries as well as in general-purpose compression utilities on Unix - It is extremely flexible - spanning several orders of magnitude of compression speeds depending on the level setting. - It is pretty much on the Pareto frontier for all of its configurations (meaning that no other algorithm is both faster and more efficient). One downside of course is that older versions of Blender will not be able to read these files, but one can always just re-save them without compression or decompress the file manually with an external tool. The implementation here saves additional metadata into the compressed file in order to allow for efficient seeking when loading. This is standard-compliant and will be ignored by other tools that support Zstd. If the metadata is not present (e.g. because you manually compressed a .blend file with another tool), Blender will fall back to sequential reading. Saving is multithreaded to improve performance. Loading is currently not multithreaded since it's not easy to predict the access patterns of the loading code when seeking is supported. In the future, we might want to look into making this more predictable or disabling seeking for the main .blend file, which would then allow for multiple background threads that decompress data ahead of time. The compression level was chosen to get sizes comparable to previous versions at much higher speeds. In the future, this could be exposed as an option. Reviewed By: campbellbarton, brecht, mont29 Differential Revision: https://developer.blender.org/D5799
2021-08-21 03:15:31 +02:00
FileReader *BLI_filereader_new_zstd(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
/** Create #FileReader from applying `Gzip` decompression on an underlying file. */
FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
#ifdef __cplusplus
}
#endif