Compare commits
120 Commits
temp-unity
...
experiment
Author | SHA1 | Date | |
---|---|---|---|
03f726a550 | |||
2112710249 | |||
96f73c9a16 | |||
2de71cff72 | |||
260bcc482d | |||
065b88ed8b | |||
d8834b0015 | |||
329cda9519 | |||
0a6186dcdf | |||
662e37d280 | |||
edfcaf0abd | |||
25c42e650d | |||
7dcd72b0d3 | |||
![]() |
fc993a7d54 | ||
659385aa0d | |||
![]() |
11c03ff03d | ||
d4595187dd | |||
![]() |
1aaf6f4a1d | ||
5090bfaa37 | |||
ff66c0b24c | |||
5f9b3950eb | |||
c7e222c5ee | |||
f595b5aaf9 | |||
f1839a872b | |||
d8fc478dd7 | |||
123fd3d5c4 | |||
54e8b0ad57 | |||
eea7e751f4 | |||
4fa97ee3c6 | |||
9939b28888 | |||
414aaad1d4 | |||
c2e155f429 | |||
9413a5a00d | |||
dbe3303180 | |||
94ff60acc1 | |||
20f0c76061 | |||
872db161dc | |||
b926fb8acc | |||
dca1a1198a | |||
241ac50684 | |||
5dc643fa98 | |||
c01174354f | |||
4c4c6951f8 | |||
f3bf9b5035 | |||
0f6b47c49d | |||
6d0eb66464 | |||
c1806d565c | |||
![]() |
a22d87bdec | ||
![]() |
1e21b4e5dd | ||
![]() |
e68ddf02bb | ||
![]() |
acfce155e7 | ||
![]() |
0590bff3ab | ||
![]() |
a765f9f712 | ||
![]() |
651e702c52 | ||
![]() |
abe39529f6 | ||
![]() |
8659ae3b2d | ||
5f820329b0 | |||
f3395c753f | |||
4e90f53dbc | |||
251b41702d | |||
c3e90684bf | |||
81a29b8c9e | |||
a4d2d344e5 | |||
445f87684a | |||
105d3fb41e | |||
eb400e3ec2 | |||
![]() |
c4df8f301f | ||
![]() |
507dc9eb29 | ||
![]() |
c1b6319e9a | ||
600ffb71d3 | |||
d04aa1fd44 | |||
7ee74c6988 | |||
578c279ed9 | |||
e70c463661 | |||
040f67df6c | |||
519a6c425d | |||
a84e8fab2b | |||
7bda09e55d | |||
b26cc7ce2e | |||
af26fadb31 | |||
![]() |
f0328b464c | ||
![]() |
be493d60b5 | ||
![]() |
0e7bbe614f | ||
32bcef550e | |||
a684165cad | |||
0feac81b3c | |||
e751367fe7 | |||
518e436a37 | |||
a855d8ca6b | |||
eb90899cd0 | |||
e9a60ce199 | |||
9500604e36 | |||
![]() |
ba0146db67 | ||
![]() |
68da165ff2 | ||
![]() |
9225ca1227 | ||
9a229688a8 | |||
00e0048175 | |||
bf4b625992 | |||
![]() |
d342af8fe4 | ||
![]() |
50253272bc | ||
![]() |
261afa7bce | ||
![]() |
192a2e6b3b | ||
![]() |
7f9d5895ad | ||
![]() |
3537a57974 | ||
3de4388252 | |||
0fc72070fe | |||
a6a2c045a9 | |||
16ddba2b0d | |||
c872ff784f | |||
8ab1fc7a24 | |||
3f97468bd4 | |||
b39820ed7c | |||
22ab8523fa | |||
452a679869 | |||
eca6e4cb2d | |||
c333af11bb | |||
2aec7410fd | |||
79327e2a52 | |||
![]() |
12dd850446 | ||
![]() |
33bcefcf71 |
66
build_files/cmake/Modules/FindZstd.cmake
Normal file
66
build_files/cmake/Modules/FindZstd.cmake
Normal file
@@ -0,0 +1,66 @@
|
||||
# - Find Zstd library
|
||||
# Find the native Zstd includes and library
|
||||
# This module defines
|
||||
# ZSTD_INCLUDE_DIRS, where to find zstd.h, Set when
|
||||
# ZSTD_INCLUDE_DIR is found.
|
||||
# ZSTD_LIBRARIES, libraries to link against to use Zstd.
|
||||
# ZSTD_ROOT_DIR, The base directory to search for Zstd.
|
||||
# This can also be an environment variable.
|
||||
# ZSTD_FOUND, If false, do not try to use Zstd.
|
||||
#
|
||||
# also defined, but not for general use are
|
||||
# ZSTD_LIBRARY, where to find the Zstd library.
|
||||
|
||||
#=============================================================================
|
||||
# Copyright 2019 Blender Foundation.
|
||||
#
|
||||
# Distributed under the OSI-approved BSD License (the "License");
|
||||
# see accompanying file Copyright.txt for details.
|
||||
#
|
||||
# This software is distributed WITHOUT ANY WARRANTY; without even the
|
||||
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See the License for more information.
|
||||
#=============================================================================
|
||||
|
||||
# If ZSTD_ROOT_DIR was defined in the environment, use it.
|
||||
IF(NOT ZSTD_ROOT_DIR AND NOT $ENV{ZSTD_ROOT_DIR} STREQUAL "")
|
||||
SET(ZSTD_ROOT_DIR $ENV{ZSTD_ROOT_DIR})
|
||||
ENDIF()
|
||||
|
||||
SET(_zstd_SEARCH_DIRS
|
||||
${ZSTD_ROOT_DIR}
|
||||
)
|
||||
|
||||
FIND_PATH(ZSTD_INCLUDE_DIR
|
||||
NAMES
|
||||
zstd.h
|
||||
HINTS
|
||||
${_zstd_SEARCH_DIRS}
|
||||
PATH_SUFFIXES
|
||||
include
|
||||
)
|
||||
|
||||
FIND_LIBRARY(ZSTD_LIBRARY
|
||||
NAMES
|
||||
zstd
|
||||
HINTS
|
||||
${_zstd_SEARCH_DIRS}
|
||||
PATH_SUFFIXES
|
||||
lib64 lib
|
||||
)
|
||||
|
||||
# handle the QUIETLY and REQUIRED arguments and set ZSTD_FOUND to TRUE if
|
||||
# all listed variables are TRUE
|
||||
INCLUDE(FindPackageHandleStandardArgs)
|
||||
FIND_PACKAGE_HANDLE_STANDARD_ARGS(Zstd DEFAULT_MSG
|
||||
ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
|
||||
|
||||
IF(ZSTD_FOUND)
|
||||
SET(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
|
||||
SET(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
|
||||
ENDIF()
|
||||
|
||||
MARK_AS_ADVANCED(
|
||||
ZSTD_INCLUDE_DIR
|
||||
ZSTD_LIBRARY
|
||||
)
|
@@ -441,6 +441,9 @@ if(WITH_HARU)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(ZSTD_ROOT_DIR ${LIBDIR}/zstd)
|
||||
find_package(Zstd REQUIRED)
|
||||
|
||||
if(EXISTS ${LIBDIR})
|
||||
without_system_libs_end()
|
||||
endif()
|
||||
|
@@ -99,6 +99,7 @@ endif()
|
||||
find_package_wrapper(JPEG REQUIRED)
|
||||
find_package_wrapper(PNG REQUIRED)
|
||||
find_package_wrapper(ZLIB REQUIRED)
|
||||
find_package_wrapper(Zstd REQUIRED)
|
||||
find_package_wrapper(Freetype REQUIRED)
|
||||
|
||||
if(WITH_PYTHON)
|
||||
|
@@ -873,3 +873,6 @@ if(WITH_HARU)
|
||||
set(WITH_HARU OFF)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(ZSTD_INCLUDE_DIRS ${LIBDIR}/zstd/include)
|
||||
set(ZSTD_LIBRARIES ${LIBDIR}/zstd/lib/zstd_static.lib)
|
||||
|
@@ -154,18 +154,17 @@ bool BLI_file_is_writable(const char *file) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL
|
||||
bool BLI_file_touch(const char *file) ATTR_NONNULL();
|
||||
bool BLI_file_alias_target(const char *filepath, char *r_targetpath) ATTR_WARN_UNUSED_RESULT;
|
||||
|
||||
#if 0 /* UNUSED */
|
||||
int BLI_file_gzip(const char *from, const char *to) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
#endif
|
||||
char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) ATTR_WARN_UNUSED_RESULT
|
||||
ATTR_NONNULL();
|
||||
size_t BLI_gzip_mem_to_file_at_pos(void *buf,
|
||||
size_t len,
|
||||
FILE *file,
|
||||
size_t gz_stream_offset,
|
||||
int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
|
||||
bool BLI_file_magic_is_gzip(const char header[4]);
|
||||
|
||||
size_t BLI_file_zstd_from_mem_at_pos(void *buf,
|
||||
size_t len,
|
||||
FILE *file,
|
||||
size_t file_offset,
|
||||
int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset)
|
||||
ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
bool BLI_file_magic_is_zstd(const char header[4]);
|
||||
|
||||
size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
|
||||
size_t BLI_file_size(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
|
||||
|
81
source/blender/blenlib/BLI_filereader.h
Normal file
81
source/blender/blenlib/BLI_filereader.h
Normal file
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
* The Original Code is Copyright (C) 2001-2002 by NaN Holding BV.
|
||||
* All rights reserved.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup bli
|
||||
* \brief Wrapper for reading from various sources (e.g. raw files, compressed files, memory...).
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef WIN32
|
||||
# include "BLI_winstuff.h"
|
||||
#else
|
||||
# include <sys/types.h>
|
||||
#endif
|
||||
|
||||
#include "BLI_compiler_attrs.h"
|
||||
#include "BLI_utildefines.h"
|
||||
|
||||
#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
|
||||
typedef int64_t off64_t;
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct FileReader;
|
||||
|
||||
typedef ssize_t (*FileReaderReadFn)(struct FileReader *reader, void *buffer, size_t size);
|
||||
typedef off64_t (*FileReaderSeekFn)(struct FileReader *reader, off64_t offset, int whence);
|
||||
typedef void (*FileReaderCloseFn)(struct FileReader *reader);
|
||||
|
||||
/* General structure for all FileReaders, implementations add custom fields at the end. */
|
||||
typedef struct FileReader {
|
||||
FileReaderReadFn read;
|
||||
FileReaderSeekFn seek;
|
||||
FileReaderCloseFn close;
|
||||
|
||||
off64_t offset;
|
||||
} FileReader;
|
||||
|
||||
/* Functions for opening the various types of FileReader.
|
||||
* They either succeed and return a valid FileReader, or fail and return NULL.
|
||||
*
|
||||
* If a FileReader is created, it has to be cleaned up and freed by calling
|
||||
* its close() function unless another FileReader has taken ownership - for example,
|
||||
* Zstd and Gzip take over the base FileReader and will clean it up when their clean() is called.
|
||||
*/
|
||||
|
||||
/* Create FileReader from raw file descriptor. */
|
||||
FileReader *BLI_filereader_new_file(int filedes) ATTR_WARN_UNUSED_RESULT;
|
||||
/* Create FileReader from raw file descriptor using memory-mapped IO. */
|
||||
FileReader *BLI_filereader_new_mmap(int filedes) ATTR_WARN_UNUSED_RESULT;
|
||||
/* Create FileReader from a region of memory. */
|
||||
FileReader *BLI_filereader_new_memory(const void *data, size_t len) ATTR_WARN_UNUSED_RESULT
|
||||
ATTR_NONNULL();
|
||||
/* Create FileReader from applying Zstd decompression on an underlying file. */
|
||||
FileReader *BLI_filereader_new_zstd(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
/* Create FileReader from applying Gzip decompression on an underlying file. */
|
||||
FileReader *BLI_filereader_new_gzip(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
@@ -31,6 +31,7 @@ set(INC
|
||||
|
||||
set(INC_SYS
|
||||
${ZLIB_INCLUDE_DIRS}
|
||||
${ZSTD_INCLUDE_DIRS}
|
||||
${FREETYPE_INCLUDE_DIRS}
|
||||
${GMP_INCLUDE_DIRS}
|
||||
)
|
||||
@@ -75,6 +76,10 @@ set(SRC
|
||||
intern/endian_switch.c
|
||||
intern/expr_pylike_eval.c
|
||||
intern/fileops.c
|
||||
intern/filereader_file.c
|
||||
intern/filereader_gzip.c
|
||||
intern/filereader_memory.c
|
||||
intern/filereader_zstd.c
|
||||
intern/fnmatch.c
|
||||
intern/freetypefont.c
|
||||
intern/gsqueue.c
|
||||
@@ -194,6 +199,7 @@ set(SRC
|
||||
BLI_enumerable_thread_specific.hh
|
||||
BLI_expr_pylike_eval.h
|
||||
BLI_fileops.h
|
||||
BLI_filereader.h
|
||||
BLI_fileops_types.h
|
||||
BLI_float2.hh
|
||||
BLI_float3.hh
|
||||
@@ -323,6 +329,7 @@ set(LIB
|
||||
|
||||
${FREETYPE_LIBRARY}
|
||||
${ZLIB_LIBRARIES}
|
||||
${ZSTD_LIBRARIES}
|
||||
)
|
||||
|
||||
if(WITH_MEM_VALGRIND)
|
||||
|
@@ -31,6 +31,7 @@
|
||||
#include <errno.h>
|
||||
|
||||
#include "zlib.h"
|
||||
#include "zstd.h"
|
||||
|
||||
#ifdef WIN32
|
||||
# include "BLI_fileops_types.h"
|
||||
@@ -61,199 +62,123 @@
|
||||
#include "BLI_sys_types.h" /* for intptr_t support */
|
||||
#include "BLI_utildefines.h"
|
||||
|
||||
#if 0 /* UNUSED */
|
||||
/* gzip the file in from and write it to "to".
|
||||
* return -1 if zlib fails, -2 if the originating file does not exist
|
||||
* NOTE: will remove the "from" file
|
||||
*/
|
||||
int BLI_file_gzip(const char *from, const char *to)
|
||||
size_t BLI_file_zstd_from_mem_at_pos(
|
||||
void *buf, size_t len, FILE *file, size_t file_offset, int compression_level)
|
||||
{
|
||||
char buffer[10240];
|
||||
int file;
|
||||
int readsize = 0;
|
||||
int rval = 0, err;
|
||||
gzFile gzfile;
|
||||
fseek(file, file_offset, SEEK_SET);
|
||||
|
||||
/* level 1 is very close to 3 (the default) in terms of file size,
|
||||
* but about twice as fast, best use for speedy saving - campbell */
|
||||
gzfile = BLI_gzopen(to, "wb1");
|
||||
if (gzfile == NULL) {
|
||||
return -1;
|
||||
}
|
||||
file = BLI_open(from, O_BINARY | O_RDONLY, 0);
|
||||
if (file == -1) {
|
||||
return -2;
|
||||
}
|
||||
ZSTD_CCtx *ctx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
|
||||
|
||||
while (1) {
|
||||
readsize = read(file, buffer, sizeof(buffer));
|
||||
ZSTD_inBuffer input = {buf, len, 0};
|
||||
|
||||
if (readsize < 0) {
|
||||
rval = -2; /* error happened in reading */
|
||||
fprintf(stderr, "Error reading file %s: %s.\n", from, strerror(errno));
|
||||
size_t out_len = ZSTD_CStreamOutSize();
|
||||
void *out_buf = MEM_mallocN(out_len, __func__);
|
||||
size_t total_written = 0;
|
||||
|
||||
/* Compress block and write it out until the input has been consumed. */
|
||||
while (input.pos < input.size) {
|
||||
ZSTD_outBuffer output = {out_buf, out_len, 0};
|
||||
size_t ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_continue);
|
||||
if (ZSTD_isError(ret)) {
|
||||
break;
|
||||
}
|
||||
else if (readsize == 0) {
|
||||
break; /* done reading */
|
||||
}
|
||||
|
||||
if (gzwrite(gzfile, buffer, readsize) <= 0) {
|
||||
rval = -1; /* error happened in writing */
|
||||
fprintf(stderr, "Error writing gz file %s: %s.\n", to, gzerror(gzfile, &err));
|
||||
if (fwrite(out_buf, 1, output.pos, file) != output.pos) {
|
||||
break;
|
||||
}
|
||||
total_written += output.pos;
|
||||
}
|
||||
|
||||
gzclose(gzfile);
|
||||
close(file);
|
||||
|
||||
return rval;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* gzip the file in from_file and write it to memory to_mem, at most size bytes.
|
||||
* return the unzipped size
|
||||
*/
|
||||
char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size)
|
||||
{
|
||||
gzFile gzfile;
|
||||
int readsize, size, alloc_size = 0;
|
||||
char *mem = NULL;
|
||||
const int chunk_size = 512 * 1024;
|
||||
|
||||
size = 0;
|
||||
|
||||
gzfile = BLI_gzopen(from_file, "rb");
|
||||
for (;;) {
|
||||
if (mem == NULL) {
|
||||
mem = MEM_callocN(chunk_size, "BLI_ungzip_to_mem");
|
||||
alloc_size = chunk_size;
|
||||
}
|
||||
else {
|
||||
mem = MEM_reallocN(mem, size + chunk_size);
|
||||
alloc_size += chunk_size;
|
||||
}
|
||||
|
||||
readsize = gzread(gzfile, mem + size, chunk_size);
|
||||
if (readsize > 0) {
|
||||
size += readsize;
|
||||
}
|
||||
else {
|
||||
/* Finalize the Zstd frame. */
|
||||
size_t ret = 1;
|
||||
while (ret != 0) {
|
||||
ZSTD_outBuffer output = {out_buf, out_len, 0};
|
||||
ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_end);
|
||||
if (ZSTD_isError(ret)) {
|
||||
break;
|
||||
}
|
||||
if (fwrite(out_buf, 1, output.pos, file) != output.pos) {
|
||||
break;
|
||||
}
|
||||
total_written += output.pos;
|
||||
}
|
||||
|
||||
gzclose(gzfile);
|
||||
MEM_freeN(out_buf);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
|
||||
if (size == 0) {
|
||||
MEM_freeN(mem);
|
||||
mem = NULL;
|
||||
}
|
||||
else if (alloc_size != size) {
|
||||
mem = MEM_reallocN(mem, size);
|
||||
}
|
||||
|
||||
*r_size = size;
|
||||
|
||||
return mem;
|
||||
return ZSTD_isError(ret) ? 0 : total_written;
|
||||
}
|
||||
|
||||
#define CHUNK (256 * 1024)
|
||||
|
||||
/* gzip byte array from memory and write it to file at certain position.
|
||||
* return size of gzip stream.
|
||||
*/
|
||||
size_t BLI_gzip_mem_to_file_at_pos(
|
||||
void *buf, size_t len, FILE *file, size_t gz_stream_offset, int compression_level)
|
||||
size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset)
|
||||
{
|
||||
int ret, flush;
|
||||
unsigned have;
|
||||
z_stream strm;
|
||||
unsigned char out[CHUNK];
|
||||
fseek(file, file_offset, SEEK_SET);
|
||||
|
||||
BLI_fseek(file, gz_stream_offset, 0);
|
||||
ZSTD_DCtx *ctx = ZSTD_createDCtx();
|
||||
|
||||
strm.zalloc = Z_NULL;
|
||||
strm.zfree = Z_NULL;
|
||||
strm.opaque = Z_NULL;
|
||||
ret = deflateInit(&strm, compression_level);
|
||||
if (ret != Z_OK) {
|
||||
return 0;
|
||||
}
|
||||
size_t in_len = ZSTD_DStreamInSize();
|
||||
void *in_buf = MEM_mallocN(in_len, __func__);
|
||||
ZSTD_inBuffer input = {in_buf, in_len, 0};
|
||||
|
||||
strm.avail_in = len;
|
||||
strm.next_in = (Bytef *)buf;
|
||||
flush = Z_FINISH;
|
||||
ZSTD_outBuffer output = {buf, len, 0};
|
||||
|
||||
do {
|
||||
strm.avail_out = CHUNK;
|
||||
strm.next_out = out;
|
||||
ret = deflate(&strm, flush);
|
||||
if (ret == Z_STREAM_ERROR) {
|
||||
return 0;
|
||||
}
|
||||
have = CHUNK - strm.avail_out;
|
||||
if (fwrite(out, 1, have, file) != have || ferror(file)) {
|
||||
deflateEnd(&strm);
|
||||
return 0;
|
||||
}
|
||||
} while (strm.avail_out == 0);
|
||||
|
||||
if (strm.avail_in != 0 || ret != Z_STREAM_END) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
deflateEnd(&strm);
|
||||
return (size_t)strm.total_out;
|
||||
}
|
||||
|
||||
/* read and decompress gzip stream from file at certain position to buffer.
|
||||
* return size of decompressed data.
|
||||
*/
|
||||
size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
|
||||
{
|
||||
int ret;
|
||||
z_stream strm;
|
||||
size_t chunk = 256 * 1024;
|
||||
unsigned char in[CHUNK];
|
||||
|
||||
BLI_fseek(file, gz_stream_offset, 0);
|
||||
|
||||
strm.zalloc = Z_NULL;
|
||||
strm.zfree = Z_NULL;
|
||||
strm.opaque = Z_NULL;
|
||||
strm.avail_in = 0;
|
||||
strm.next_in = Z_NULL;
|
||||
ret = inflateInit(&strm);
|
||||
if (ret != Z_OK) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
do {
|
||||
strm.avail_in = fread(in, 1, chunk, file);
|
||||
strm.next_in = in;
|
||||
if (ferror(file)) {
|
||||
inflateEnd(&strm);
|
||||
return 0;
|
||||
size_t ret = 0;
|
||||
/* Read and decompress chunks of input data until we have enough output. */
|
||||
while (output.pos < output.size && !ZSTD_isError(ret)) {
|
||||
input.size = fread(in_buf, 1, in_len, file);
|
||||
if (input.size == 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
do {
|
||||
strm.avail_out = len;
|
||||
strm.next_out = (Bytef *)buf + strm.total_out;
|
||||
/* Consume input data until we run out or have enough output. */
|
||||
input.pos = 0;
|
||||
while (input.pos < input.size && output.pos < output.size) {
|
||||
ret = ZSTD_decompressStream(ctx, &output, &input);
|
||||
|
||||
ret = inflate(&strm, Z_NO_FLUSH);
|
||||
if (ret == Z_STREAM_ERROR) {
|
||||
return 0;
|
||||
if (ZSTD_isError(ret)) {
|
||||
break;
|
||||
}
|
||||
} while (strm.avail_out == 0);
|
||||
}
|
||||
}
|
||||
|
||||
} while (ret != Z_STREAM_END);
|
||||
MEM_freeN(in_buf);
|
||||
ZSTD_freeDCtx(ctx);
|
||||
|
||||
inflateEnd(&strm);
|
||||
return (size_t)strm.total_out;
|
||||
return ZSTD_isError(ret) ? 0 : output.pos;
|
||||
}
|
||||
|
||||
#undef CHUNK
|
||||
bool BLI_file_magic_is_gzip(const char header[4])
|
||||
{
|
||||
/* GZIP itself starts with the magic bytes 0x1f 0x8b.
|
||||
* The third byte indicates the compression method, which is 0x08 for DEFLATE. */
|
||||
return header[0] == 0x1f && header[1] == 0x8b && header[2] == 0x08;
|
||||
}
|
||||
|
||||
bool BLI_file_magic_is_zstd(const char header[4])
|
||||
{
|
||||
/* ZSTD files consist of concatenated frames, each either a Zstd frame or a skippable frame.
|
||||
* Both types of frames start with a magic number: 0xFD2FB528 for Zstd frames and 0x184D2A5*
|
||||
* for skippable frames, with the * being anything from 0 to F.
|
||||
*
|
||||
* To check whether a file is Zstd-compressed, we just check whether the first frame matches
|
||||
* either. Seeking through the file until a Zstd frame is found would make things more
|
||||
* complicated and the probability of a false positive is rather low anyways.
|
||||
*
|
||||
* Note that LZ4 uses a compatible format, so even though its compressed frames have a
|
||||
* different magic number, a valid LZ4 file might also start with a skippable frame matching
|
||||
* the second check here.
|
||||
*
|
||||
* For more details, see https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
|
||||
*/
|
||||
|
||||
uint32_t magic = *((uint32_t *)header);
|
||||
if (magic == 0xFD2FB528) {
|
||||
return true;
|
||||
}
|
||||
if ((magic >> 4) == 0x184D2A5) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the file with the specified name can be written.
|
||||
|
80
source/blender/blenlib/intern/filereader_file.c
Normal file
80
source/blender/blenlib/intern/filereader_file.c
Normal file
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
* The Original Code is Copyright (C) 2004-2021 Blender Foundation
|
||||
* All rights reserved.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup bli
|
||||
*/
|
||||
|
||||
#ifndef WIN32
|
||||
# include <unistd.h> /* for read close */
|
||||
#else
|
||||
# include "BLI_winstuff.h"
|
||||
# include "winsock2.h"
|
||||
# include <io.h> /* for open close read */
|
||||
#endif
|
||||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_filereader.h"
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
typedef struct {
|
||||
FileReader reader;
|
||||
|
||||
int filedes;
|
||||
} RawFileReader;
|
||||
|
||||
static ssize_t file_read(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
RawFileReader *rawfile = (RawFileReader *)reader;
|
||||
ssize_t readsize = read(rawfile->filedes, buffer, size);
|
||||
|
||||
if (readsize >= 0) {
|
||||
rawfile->reader.offset += readsize;
|
||||
}
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static off64_t file_seek(FileReader *reader, off64_t offset, int whence)
|
||||
{
|
||||
RawFileReader *rawfile = (RawFileReader *)reader;
|
||||
rawfile->reader.offset = BLI_lseek(rawfile->filedes, offset, whence);
|
||||
return rawfile->reader.offset;
|
||||
}
|
||||
|
||||
static void file_close(FileReader *reader)
|
||||
{
|
||||
RawFileReader *rawfile = (RawFileReader *)reader;
|
||||
close(rawfile->filedes);
|
||||
MEM_freeN(rawfile);
|
||||
}
|
||||
|
||||
FileReader *BLI_filereader_new_file(int filedes)
|
||||
{
|
||||
RawFileReader *rawfile = MEM_callocN(sizeof(RawFileReader), __func__);
|
||||
|
||||
rawfile->filedes = filedes;
|
||||
|
||||
rawfile->reader.read = file_read;
|
||||
rawfile->reader.seek = file_seek;
|
||||
rawfile->reader.close = file_close;
|
||||
|
||||
return (FileReader *)rawfile;
|
||||
}
|
108
source/blender/blenlib/intern/filereader_gzip.c
Normal file
108
source/blender/blenlib/intern/filereader_gzip.c
Normal file
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
* The Original Code is Copyright (C) 2004-2021 Blender Foundation
|
||||
* All rights reserved.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup bli
|
||||
*/
|
||||
|
||||
#include <zlib.h>
|
||||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_filereader.h"
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
typedef struct {
|
||||
FileReader reader;
|
||||
|
||||
FileReader *base;
|
||||
|
||||
z_stream strm;
|
||||
|
||||
void *in_buf;
|
||||
size_t in_size;
|
||||
} GzipReader;
|
||||
|
||||
static ssize_t gzip_read(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
GzipReader *gzip = (GzipReader *)reader;
|
||||
|
||||
gzip->strm.avail_out = size;
|
||||
gzip->strm.next_out = buffer;
|
||||
|
||||
while (gzip->strm.avail_out > 0) {
|
||||
if (gzip->strm.avail_in == 0) {
|
||||
/* Ran out of buffered input data, read some more. */
|
||||
size_t readsize = gzip->base->read(gzip->base, gzip->in_buf, gzip->in_size);
|
||||
|
||||
if (readsize > 0) {
|
||||
/* We got some data, so mark the buffer as refilled. */
|
||||
gzip->strm.avail_in = readsize;
|
||||
gzip->strm.next_in = gzip->in_buf;
|
||||
}
|
||||
else {
|
||||
/* The underlying file is EOF, so return as much as we can. */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int ret = inflate(&gzip->strm, Z_NO_FLUSH);
|
||||
|
||||
if (ret != Z_OK && ret != Z_BUF_ERROR) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ssize_t read_len = size - gzip->strm.avail_out;
|
||||
gzip->reader.offset += read_len;
|
||||
return read_len;
|
||||
}
|
||||
|
||||
static void gzip_close(FileReader *reader)
|
||||
{
|
||||
GzipReader *gzip = (GzipReader *)reader;
|
||||
|
||||
if (inflateEnd(&gzip->strm) != Z_OK) {
|
||||
printf("close gzip stream error\n");
|
||||
}
|
||||
MEM_freeN((void *)gzip->in_buf);
|
||||
|
||||
gzip->base->close(gzip->base);
|
||||
MEM_freeN(gzip);
|
||||
}
|
||||
|
||||
FileReader *BLI_filereader_new_gzip(FileReader *base)
|
||||
{
|
||||
GzipReader *gzip = MEM_callocN(sizeof(GzipReader), __func__);
|
||||
gzip->base = base;
|
||||
|
||||
if (inflateInit2(&gzip->strm, 16 + MAX_WBITS) != Z_OK) {
|
||||
MEM_freeN(gzip);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
gzip->in_size = 256 * 2014;
|
||||
gzip->in_buf = MEM_mallocN(gzip->in_size, "gzip in buf");
|
||||
|
||||
gzip->reader.read = gzip_read;
|
||||
gzip->reader.seek = NULL;
|
||||
gzip->reader.close = gzip_close;
|
||||
|
||||
return (FileReader *)gzip;
|
||||
}
|
145
source/blender/blenlib/intern/filereader_memory.c
Normal file
145
source/blender/blenlib/intern/filereader_memory.c
Normal file
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
* The Original Code is Copyright (C) 2004-2021 Blender Foundation
|
||||
* All rights reserved.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup bli
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_filereader.h"
|
||||
#include "BLI_mmap.h"
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
/* This file implements both memory-backed and memory-mapped-file-backed reading. */
|
||||
typedef struct {
|
||||
FileReader reader;
|
||||
|
||||
const char *data;
|
||||
BLI_mmap_file *mmap;
|
||||
size_t length;
|
||||
} MemoryReader;
|
||||
|
||||
static ssize_t memory_read_raw(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
MemoryReader *mem = (MemoryReader *)reader;
|
||||
|
||||
/* Don't read more bytes than there are available in the buffer. */
|
||||
size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
|
||||
|
||||
memcpy(buffer, mem->data + mem->reader.offset, readsize);
|
||||
mem->reader.offset += readsize;
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static off64_t memory_seek(FileReader *reader, off64_t offset, int whence)
|
||||
{
|
||||
MemoryReader *mem = (MemoryReader *)reader;
|
||||
|
||||
off64_t new_pos;
|
||||
if (whence == SEEK_CUR) {
|
||||
new_pos = mem->reader.offset + offset;
|
||||
}
|
||||
else if (whence == SEEK_SET) {
|
||||
new_pos = offset;
|
||||
}
|
||||
else if (whence == SEEK_END) {
|
||||
new_pos = mem->length + offset;
|
||||
}
|
||||
else {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (new_pos < 0 || new_pos > mem->length) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
mem->reader.offset = new_pos;
|
||||
return mem->reader.offset;
|
||||
}
|
||||
|
||||
static void memory_close_raw(FileReader *reader)
|
||||
{
|
||||
MEM_freeN(reader);
|
||||
}
|
||||
|
||||
FileReader *BLI_filereader_new_memory(const void *data, size_t len)
|
||||
{
|
||||
MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
|
||||
|
||||
mem->data = (const char *)data;
|
||||
mem->length = len;
|
||||
|
||||
mem->reader.read = memory_read_raw;
|
||||
mem->reader.seek = memory_seek;
|
||||
mem->reader.close = memory_close_raw;
|
||||
|
||||
return (FileReader *)mem;
|
||||
}
|
||||
|
||||
/* Memory-mapped file reading.
|
||||
* By using `mmap()`, we can map a file so that it can be treated like normal memory,
|
||||
* meaning that we can just read from it with `memcpy()` etc.
|
||||
* This avoids system call overhead and can significantly speed up file loading.
|
||||
*/
|
||||
|
||||
static ssize_t memory_read_mmap(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
MemoryReader *mem = (MemoryReader *)reader;
|
||||
|
||||
/* Don't read more bytes than there are available in the buffer. */
|
||||
size_t readsize = MIN2(size, (size_t)(mem->length - mem->reader.offset));
|
||||
|
||||
if (!BLI_mmap_read(mem->mmap, buffer, mem->reader.offset, readsize)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
mem->reader.offset += readsize;
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static void memory_close_mmap(FileReader *reader)
|
||||
{
|
||||
MemoryReader *mem = (MemoryReader *)reader;
|
||||
BLI_mmap_free(mem->mmap);
|
||||
MEM_freeN(mem);
|
||||
}
|
||||
|
||||
FileReader *BLI_filereader_new_mmap(int filedes)
|
||||
{
|
||||
BLI_mmap_file *mmap = BLI_mmap_open(filedes);
|
||||
if (mmap == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
MemoryReader *mem = MEM_callocN(sizeof(MemoryReader), __func__);
|
||||
|
||||
mem->mmap = mmap;
|
||||
mem->length = BLI_lseek(filedes, 0, SEEK_END);
|
||||
|
||||
mem->reader.read = memory_read_mmap;
|
||||
mem->reader.seek = memory_seek;
|
||||
mem->reader.close = memory_close_mmap;
|
||||
|
||||
return (FileReader *)mem;
|
||||
}
|
335
source/blender/blenlib/intern/filereader_zstd.c
Normal file
335
source/blender/blenlib/intern/filereader_zstd.c
Normal file
@@ -0,0 +1,335 @@
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
* The Original Code is Copyright (C) 2021 Blender Foundation
|
||||
* All rights reserved.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup bli
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
#include <zstd.h>
|
||||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_endian_switch.h"
|
||||
#include "BLI_filereader.h"
|
||||
#include "BLI_math_base.h"
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
typedef struct {
|
||||
FileReader reader;
|
||||
|
||||
FileReader *base;
|
||||
|
||||
ZSTD_DCtx *ctx;
|
||||
ZSTD_inBuffer in_buf;
|
||||
size_t in_buf_max_size;
|
||||
|
||||
struct {
|
||||
int num_frames;
|
||||
size_t *compressed_ofs;
|
||||
size_t *uncompressed_ofs;
|
||||
|
||||
char *cached_content;
|
||||
int cached_frame;
|
||||
} seek;
|
||||
} ZstdReader;
|
||||
|
||||
static bool zstd_read_u32(FileReader *base, uint32_t *val)
|
||||
{
|
||||
if (base->read(base, val, sizeof(uint32_t)) != sizeof(uint32_t)) {
|
||||
return false;
|
||||
}
|
||||
#ifdef __BIG_ENDIAN__
|
||||
BLI_endian_switch_uint32(val);
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool zstd_read_seek_table(ZstdReader *zstd)
|
||||
{
|
||||
FileReader *base = zstd->base;
|
||||
|
||||
/* The seek table frame is at the end of the file, so seek there
|
||||
* and verify that there is enough data. */
|
||||
if (base->seek(base, -4, SEEK_END) < 13) {
|
||||
return false;
|
||||
}
|
||||
uint32_t magic;
|
||||
if (!zstd_read_u32(base, &magic) || magic != 0x8F92EAB1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint8_t flags;
|
||||
if (base->seek(base, -5, SEEK_END) < 0 || base->read(base, &flags, 1) != 1) {
|
||||
return false;
|
||||
}
|
||||
/* Bit 7 indicates checksums. Bits 5 and 6 must be zero. */
|
||||
bool has_checksums = (flags & 0x80);
|
||||
if (flags & 0x60) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t num_frames;
|
||||
if (base->seek(base, -9, SEEK_END) < 0 || !zstd_read_u32(base, &num_frames)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Each frame has either 2 or 3 uint32_t, and after that we have
|
||||
* num_frames, flags and magic for another 9 bytes. */
|
||||
uint32_t expected_frame_length = num_frames * (has_checksums ? 12 : 8) + 9;
|
||||
/* The frame starts with another magic number and its length, but these
|
||||
* two fields are not included when counting length. */
|
||||
off64_t frame_start_ofs = 8 + expected_frame_length;
|
||||
/* Sanity check: Before the start of the seek table frame,
|
||||
* there must be num_frames frames, each of which at least 8 bytes long. */
|
||||
off64_t seek_frame_start = base->seek(base, -frame_start_ofs, SEEK_END);
|
||||
if (seek_frame_start < num_frames * 8) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!zstd_read_u32(base, &magic) || magic != 0x184D2A5E) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t frame_length;
|
||||
if (!zstd_read_u32(base, &frame_length) || frame_length != expected_frame_length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
zstd->seek.num_frames = num_frames;
|
||||
zstd->seek.compressed_ofs = MEM_malloc_arrayN(num_frames + 1, sizeof(size_t), __func__);
|
||||
zstd->seek.uncompressed_ofs = MEM_malloc_arrayN(num_frames + 1, sizeof(size_t), __func__);
|
||||
|
||||
size_t compressed_ofs = 0;
|
||||
size_t uncompressed_ofs = 0;
|
||||
for (int i = 0; i < num_frames; i++) {
|
||||
uint32_t compressed_size, uncompressed_size;
|
||||
if (!zstd_read_u32(base, &compressed_size) || !zstd_read_u32(base, &uncompressed_size)) {
|
||||
break;
|
||||
}
|
||||
if (has_checksums && base->seek(base, 4, SEEK_CUR) < 0) {
|
||||
break;
|
||||
}
|
||||
zstd->seek.compressed_ofs[i] = compressed_ofs;
|
||||
zstd->seek.uncompressed_ofs[i] = uncompressed_ofs;
|
||||
compressed_ofs += compressed_size;
|
||||
uncompressed_ofs += uncompressed_size;
|
||||
}
|
||||
zstd->seek.compressed_ofs[num_frames] = compressed_ofs;
|
||||
zstd->seek.uncompressed_ofs[num_frames] = uncompressed_ofs;
|
||||
|
||||
/* Seek to the end of the previous frame for the following BHead frame detection. */
|
||||
if (seek_frame_start != compressed_ofs || base->seek(base, seek_frame_start, SEEK_SET) < 0) {
|
||||
MEM_freeN(zstd->seek.compressed_ofs);
|
||||
MEM_freeN(zstd->seek.uncompressed_ofs);
|
||||
memset(&zstd->seek, 0, sizeof(zstd->seek));
|
||||
return false;
|
||||
}
|
||||
|
||||
zstd->seek.cached_frame = -1;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Find out which frame contains the given position in the uncompressed stream.
|
||||
* Basically just bisection. */
|
||||
static int zstd_frame_from_pos(ZstdReader *zstd, size_t pos)
|
||||
{
|
||||
int low = 0, high = zstd->seek.num_frames;
|
||||
|
||||
if (pos >= zstd->seek.uncompressed_ofs[zstd->seek.num_frames]) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
while (low + 1 < high) {
|
||||
int mid = low + ((high - low) >> 1);
|
||||
if (zstd->seek.uncompressed_ofs[mid] <= pos) {
|
||||
low = mid;
|
||||
}
|
||||
else {
|
||||
high = mid;
|
||||
}
|
||||
}
|
||||
|
||||
return low;
|
||||
}
|
||||
|
||||
/* Ensure that the currently loaded frame is the correct one. */
|
||||
static const char *zstd_ensure_cache(ZstdReader *zstd, int frame)
|
||||
{
|
||||
if (zstd->seek.cached_frame == frame) {
|
||||
/* Cached frame matches, so just return it. */
|
||||
return zstd->seek.cached_content;
|
||||
}
|
||||
|
||||
/* Cached frame doesn't match, so discard it and cache the wanted one onstead. */
|
||||
MEM_SAFE_FREE(zstd->seek.cached_content);
|
||||
|
||||
size_t compressed_size = zstd->seek.compressed_ofs[frame + 1] - zstd->seek.compressed_ofs[frame];
|
||||
size_t uncompressed_size = zstd->seek.uncompressed_ofs[frame + 1] -
|
||||
zstd->seek.uncompressed_ofs[frame];
|
||||
|
||||
char *uncompressed_data = MEM_mallocN(uncompressed_size, __func__);
|
||||
char *compressed_data = MEM_mallocN(compressed_size, __func__);
|
||||
if (zstd->base->seek(zstd->base, zstd->seek.compressed_ofs[frame], SEEK_SET) < 0 ||
|
||||
zstd->base->read(zstd->base, compressed_data, compressed_size) < compressed_size) {
|
||||
MEM_freeN(compressed_data);
|
||||
MEM_freeN(uncompressed_data);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t res = ZSTD_decompressDCtx(
|
||||
zstd->ctx, uncompressed_data, uncompressed_size, compressed_data, compressed_size);
|
||||
MEM_freeN(compressed_data);
|
||||
if (ZSTD_isError(res) || res < uncompressed_size) {
|
||||
MEM_freeN(uncompressed_data);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
zstd->seek.cached_frame = frame;
|
||||
zstd->seek.cached_content = uncompressed_data;
|
||||
return uncompressed_data;
|
||||
}
|
||||
|
||||
static ssize_t zstd_read_seekable(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
ZstdReader *zstd = (ZstdReader *)reader;
|
||||
|
||||
size_t end_offset = zstd->reader.offset + size, read_len = 0;
|
||||
while (zstd->reader.offset < end_offset) {
|
||||
int frame = zstd_frame_from_pos(zstd, zstd->reader.offset);
|
||||
if (frame < 0) {
|
||||
/* EOF is reached, so return as much as we can. */
|
||||
break;
|
||||
}
|
||||
|
||||
const char *framedata = zstd_ensure_cache(zstd, frame);
|
||||
if (framedata == NULL) {
|
||||
/* Error while reading the frame, so return as much as we can. */
|
||||
break;
|
||||
}
|
||||
|
||||
size_t frame_end_offset = min_zz(zstd->seek.uncompressed_ofs[frame + 1], end_offset);
|
||||
size_t frame_read_len = frame_end_offset - zstd->reader.offset;
|
||||
|
||||
size_t offset_in_frame = zstd->reader.offset - zstd->seek.uncompressed_ofs[frame];
|
||||
memcpy((char *)buffer + read_len, framedata + offset_in_frame, frame_read_len);
|
||||
read_len += frame_read_len;
|
||||
zstd->reader.offset = frame_end_offset;
|
||||
}
|
||||
|
||||
return read_len;
|
||||
}
|
||||
|
||||
static off64_t zstd_seek(FileReader *reader, off64_t offset, int whence)
|
||||
{
|
||||
ZstdReader *zstd = (ZstdReader *)reader;
|
||||
off64_t new_pos;
|
||||
if (whence == SEEK_SET) {
|
||||
new_pos = offset;
|
||||
}
|
||||
else if (whence == SEEK_END) {
|
||||
new_pos = zstd->seek.uncompressed_ofs[zstd->seek.num_frames] + offset;
|
||||
}
|
||||
else {
|
||||
new_pos = zstd->reader.offset + offset;
|
||||
}
|
||||
|
||||
if (new_pos < 0 || new_pos > zstd->seek.uncompressed_ofs[zstd->seek.num_frames]) {
|
||||
return -1;
|
||||
}
|
||||
zstd->reader.offset = new_pos;
|
||||
return zstd->reader.offset;
|
||||
}
|
||||
|
||||
static ssize_t zstd_read(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
ZstdReader *zstd = (ZstdReader *)reader;
|
||||
ZSTD_outBuffer output = {buffer, size, 0};
|
||||
|
||||
while (output.pos < output.size) {
|
||||
if (zstd->in_buf.pos == zstd->in_buf.size) {
|
||||
/* Ran out of buffered input data, read some more. */
|
||||
zstd->in_buf.pos = 0;
|
||||
ssize_t readsize = zstd->base->read(
|
||||
zstd->base, (char *)zstd->in_buf.src, zstd->in_buf_max_size);
|
||||
|
||||
if (readsize > 0) {
|
||||
/* We got some data, so mark the buffer as refilled. */
|
||||
zstd->in_buf.size = readsize;
|
||||
}
|
||||
else {
|
||||
/* The underlying file is EOF, so return as much as we can. */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ZSTD_isError(ZSTD_decompressStream(zstd->ctx, &output, &zstd->in_buf))) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
zstd->reader.offset += output.pos;
|
||||
return output.pos;
|
||||
}
|
||||
|
||||
static void zstd_close(FileReader *reader)
|
||||
{
|
||||
ZstdReader *zstd = (ZstdReader *)reader;
|
||||
|
||||
ZSTD_freeDCtx(zstd->ctx);
|
||||
if (zstd->reader.seek) {
|
||||
MEM_freeN(zstd->seek.uncompressed_ofs);
|
||||
MEM_freeN(zstd->seek.compressed_ofs);
|
||||
MEM_freeN(zstd->seek.cached_content);
|
||||
}
|
||||
else {
|
||||
MEM_freeN((void *)zstd->in_buf.src);
|
||||
}
|
||||
|
||||
zstd->base->close(zstd->base);
|
||||
MEM_freeN(zstd);
|
||||
}
|
||||
|
||||
FileReader *BLI_filereader_new_zstd(FileReader *base)
|
||||
{
|
||||
ZstdReader *zstd = MEM_callocN(sizeof(ZstdReader), __func__);
|
||||
|
||||
zstd->ctx = ZSTD_createDCtx();
|
||||
zstd->base = base;
|
||||
|
||||
if (zstd_read_seek_table(zstd)) {
|
||||
zstd->reader.read = zstd_read_seekable;
|
||||
zstd->reader.seek = zstd_seek;
|
||||
}
|
||||
else {
|
||||
zstd->reader.read = zstd_read;
|
||||
zstd->reader.seek = NULL;
|
||||
|
||||
zstd->in_buf_max_size = ZSTD_DStreamInSize();
|
||||
zstd->in_buf.src = MEM_mallocN(zstd->in_buf_max_size, "zstd in buf");
|
||||
zstd->in_buf.size = zstd->in_buf_max_size;
|
||||
/* This signals that the buffer has run out,
|
||||
* which will make the read function refill it on the first call. */
|
||||
zstd->in_buf.pos = zstd->in_buf_max_size;
|
||||
}
|
||||
zstd->reader.close = zstd_close;
|
||||
|
||||
return (FileReader *)zstd;
|
||||
}
|
@@ -24,6 +24,8 @@
|
||||
* \ingroup blenloader
|
||||
*/
|
||||
|
||||
#include "BLI_filereader.h"
|
||||
|
||||
struct GHash;
|
||||
struct Scene;
|
||||
|
||||
@@ -65,6 +67,16 @@ typedef struct MemFileUndoData {
|
||||
size_t undo_size;
|
||||
} MemFileUndoData;
|
||||
|
||||
/* FileReader-compatible wrapper for reading MemFiles */
|
||||
typedef struct {
|
||||
FileReader reader;
|
||||
|
||||
MemFile *memfile;
|
||||
int undo_direction;
|
||||
|
||||
bool memchunk_identical;
|
||||
} UndoReader;
|
||||
|
||||
/* actually only used writefile.c */
|
||||
|
||||
void BLO_memfile_write_init(MemFileWriteData *mem_data,
|
||||
@@ -84,3 +96,5 @@ extern struct Main *BLO_memfile_main_get(struct MemFile *memfile,
|
||||
struct Main *bmain,
|
||||
struct Scene **r_scene);
|
||||
extern bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename);
|
||||
|
||||
FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction);
|
||||
|
@@ -42,7 +42,7 @@ set(INC
|
||||
)
|
||||
|
||||
set(INC_SYS
|
||||
${ZLIB_INCLUDE_DIRS}
|
||||
${ZSTD_INCLUDE_DIRS}
|
||||
)
|
||||
|
||||
set(SRC
|
||||
|
@@ -21,8 +21,6 @@
|
||||
* \ingroup blenloader
|
||||
*/
|
||||
|
||||
#include "zlib.h"
|
||||
|
||||
#include <ctype.h> /* for isdigit. */
|
||||
#include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */
|
||||
#include <limits.h>
|
||||
@@ -71,7 +69,6 @@
|
||||
#include "BLI_math.h"
|
||||
#include "BLI_memarena.h"
|
||||
#include "BLI_mempool.h"
|
||||
#include "BLI_mmap.h"
|
||||
#include "BLI_threads.h"
|
||||
|
||||
#include "PIL_time.h"
|
||||
@@ -788,7 +785,7 @@ static BHeadN *get_bhead(FileData *fd)
|
||||
*/
|
||||
if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) {
|
||||
bhead4.code = DATA;
|
||||
readsize = fd->read(fd, &bhead4, sizeof(bhead4), NULL);
|
||||
readsize = fd->file->read(fd->file, &bhead4, sizeof(bhead4));
|
||||
|
||||
if (readsize == sizeof(bhead4) || bhead4.code == ENDB) {
|
||||
if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
|
||||
@@ -811,7 +808,7 @@ static BHeadN *get_bhead(FileData *fd)
|
||||
}
|
||||
else {
|
||||
bhead8.code = DATA;
|
||||
readsize = fd->read(fd, &bhead8, sizeof(bhead8), NULL);
|
||||
readsize = fd->file->read(fd->file, &bhead8, sizeof(bhead8));
|
||||
|
||||
if (readsize == sizeof(bhead8) || bhead8.code == ENDB) {
|
||||
if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
|
||||
@@ -845,22 +842,22 @@ static BHeadN *get_bhead(FileData *fd)
|
||||
/* pass */
|
||||
}
|
||||
#ifdef USE_BHEAD_READ_ON_DEMAND
|
||||
else if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
|
||||
else if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) {
|
||||
/* Delay reading bhead content. */
|
||||
new_bhead = MEM_mallocN(sizeof(BHeadN), "new_bhead");
|
||||
if (new_bhead) {
|
||||
new_bhead->next = new_bhead->prev = NULL;
|
||||
new_bhead->file_offset = fd->file_offset;
|
||||
new_bhead->file_offset = fd->file->offset;
|
||||
new_bhead->has_data = false;
|
||||
new_bhead->is_memchunk_identical = false;
|
||||
new_bhead->bhead = bhead;
|
||||
off64_t seek_new = fd->seek(fd, bhead.len, SEEK_CUR);
|
||||
off64_t seek_new = fd->file->seek(fd->file, bhead.len, SEEK_CUR);
|
||||
if (seek_new == -1) {
|
||||
fd->is_eof = true;
|
||||
MEM_freeN(new_bhead);
|
||||
new_bhead = NULL;
|
||||
}
|
||||
BLI_assert(fd->file_offset == seek_new);
|
||||
BLI_assert(fd->file->offset == seek_new);
|
||||
}
|
||||
else {
|
||||
fd->is_eof = true;
|
||||
@@ -878,14 +875,17 @@ static BHeadN *get_bhead(FileData *fd)
|
||||
new_bhead->is_memchunk_identical = false;
|
||||
new_bhead->bhead = bhead;
|
||||
|
||||
readsize = fd->read(
|
||||
fd, new_bhead + 1, (size_t)bhead.len, &new_bhead->is_memchunk_identical);
|
||||
readsize = fd->file->read(fd->file, new_bhead + 1, (size_t)bhead.len);
|
||||
|
||||
if (readsize != (ssize_t)bhead.len) {
|
||||
if (readsize != bhead.len) {
|
||||
fd->is_eof = true;
|
||||
MEM_freeN(new_bhead);
|
||||
new_bhead = NULL;
|
||||
}
|
||||
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
|
||||
}
|
||||
}
|
||||
else {
|
||||
fd->is_eof = true;
|
||||
@@ -964,17 +964,19 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf)
|
||||
bool success = true;
|
||||
BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock);
|
||||
BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0);
|
||||
off64_t offset_backup = fd->file_offset;
|
||||
if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) {
|
||||
off64_t offset_backup = fd->file->offset;
|
||||
if (UNLIKELY(fd->file->seek(fd->file, new_bhead->file_offset, SEEK_SET) == -1)) {
|
||||
success = false;
|
||||
}
|
||||
else {
|
||||
if (fd->read(fd, buf, (size_t)new_bhead->bhead.len, &new_bhead->is_memchunk_identical) !=
|
||||
(ssize_t)new_bhead->bhead.len) {
|
||||
if (fd->file->read(fd->file, buf, (size_t)new_bhead->bhead.len) != new_bhead->bhead.len) {
|
||||
success = false;
|
||||
}
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical;
|
||||
}
|
||||
}
|
||||
if (fd->seek(fd, offset_backup, SEEK_SET) == -1) {
|
||||
if (fd->file->seek(fd->file, offset_backup, SEEK_SET) == -1) {
|
||||
success = false;
|
||||
}
|
||||
return success;
|
||||
@@ -1017,7 +1019,7 @@ static void decode_blender_header(FileData *fd)
|
||||
ssize_t readsize;
|
||||
|
||||
/* read in the header data */
|
||||
readsize = fd->read(fd, header, sizeof(header), NULL);
|
||||
readsize = fd->file->read(fd->file, header, sizeof(header));
|
||||
|
||||
if (readsize == sizeof(header) && STREQLEN(header, "BLENDER", 7) && ELEM(header[7], '_', '-') &&
|
||||
ELEM(header[8], 'v', 'V') &&
|
||||
@@ -1147,210 +1149,12 @@ static int *read_file_thumbnail(FileData *fd)
|
||||
|
||||
/** \} */
|
||||
|
||||
/* -------------------------------------------------------------------- */
|
||||
/** \name File Data API
|
||||
* \{ */
|
||||
|
||||
/* Regular file reading. */
|
||||
|
||||
static ssize_t fd_read_data_from_file(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
ssize_t readsize = read(filedata->filedes, buffer, size);
|
||||
|
||||
if (readsize < 0) {
|
||||
readsize = EOF;
|
||||
}
|
||||
else {
|
||||
filedata->file_offset += readsize;
|
||||
}
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int whence)
|
||||
{
|
||||
filedata->file_offset = BLI_lseek(filedata->filedes, offset, whence);
|
||||
return filedata->file_offset;
|
||||
}
|
||||
|
||||
/* GZip file reading. */
|
||||
|
||||
static ssize_t fd_read_gzip_from_file(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
BLI_assert(size <= INT_MAX);
|
||||
|
||||
ssize_t readsize = gzread(filedata->gzfiledes, buffer, (uint)size);
|
||||
|
||||
if (readsize < 0) {
|
||||
readsize = EOF;
|
||||
}
|
||||
else {
|
||||
filedata->file_offset += readsize;
|
||||
}
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
/* Memory reading. */
|
||||
|
||||
static ssize_t fd_read_from_memory(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
/* don't read more bytes than there are available in the buffer */
|
||||
ssize_t readsize = (ssize_t)MIN2(size, filedata->buffersize - (size_t)filedata->file_offset);
|
||||
|
||||
memcpy(buffer, filedata->buffer + filedata->file_offset, (size_t)readsize);
|
||||
filedata->file_offset += readsize;
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
/* Memory-mapped file reading.
|
||||
* By using mmap(), we can map a file so that it can be treated like normal memory,
|
||||
* meaning that we can just read from it with memcpy() etc.
|
||||
* This avoids system call overhead and can significantly speed up file loading.
|
||||
*/
|
||||
|
||||
static ssize_t fd_read_from_mmap(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
/* don't read more bytes than there are available in the buffer */
|
||||
size_t readsize = MIN2(size, (size_t)(filedata->buffersize - filedata->file_offset));
|
||||
|
||||
if (!BLI_mmap_read(filedata->mmap_file, buffer, filedata->file_offset, readsize)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
filedata->file_offset += readsize;
|
||||
|
||||
return readsize;
|
||||
}
|
||||
|
||||
static off64_t fd_seek_from_mmap(FileData *filedata, off64_t offset, int whence)
|
||||
{
|
||||
off64_t new_pos;
|
||||
if (whence == SEEK_CUR) {
|
||||
new_pos = filedata->file_offset + offset;
|
||||
}
|
||||
else if (whence == SEEK_SET) {
|
||||
new_pos = offset;
|
||||
}
|
||||
else if (whence == SEEK_END) {
|
||||
new_pos = filedata->buffersize + offset;
|
||||
}
|
||||
else {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (new_pos < 0 || new_pos > filedata->buffersize) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
filedata->file_offset = new_pos;
|
||||
return filedata->file_offset;
|
||||
}
|
||||
|
||||
/* MemFile reading. */
|
||||
|
||||
static ssize_t fd_read_from_memfile(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *r_is_memchunck_identical)
|
||||
{
|
||||
static size_t seek = SIZE_MAX; /* the current position */
|
||||
static size_t offset = 0; /* size of previous chunks */
|
||||
static MemFileChunk *chunk = NULL;
|
||||
size_t chunkoffset, readsize, totread;
|
||||
|
||||
if (r_is_memchunck_identical != NULL) {
|
||||
*r_is_memchunck_identical = true;
|
||||
}
|
||||
|
||||
if (size == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (seek != (size_t)filedata->file_offset) {
|
||||
chunk = filedata->memfile->chunks.first;
|
||||
seek = 0;
|
||||
|
||||
while (chunk) {
|
||||
if (seek + chunk->size > (size_t)filedata->file_offset) {
|
||||
break;
|
||||
}
|
||||
seek += chunk->size;
|
||||
chunk = chunk->next;
|
||||
}
|
||||
offset = seek;
|
||||
seek = (size_t)filedata->file_offset;
|
||||
}
|
||||
|
||||
if (chunk) {
|
||||
totread = 0;
|
||||
|
||||
do {
|
||||
/* first check if it's on the end if current chunk */
|
||||
if (seek - offset == chunk->size) {
|
||||
offset += chunk->size;
|
||||
chunk = chunk->next;
|
||||
}
|
||||
|
||||
/* debug, should never happen */
|
||||
if (chunk == NULL) {
|
||||
CLOG_ERROR(&LOG, "Illegal read, got a NULL chunk");
|
||||
return 0;
|
||||
}
|
||||
|
||||
chunkoffset = seek - offset;
|
||||
readsize = size - totread;
|
||||
|
||||
/* data can be spread over multiple chunks, so clamp size
|
||||
* to within this chunk, and then it will read further in
|
||||
* the next chunk */
|
||||
if (chunkoffset + readsize > chunk->size) {
|
||||
readsize = chunk->size - chunkoffset;
|
||||
}
|
||||
|
||||
memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
|
||||
totread += readsize;
|
||||
filedata->file_offset += readsize;
|
||||
seek += readsize;
|
||||
if (r_is_memchunck_identical != NULL) {
|
||||
/* `is_identical` of current chunk represents whether it changed compared to previous undo
|
||||
* step. this is fine in redo case, but not in undo case, where we need an extra flag
|
||||
* defined when saving the next (future) step after the one we want to restore, as we are
|
||||
* supposed to 'come from' that future undo step, and not the one before current one. */
|
||||
*r_is_memchunck_identical &= filedata->undo_direction == STEP_REDO ?
|
||||
chunk->is_identical :
|
||||
chunk->is_identical_future;
|
||||
}
|
||||
} while (totread < size);
|
||||
|
||||
return (ssize_t)totread;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FileData *filedata_new(BlendFileReadReport *reports)
|
||||
{
|
||||
BLI_assert(reports != NULL);
|
||||
|
||||
FileData *fd = MEM_callocN(sizeof(FileData), "FileData");
|
||||
|
||||
fd->filedes = -1;
|
||||
fd->gzfiledes = NULL;
|
||||
|
||||
fd->memsdna = DNA_sdna_current_get();
|
||||
|
||||
fd->datamap = oldnewmap_new();
|
||||
@@ -1387,78 +1191,66 @@ static FileData *blo_decode_and_check(FileData *fd, ReportList *reports)
|
||||
|
||||
static FileData *blo_filedata_from_file_descriptor(const char *filepath,
|
||||
BlendFileReadReport *reports,
|
||||
int file)
|
||||
int filedes)
|
||||
{
|
||||
FileDataReadFn *read_fn = NULL;
|
||||
FileDataSeekFn *seek_fn = NULL; /* Optional. */
|
||||
size_t buffersize = 0;
|
||||
BLI_mmap_file *mmap_file = NULL;
|
||||
|
||||
gzFile gzfile = (gzFile)Z_NULL;
|
||||
|
||||
char header[7];
|
||||
FileReader *rawfile = BLI_filereader_new_file(filedes);
|
||||
FileReader *file = NULL;
|
||||
|
||||
/* Regular file. */
|
||||
errno = 0;
|
||||
if (read(file, header, sizeof(header)) != sizeof(header)) {
|
||||
/* If opening the file failed or we can't read the header, give up. */
|
||||
if (rawfile == NULL || rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) {
|
||||
BKE_reportf(reports->reports,
|
||||
RPT_WARNING,
|
||||
"Unable to read '%s': %s",
|
||||
filepath,
|
||||
errno ? strerror(errno) : TIP_("insufficient content"));
|
||||
if (rawfile) {
|
||||
rawfile->close(rawfile);
|
||||
}
|
||||
else {
|
||||
close(filedes);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Regular file. */
|
||||
/* Rewind the file after reading the header. */
|
||||
rawfile->seek(rawfile, 0, SEEK_SET);
|
||||
|
||||
/* Check if we have a regular file. */
|
||||
if (memcmp(header, "BLENDER", sizeof(header)) == 0) {
|
||||
read_fn = fd_read_data_from_file;
|
||||
seek_fn = fd_seek_data_from_file;
|
||||
|
||||
mmap_file = BLI_mmap_open(file);
|
||||
if (mmap_file != NULL) {
|
||||
read_fn = fd_read_from_mmap;
|
||||
seek_fn = fd_seek_from_mmap;
|
||||
buffersize = BLI_lseek(file, 0, SEEK_END);
|
||||
/* Try opening the file with memory-mapped IO. */
|
||||
file = BLI_filereader_new_mmap(filedes);
|
||||
if (file == NULL) {
|
||||
/* mmap failed, so just keep using rawfile. */
|
||||
file = rawfile;
|
||||
rawfile = NULL;
|
||||
}
|
||||
}
|
||||
else if (BLI_file_magic_is_gzip(header)) {
|
||||
file = BLI_filereader_new_gzip(rawfile);
|
||||
if (file != NULL) {
|
||||
rawfile = NULL; /* The Gzip FileReader takes ownership of `rawfile`. */
|
||||
}
|
||||
}
|
||||
else if (BLI_file_magic_is_zstd(header)) {
|
||||
file = BLI_filereader_new_zstd(rawfile);
|
||||
if (file != NULL) {
|
||||
rawfile = NULL; /* The Zstd FileReader takes ownership of `rawfile`. */
|
||||
}
|
||||
}
|
||||
|
||||
BLI_lseek(file, 0, SEEK_SET);
|
||||
|
||||
/* Gzip file. */
|
||||
errno = 0;
|
||||
if ((read_fn == NULL) &&
|
||||
/* Check header magic. */
|
||||
(header[0] == 0x1f && header[1] == 0x8b)) {
|
||||
gzfile = BLI_gzopen(filepath, "rb");
|
||||
if (gzfile == (gzFile)Z_NULL) {
|
||||
BKE_reportf(reports->reports,
|
||||
RPT_WARNING,
|
||||
"Unable to open '%s': %s",
|
||||
filepath,
|
||||
errno ? strerror(errno) : TIP_("unknown error reading file"));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* 'seek_fn' is too slow for gzip, don't set it. */
|
||||
read_fn = fd_read_gzip_from_file;
|
||||
/* Caller must close. */
|
||||
file = -1;
|
||||
/* Clean up `rawfile` if it wasn't taken over. */
|
||||
if (rawfile != NULL) {
|
||||
rawfile->close(rawfile);
|
||||
}
|
||||
|
||||
if (read_fn == NULL) {
|
||||
if (file == NULL) {
|
||||
BKE_reportf(reports->reports, RPT_WARNING, "Unrecognized file format '%s'", filepath);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
FileData *fd = filedata_new(reports);
|
||||
|
||||
fd->filedes = file;
|
||||
fd->gzfiledes = gzfile;
|
||||
|
||||
fd->read = read_fn;
|
||||
fd->seek = seek_fn;
|
||||
fd->mmap_file = mmap_file;
|
||||
fd->buffersize = buffersize;
|
||||
fd->file = file;
|
||||
|
||||
return fd;
|
||||
}
|
||||
@@ -1475,11 +1267,7 @@ static FileData *blo_filedata_from_file_open(const char *filepath, BlendFileRead
|
||||
errno ? strerror(errno) : TIP_("unknown error reading file"));
|
||||
return NULL;
|
||||
}
|
||||
FileData *fd = blo_filedata_from_file_descriptor(filepath, reports, file);
|
||||
if ((fd == NULL) || (fd->filedes == -1)) {
|
||||
close(file);
|
||||
}
|
||||
return fd;
|
||||
return blo_filedata_from_file_descriptor(filepath, reports, file);
|
||||
}
|
||||
|
||||
/* cannot be called with relative paths anymore! */
|
||||
@@ -1513,50 +1301,6 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static ssize_t fd_read_gzip_from_memory(FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *UNUSED(r_is_memchunck_identical))
|
||||
{
|
||||
int err;
|
||||
|
||||
filedata->strm.next_out = (Bytef *)buffer;
|
||||
filedata->strm.avail_out = (uint)size;
|
||||
|
||||
/* Inflate another chunk. */
|
||||
err = inflate(&filedata->strm, Z_SYNC_FLUSH);
|
||||
|
||||
if (err == Z_STREAM_END) {
|
||||
return 0;
|
||||
}
|
||||
if (err != Z_OK) {
|
||||
CLOG_ERROR(&LOG, "ZLib error (code %d)", err);
|
||||
return 0;
|
||||
}
|
||||
|
||||
filedata->file_offset += size;
|
||||
|
||||
return (ssize_t)size;
|
||||
}
|
||||
|
||||
static int fd_read_gzip_from_memory_init(FileData *fd)
|
||||
{
|
||||
|
||||
fd->strm.next_in = (Bytef *)fd->buffer;
|
||||
fd->strm.avail_in = fd->buffersize;
|
||||
fd->strm.total_out = 0;
|
||||
fd->strm.zalloc = Z_NULL;
|
||||
fd->strm.zfree = Z_NULL;
|
||||
|
||||
if (inflateInit2(&fd->strm, (16 + MAX_WBITS)) != Z_OK) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
fd->read = fd_read_gzip_from_memory;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadReport *reports)
|
||||
{
|
||||
if (!mem || memsize < SIZEOFBLENDERHEADER) {
|
||||
@@ -1565,24 +1309,24 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadRe
|
||||
return NULL;
|
||||
}
|
||||
|
||||
FileReader *mem_file = BLI_filereader_new_memory(mem, memsize);
|
||||
FileReader *file = mem_file;
|
||||
|
||||
if (BLI_file_magic_is_gzip(mem)) {
|
||||
file = BLI_filereader_new_gzip(mem_file);
|
||||
}
|
||||
else if (BLI_file_magic_is_zstd(mem)) {
|
||||
file = BLI_filereader_new_zstd(mem_file);
|
||||
}
|
||||
|
||||
if (file == NULL) {
|
||||
/* Compression initialization failed. */
|
||||
mem_file->close(mem_file);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
FileData *fd = filedata_new(reports);
|
||||
const char *cp = mem;
|
||||
|
||||
fd->buffer = mem;
|
||||
fd->buffersize = memsize;
|
||||
|
||||
/* test if gzip */
|
||||
if (cp[0] == 0x1f && cp[1] == 0x8b) {
|
||||
if (0 == fd_read_gzip_from_memory_init(fd)) {
|
||||
blo_filedata_free(fd);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
fd->read = fd_read_from_memory;
|
||||
}
|
||||
|
||||
fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
|
||||
fd->file = file;
|
||||
|
||||
return blo_decode_and_check(fd, reports->reports);
|
||||
}
|
||||
@@ -1597,11 +1341,9 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
|
||||
}
|
||||
|
||||
FileData *fd = filedata_new(reports);
|
||||
fd->memfile = memfile;
|
||||
fd->file = BLO_memfile_new_filereader(memfile, params->undo_direction);
|
||||
fd->undo_direction = params->undo_direction;
|
||||
|
||||
fd->read = fd_read_from_memfile;
|
||||
fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
|
||||
fd->flags |= FD_FLAGS_IS_MEMFILE;
|
||||
|
||||
return blo_decode_and_check(fd, reports->reports);
|
||||
}
|
||||
@@ -1609,30 +1351,7 @@ FileData *blo_filedata_from_memfile(MemFile *memfile,
|
||||
void blo_filedata_free(FileData *fd)
|
||||
{
|
||||
if (fd) {
|
||||
if (fd->filedes != -1) {
|
||||
close(fd->filedes);
|
||||
}
|
||||
|
||||
if (fd->gzfiledes != NULL) {
|
||||
gzclose(fd->gzfiledes);
|
||||
}
|
||||
|
||||
if (fd->strm.next_in) {
|
||||
int err = inflateEnd(&fd->strm);
|
||||
if (err != Z_OK) {
|
||||
CLOG_ERROR(&LOG, "Close gzip stream error (code %d)", err);
|
||||
}
|
||||
}
|
||||
|
||||
if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) {
|
||||
MEM_freeN((void *)fd->buffer);
|
||||
fd->buffer = NULL;
|
||||
}
|
||||
|
||||
if (fd->mmap_file) {
|
||||
BLI_mmap_free(fd->mmap_file);
|
||||
fd->mmap_file = NULL;
|
||||
}
|
||||
fd->file->close(fd->file);
|
||||
|
||||
/* Free all BHeadN data blocks */
|
||||
#ifndef NDEBUG
|
||||
@@ -1640,7 +1359,7 @@ void blo_filedata_free(FileData *fd)
|
||||
#else
|
||||
/* Sanity check we're not keeping memory we don't need. */
|
||||
LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->bhead_list) {
|
||||
if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
|
||||
if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
|
||||
BLI_assert(new_bhead->has_data == 0);
|
||||
}
|
||||
MEM_freeN(new_bhead);
|
||||
@@ -2096,7 +1815,7 @@ static void blo_cache_storage_entry_clear_in_old(ID *UNUSED(id),
|
||||
|
||||
void blo_cache_storage_init(FileData *fd, Main *bmain)
|
||||
{
|
||||
if (fd->memfile != NULL) {
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
BLI_assert(fd->cache_storage == NULL);
|
||||
fd->cache_storage = MEM_mallocN(sizeof(*fd->cache_storage), __func__);
|
||||
fd->cache_storage->memarena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
|
||||
@@ -2261,7 +1980,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname)
|
||||
* undo since DNA must match. */
|
||||
static const void *peek_struct_undo(FileData *fd, BHead *bhead)
|
||||
{
|
||||
BLI_assert(fd->memfile != NULL);
|
||||
BLI_assert(fd->flags & FD_FLAGS_IS_MEMFILE);
|
||||
UNUSED_VARS_NDEBUG(fd);
|
||||
return (bhead->len) ? (const void *)(bhead + 1) : NULL;
|
||||
}
|
||||
@@ -3679,7 +3398,7 @@ static BHead *read_libblock(FileData *fd,
|
||||
* When datablocks are changed but still exist, we restore them at the old
|
||||
* address and inherit recalc flags for the dependency graph. */
|
||||
ID *id_old = NULL;
|
||||
if (fd->memfile != NULL) {
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
if (read_libblock_undo_restore(fd, main, bhead, tag, &id_old)) {
|
||||
if (r_id) {
|
||||
*r_id = id_old;
|
||||
@@ -3980,13 +3699,14 @@ static void lib_link_all(FileData *fd, Main *bmain)
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fd->memfile != NULL && GS(id->name) == ID_WM) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) && GS(id->name) == ID_WM) {
|
||||
/* No load UI for undo memfiles.
|
||||
* Only WM currently, SCR needs it still (see below), and so does WS? */
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fd->memfile != NULL && do_partial_undo && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) && do_partial_undo &&
|
||||
(id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
|
||||
/* This ID has been re-used from 'old' bmain. Since it was therefore unchanged across
|
||||
* current undo step, and old IDs re-use their old memory address, we do not need to liblink
|
||||
* it at all. */
|
||||
@@ -4165,7 +3885,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
|
||||
BlendFileData *bfd;
|
||||
ListBase mainlist = {NULL, NULL};
|
||||
|
||||
if (fd->memfile != NULL) {
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
CLOG_INFO(&LOG_UNDO, 2, "UNDO: read step");
|
||||
}
|
||||
|
||||
@@ -4256,7 +3976,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
|
||||
}
|
||||
|
||||
/* do before read_libraries, but skip undo case */
|
||||
if (fd->memfile == NULL) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
|
||||
if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) {
|
||||
do_versions(fd, NULL, bfd->main);
|
||||
}
|
||||
@@ -4278,7 +3998,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
|
||||
fd->reports->duration.libraries = PIL_check_seconds_timer() - fd->reports->duration.libraries;
|
||||
|
||||
/* Skip in undo case. */
|
||||
if (fd->memfile == NULL) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
|
||||
/* Note that we can't recompute user-counts at this point in undo case, we play too much with
|
||||
* IDs from different memory realms, and Main database is not in a fully valid state yet.
|
||||
*/
|
||||
@@ -4311,7 +4031,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
|
||||
|
||||
/* Now that all our data-blocks are loaded,
|
||||
* we can re-generate overrides from their references. */
|
||||
if (fd->memfile == NULL) {
|
||||
if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) {
|
||||
/* Do not apply in undo case! */
|
||||
fd->reports->duration.lib_overrides = PIL_check_seconds_timer();
|
||||
|
||||
@@ -4391,7 +4111,7 @@ static void sort_bhead_old_map(FileData *fd)
|
||||
static BHead *find_previous_lib(FileData *fd, BHead *bhead)
|
||||
{
|
||||
/* Skip library data-blocks in undo, see comment in read_libblock. */
|
||||
if (fd->memfile) {
|
||||
if (fd->flags & FD_FLAGS_IS_MEMFILE) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -5850,7 +5570,7 @@ void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p)
|
||||
|
||||
bool BLO_read_data_is_undo(BlendDataReader *reader)
|
||||
{
|
||||
return reader->fd->memfile != NULL;
|
||||
return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
|
||||
}
|
||||
|
||||
void BLO_read_data_globmap_add(BlendDataReader *reader, void *oldaddr, void *newaddr)
|
||||
@@ -5870,7 +5590,7 @@ BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader)
|
||||
|
||||
bool BLO_read_lib_is_undo(BlendLibReader *reader)
|
||||
{
|
||||
return reader->fd->memfile != NULL;
|
||||
return (reader->fd->flags & FD_FLAGS_IS_MEMFILE);
|
||||
}
|
||||
|
||||
Main *BLO_read_lib_get_main(BlendLibReader *reader)
|
||||
|
@@ -28,10 +28,10 @@
|
||||
# include "BLI_winstuff.h"
|
||||
#endif
|
||||
|
||||
#include "BLI_filereader.h"
|
||||
#include "DNA_sdna_types.h"
|
||||
#include "DNA_space_types.h"
|
||||
#include "DNA_windowmanager_types.h" /* for ReportType */
|
||||
#include "zlib.h"
|
||||
|
||||
struct BLI_mmap_file;
|
||||
struct BLOCacheStorage;
|
||||
@@ -50,7 +50,7 @@ enum eFileDataFlag {
|
||||
FD_FLAGS_FILE_POINTSIZE_IS_4 = 1 << 1,
|
||||
FD_FLAGS_POINTSIZE_DIFFERS = 1 << 2,
|
||||
FD_FLAGS_FILE_OK = 1 << 3,
|
||||
FD_FLAGS_NOT_MY_BUFFER = 1 << 4,
|
||||
FD_FLAGS_IS_MEMFILE = 1 << 4,
|
||||
/* XXX Unused in practice (checked once but never set). */
|
||||
FD_FLAGS_NOT_MY_LIBMAP = 1 << 5,
|
||||
};
|
||||
@@ -60,44 +60,18 @@ enum eFileDataFlag {
|
||||
# pragma GCC poison off_t
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__)
|
||||
typedef int64_t off64_t;
|
||||
#endif
|
||||
|
||||
typedef ssize_t(FileDataReadFn)(struct FileData *filedata,
|
||||
void *buffer,
|
||||
size_t size,
|
||||
bool *r_is_memchunk_identical);
|
||||
typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence);
|
||||
|
||||
typedef struct FileData {
|
||||
/** Linked list of BHeadN's. */
|
||||
ListBase bhead_list;
|
||||
enum eFileDataFlag flags;
|
||||
bool is_eof;
|
||||
size_t buffersize;
|
||||
off64_t file_offset;
|
||||
|
||||
FileDataReadFn *read;
|
||||
FileDataSeekFn *seek;
|
||||
FileReader *file;
|
||||
|
||||
/** Regular file reading. */
|
||||
int filedes;
|
||||
|
||||
/** Variables needed for reading from memory / stream / memory-mapped files. */
|
||||
const char *buffer;
|
||||
struct BLI_mmap_file *mmap_file;
|
||||
/** Variables needed for reading from memfile (undo). */
|
||||
struct MemFile *memfile;
|
||||
/** Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use
|
||||
* to detect unchanged data from memfile. */
|
||||
int undo_direction; /* eUndoStepDir */
|
||||
|
||||
/** Variables needed for reading from file. */
|
||||
gzFile gzfiledes;
|
||||
/** Gzip stream for memory decompression. */
|
||||
z_stream strm;
|
||||
|
||||
/** Now only in use for library appending. */
|
||||
char relabase[FILE_MAX];
|
||||
|
||||
|
@@ -48,6 +48,7 @@
|
||||
|
||||
#include "BKE_lib_id.h"
|
||||
#include "BKE_main.h"
|
||||
#include "BKE_undo_system.h"
|
||||
|
||||
/* keep last */
|
||||
#include "BLI_strict_flags.h"
|
||||
@@ -273,3 +274,97 @@ bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename)
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static ssize_t undo_read(FileReader *reader, void *buffer, size_t size)
|
||||
{
|
||||
UndoReader *undo = (UndoReader *)reader;
|
||||
|
||||
static size_t seek = SIZE_MAX; /* The current position. */
|
||||
static size_t offset = 0; /* Size of previous chunks. */
|
||||
static MemFileChunk *chunk = NULL;
|
||||
size_t chunkoffset, readsize, totread;
|
||||
|
||||
undo->memchunk_identical = true;
|
||||
|
||||
if (size == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (seek != (size_t)undo->reader.offset) {
|
||||
chunk = undo->memfile->chunks.first;
|
||||
seek = 0;
|
||||
|
||||
while (chunk) {
|
||||
if (seek + chunk->size > (size_t)undo->reader.offset) {
|
||||
break;
|
||||
}
|
||||
seek += chunk->size;
|
||||
chunk = chunk->next;
|
||||
}
|
||||
offset = seek;
|
||||
seek = (size_t)undo->reader.offset;
|
||||
}
|
||||
|
||||
if (chunk) {
|
||||
totread = 0;
|
||||
|
||||
do {
|
||||
/* First check if it's on the end if current chunk. */
|
||||
if (seek - offset == chunk->size) {
|
||||
offset += chunk->size;
|
||||
chunk = chunk->next;
|
||||
}
|
||||
|
||||
/* Debug, should never happen. */
|
||||
if (chunk == NULL) {
|
||||
printf("illegal read, chunk zero\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
chunkoffset = seek - offset;
|
||||
readsize = size - totread;
|
||||
|
||||
/* Data can be spread over multiple chunks, so clamp size
|
||||
* to within this chunk, and then it will read further in
|
||||
* the next chunk. */
|
||||
if (chunkoffset + readsize > chunk->size) {
|
||||
readsize = chunk->size - chunkoffset;
|
||||
}
|
||||
|
||||
memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize);
|
||||
totread += readsize;
|
||||
undo->reader.offset += (off64_t)readsize;
|
||||
seek += readsize;
|
||||
|
||||
/* `is_identical` of current chunk represents whether it changed compared to previous undo
|
||||
* step. this is fine in redo case, but not in undo case, where we need an extra flag
|
||||
* defined when saving the next (future) step after the one we want to restore, as we are
|
||||
* supposed to 'come from' that future undo step, and not the one before current one. */
|
||||
undo->memchunk_identical &= undo->undo_direction == STEP_REDO ? chunk->is_identical :
|
||||
chunk->is_identical_future;
|
||||
} while (totread < size);
|
||||
|
||||
return (ssize_t)totread;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void undo_close(FileReader *reader)
|
||||
{
|
||||
MEM_freeN(reader);
|
||||
}
|
||||
|
||||
FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction)
|
||||
{
|
||||
UndoReader *undo = MEM_callocN(sizeof(UndoReader), __func__);
|
||||
|
||||
undo->memfile = memfile;
|
||||
undo->undo_direction = undo_direction;
|
||||
|
||||
undo->reader.read = undo_read;
|
||||
undo->reader.seek = NULL;
|
||||
undo->reader.close = undo_close;
|
||||
|
||||
return (FileReader *)undo;
|
||||
}
|
||||
|
@@ -23,8 +23,7 @@
|
||||
#else
|
||||
# include "BLI_winstuff.h"
|
||||
# include "winsock2.h"
|
||||
# include <io.h> /* for open close read */
|
||||
# include <zlib.h> /* odd include order-issue */
|
||||
# include <io.h> /* for open close read */
|
||||
#endif
|
||||
|
||||
/* allow readfile to use deprecated functionality */
|
||||
|
@@ -28,8 +28,7 @@
|
||||
#else
|
||||
# include "BLI_winstuff.h"
|
||||
# include "winsock2.h"
|
||||
# include <io.h> /* for open close read */
|
||||
# include <zlib.h> /* odd include order-issue */
|
||||
# include <io.h> /* for open close read */
|
||||
#endif
|
||||
|
||||
/* allow readfile to use deprecated functionality */
|
||||
|
@@ -83,7 +83,6 @@
|
||||
# include "BLI_winstuff.h"
|
||||
# include "winsock2.h"
|
||||
# include <io.h>
|
||||
# include <zlib.h> /* odd include order-issue */
|
||||
#else
|
||||
# include <unistd.h> /* FreeBSD, for write() and close(). */
|
||||
#endif
|
||||
@@ -101,7 +100,12 @@
|
||||
#include "BLI_bitmap.h"
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_endian_defines.h"
|
||||
#include "BLI_endian_switch.h"
|
||||
#include "BLI_link_utils.h"
|
||||
#include "BLI_linklist.h"
|
||||
#include "BLI_math_base.h"
|
||||
#include "BLI_mempool.h"
|
||||
#include "BLI_threads.h"
|
||||
#include "MEM_guardedalloc.h" /* MEM_freeN */
|
||||
|
||||
#include "BKE_blender_version.h"
|
||||
@@ -129,14 +133,21 @@
|
||||
|
||||
#include <errno.h>
|
||||
|
||||
#include <zstd.h>
|
||||
|
||||
/* Make preferences read-only. */
|
||||
#define U (*((const UserDef *)&U))
|
||||
|
||||
/* ********* my write, buffered writing with minimum size chunks ************ */
|
||||
|
||||
/* Use optimal allocation since blocks of this size are kept in memory for undo. */
|
||||
#define MYWRITE_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */
|
||||
#define MYWRITE_MAX_CHUNK (MEM_SIZE_OPTIMAL(1 << 15)) /* ~32kb */
|
||||
#define MEM_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */
|
||||
#define MEM_CHUNK_SIZE (MEM_SIZE_OPTIMAL(1 << 15)) /* ~32kb */
|
||||
|
||||
#define ZSTD_BUFFER_SIZE (1 << 21) /* 2mb */
|
||||
#define ZSTD_CHUNK_SIZE (1 << 20) /* 1mb */
|
||||
|
||||
#define ZSTD_COMPRESSION_LEVEL 3
|
||||
|
||||
/** Use if we want to store how many bytes have been written to the file. */
|
||||
// #define USE_WRITE_DATA_LEN
|
||||
@@ -147,9 +158,16 @@
|
||||
|
||||
typedef enum {
|
||||
WW_WRAP_NONE = 1,
|
||||
WW_WRAP_ZLIB,
|
||||
WW_WRAP_ZSTD,
|
||||
} eWriteWrapType;
|
||||
|
||||
typedef struct ZstdFrame {
|
||||
struct ZstdFrame *next, *prev;
|
||||
|
||||
uint32_t compressed_size;
|
||||
uint32_t uncompressed_size;
|
||||
} ZstdFrame;
|
||||
|
||||
typedef struct WriteWrap WriteWrap;
|
||||
struct WriteWrap {
|
||||
/* callbacks */
|
||||
@@ -161,15 +179,23 @@ struct WriteWrap {
|
||||
bool use_buf;
|
||||
|
||||
/* internal */
|
||||
union {
|
||||
int file_handle;
|
||||
gzFile gz_handle;
|
||||
} _user_data;
|
||||
int file_handle;
|
||||
struct {
|
||||
ListBase threadpool;
|
||||
ListBase tasks;
|
||||
ThreadMutex mutex;
|
||||
ThreadCondition condition;
|
||||
int next_frame;
|
||||
int num_frames;
|
||||
|
||||
int level;
|
||||
ListBase frames;
|
||||
|
||||
bool write_error;
|
||||
} zstd;
|
||||
};
|
||||
|
||||
/* none */
|
||||
#define FILE_HANDLE(ww) (ww)->_user_data.file_handle
|
||||
|
||||
static bool ww_open_none(WriteWrap *ww, const char *filepath)
|
||||
{
|
||||
int file;
|
||||
@@ -177,7 +203,7 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath)
|
||||
file = BLI_open(filepath, O_BINARY + O_WRONLY + O_CREAT + O_TRUNC, 0666);
|
||||
|
||||
if (file != -1) {
|
||||
FILE_HANDLE(ww) = file;
|
||||
ww->file_handle = file;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -185,39 +211,170 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath)
|
||||
}
|
||||
static bool ww_close_none(WriteWrap *ww)
|
||||
{
|
||||
return (close(FILE_HANDLE(ww)) != -1);
|
||||
return (close(ww->file_handle) != -1);
|
||||
}
|
||||
static size_t ww_write_none(WriteWrap *ww, const char *buf, size_t buf_len)
|
||||
{
|
||||
return write(FILE_HANDLE(ww), buf, buf_len);
|
||||
return write(ww->file_handle, buf, buf_len);
|
||||
}
|
||||
#undef FILE_HANDLE
|
||||
|
||||
/* zlib */
|
||||
#define FILE_HANDLE(ww) (ww)->_user_data.gz_handle
|
||||
/* zstd */
|
||||
|
||||
static bool ww_open_zlib(WriteWrap *ww, const char *filepath)
|
||||
typedef struct {
|
||||
struct ZstdWriteBlockTask *next, *prev;
|
||||
void *data;
|
||||
size_t size;
|
||||
int frame_number;
|
||||
WriteWrap *ww;
|
||||
} ZstdWriteBlockTask;
|
||||
|
||||
static void *zstd_write_task(void *userdata)
|
||||
{
|
||||
gzFile file;
|
||||
ZstdWriteBlockTask *task = userdata;
|
||||
WriteWrap *ww = task->ww;
|
||||
|
||||
file = BLI_gzopen(filepath, "wb1");
|
||||
size_t out_buf_len = ZSTD_compressBound(task->size);
|
||||
void *out_buf = MEM_mallocN(out_buf_len, "Zstd out buffer");
|
||||
size_t out_size = ZSTD_compress(
|
||||
out_buf, out_buf_len, task->data, task->size, ZSTD_COMPRESSION_LEVEL);
|
||||
|
||||
if (file != Z_NULL) {
|
||||
FILE_HANDLE(ww) = file;
|
||||
return true;
|
||||
MEM_freeN(task->data);
|
||||
|
||||
BLI_mutex_lock(&ww->zstd.mutex);
|
||||
|
||||
while (ww->zstd.next_frame != task->frame_number) {
|
||||
BLI_condition_wait(&ww->zstd.condition, &ww->zstd.mutex);
|
||||
}
|
||||
|
||||
return false;
|
||||
if (ZSTD_isError(out_size)) {
|
||||
ww->zstd.write_error = true;
|
||||
}
|
||||
else {
|
||||
if (ww_write_none(ww, out_buf, out_size) == out_size) {
|
||||
ZstdFrame *frameinfo = MEM_mallocN(sizeof(ZstdFrame), "zstd frameinfo");
|
||||
frameinfo->uncompressed_size = task->size;
|
||||
frameinfo->compressed_size = out_size;
|
||||
BLI_addtail(&ww->zstd.frames, frameinfo);
|
||||
}
|
||||
else {
|
||||
ww->zstd.write_error = true;
|
||||
}
|
||||
}
|
||||
|
||||
ww->zstd.next_frame++;
|
||||
|
||||
BLI_mutex_unlock(&ww->zstd.mutex);
|
||||
BLI_condition_notify_all(&ww->zstd.condition);
|
||||
|
||||
MEM_freeN(out_buf);
|
||||
return NULL;
|
||||
}
|
||||
static bool ww_close_zlib(WriteWrap *ww)
|
||||
|
||||
static bool ww_open_zstd(WriteWrap *ww, const char *filepath)
|
||||
{
|
||||
return (gzclose(FILE_HANDLE(ww)) == Z_OK);
|
||||
if (!ww_open_none(ww, filepath)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Leave one thread open for the main writing logic, unless we only have one HW thread. */
|
||||
int num_threads = max_ii(1, BLI_system_thread_count() - 1);
|
||||
BLI_threadpool_init(&ww->zstd.threadpool, zstd_write_task, num_threads);
|
||||
BLI_mutex_init(&ww->zstd.mutex);
|
||||
BLI_condition_init(&ww->zstd.condition);
|
||||
|
||||
return true;
|
||||
}
|
||||
static size_t ww_write_zlib(WriteWrap *ww, const char *buf, size_t buf_len)
|
||||
|
||||
static void zstd_write_u32_le(WriteWrap *ww, uint32_t val)
|
||||
{
|
||||
return gzwrite(FILE_HANDLE(ww), buf, buf_len);
|
||||
#ifdef __BIG_ENDIAN__
|
||||
BLI_endian_switch_uint32(&val);
|
||||
#endif
|
||||
ww_write_none(ww, (char *)&val, sizeof(uint32_t));
|
||||
}
|
||||
|
||||
/* In order to implement efficient seeking when reading the .blend, we append
|
||||
* a skippable frame that encodes information about the other frames present
|
||||
* in the file.
|
||||
* The format here follows the upstream spec for seekable files:
|
||||
* https://github.com/facebook/zstd/blob/master/contrib/seekable_format/zstd_seekable_compression_format.md
|
||||
* If this information is not present in a file (e.g. if it was compressed
|
||||
* with external tools), it can still be opened in Blender, but seeking will
|
||||
* not be supported, so more memory might be needed. */
|
||||
static void zstd_write_seekable_frames(WriteWrap *ww)
|
||||
{
|
||||
/* Write seek table header (magic number and frame size). */
|
||||
zstd_write_u32_le(ww, 0x184D2A5E);
|
||||
|
||||
/* The actual frame number might not match ww->zstd.num_frames if there was a write error. */
|
||||
const uint32_t num_frames = BLI_listbase_count(&ww->zstd.frames);
|
||||
/* Each frame consists of two u32, so 8 bytes each.
|
||||
* After the frames, a footer containing two u32 and one byte (9 bytes total) is written. */
|
||||
const uint32_t frame_size = num_frames * 8 + 9;
|
||||
zstd_write_u32_le(ww, frame_size);
|
||||
|
||||
/* Write seek table entries. */
|
||||
LISTBASE_FOREACH (ZstdFrame *, frame, &ww->zstd.frames) {
|
||||
zstd_write_u32_le(ww, frame->compressed_size);
|
||||
zstd_write_u32_le(ww, frame->uncompressed_size);
|
||||
}
|
||||
|
||||
/* Write seek table footer (number of frames, option flags and second magic number). */
|
||||
zstd_write_u32_le(ww, num_frames);
|
||||
const char flags = 0; /* We don't store checksums for each frame. */
|
||||
ww_write_none(ww, &flags, 1);
|
||||
zstd_write_u32_le(ww, 0x8F92EAB1);
|
||||
}
|
||||
|
||||
static bool ww_close_zstd(WriteWrap *ww)
|
||||
{
|
||||
BLI_threadpool_end(&ww->zstd.threadpool);
|
||||
BLI_freelistN(&ww->zstd.tasks);
|
||||
|
||||
BLI_mutex_end(&ww->zstd.mutex);
|
||||
BLI_condition_end(&ww->zstd.condition);
|
||||
|
||||
zstd_write_seekable_frames(ww);
|
||||
BLI_freelistN(&ww->zstd.frames);
|
||||
|
||||
return ww_close_none(ww) && !ww->zstd.write_error;
|
||||
}
|
||||
|
||||
static size_t ww_write_zstd(WriteWrap *ww, const char *buf, size_t buf_len)
|
||||
{
|
||||
if (ww->zstd.write_error) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZstdWriteBlockTask *task = MEM_mallocN(sizeof(ZstdWriteBlockTask), __func__);
|
||||
task->data = MEM_mallocN(buf_len, __func__);
|
||||
memcpy(task->data, buf, buf_len);
|
||||
task->size = buf_len;
|
||||
task->frame_number = ww->zstd.num_frames++;
|
||||
task->ww = ww;
|
||||
|
||||
BLI_mutex_lock(&ww->zstd.mutex);
|
||||
BLI_addtail(&ww->zstd.tasks, task);
|
||||
|
||||
/* If there's a free worker thread, just push the block into that thread.
|
||||
* Otherwise, we wait for the earliest thread to finish.
|
||||
* We look up the earliest thread while holding the mutex, but release it
|
||||
* before joining the thread to prevent a deadlock. */
|
||||
ZstdWriteBlockTask *first_task = ww->zstd.tasks.first;
|
||||
BLI_mutex_unlock(&ww->zstd.mutex);
|
||||
if (!BLI_available_threads(&ww->zstd.threadpool)) {
|
||||
BLI_threadpool_remove(&ww->zstd.threadpool, first_task);
|
||||
|
||||
/* If the task list was empty before we pushed our task, there should
|
||||
* always be a free thread. */
|
||||
BLI_assert(first_task != task);
|
||||
BLI_remlink(&ww->zstd.tasks, first_task);
|
||||
MEM_freeN(first_task);
|
||||
}
|
||||
BLI_threadpool_insert(&ww->zstd.threadpool, task);
|
||||
|
||||
return buf_len;
|
||||
}
|
||||
#undef FILE_HANDLE
|
||||
|
||||
/* --- end compression types --- */
|
||||
|
||||
@@ -226,11 +383,11 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww)
|
||||
memset(r_ww, 0, sizeof(*r_ww));
|
||||
|
||||
switch (ww_type) {
|
||||
case WW_WRAP_ZLIB: {
|
||||
r_ww->open = ww_open_zlib;
|
||||
r_ww->close = ww_close_zlib;
|
||||
r_ww->write = ww_write_zlib;
|
||||
r_ww->use_buf = false;
|
||||
case WW_WRAP_ZSTD: {
|
||||
r_ww->open = ww_open_zstd;
|
||||
r_ww->close = ww_close_zstd;
|
||||
r_ww->write = ww_write_zstd;
|
||||
r_ww->use_buf = true;
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
@@ -252,10 +409,17 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww)
|
||||
typedef struct {
|
||||
const struct SDNA *sdna;
|
||||
|
||||
/** Use for file and memory writing (fixed size of #MYWRITE_BUFFER_SIZE). */
|
||||
uchar *buf;
|
||||
/** Number of bytes used in #WriteData.buf (flushed when exceeded). */
|
||||
size_t buf_used_len;
|
||||
struct {
|
||||
/** Use for file and memory writing (size stored in max_size). */
|
||||
uchar *buf;
|
||||
/** Number of bytes used in #WriteData.buf (flushed when exceeded). */
|
||||
size_t used_len;
|
||||
|
||||
/** Maximum size of the buffer. */
|
||||
size_t max_size;
|
||||
/** Threshold above which writes get their own chunk. */
|
||||
size_t chunk_size;
|
||||
} buffer;
|
||||
|
||||
#ifdef USE_WRITE_DATA_LEN
|
||||
/** Total number of bytes written. */
|
||||
@@ -271,7 +435,7 @@ typedef struct {
|
||||
bool use_memfile;
|
||||
|
||||
/**
|
||||
* Wrap writing, so we can use zlib or
|
||||
* Wrap writing, so we can use zstd or
|
||||
* other compression types later, see: G_FILE_COMPRESS
|
||||
* Will be NULL for UNDO.
|
||||
*/
|
||||
@@ -291,7 +455,15 @@ static WriteData *writedata_new(WriteWrap *ww)
|
||||
wd->ww = ww;
|
||||
|
||||
if ((ww == NULL) || (ww->use_buf)) {
|
||||
wd->buf = MEM_mallocN(MYWRITE_BUFFER_SIZE, "wd->buf");
|
||||
if (ww == NULL) {
|
||||
wd->buffer.max_size = MEM_BUFFER_SIZE;
|
||||
wd->buffer.chunk_size = MEM_CHUNK_SIZE;
|
||||
}
|
||||
else {
|
||||
wd->buffer.max_size = ZSTD_BUFFER_SIZE;
|
||||
wd->buffer.chunk_size = ZSTD_CHUNK_SIZE;
|
||||
}
|
||||
wd->buffer.buf = MEM_mallocN(wd->buffer.max_size, "wd->buffer.buf");
|
||||
}
|
||||
|
||||
return wd;
|
||||
@@ -325,8 +497,8 @@ static void writedata_do_write(WriteData *wd, const void *mem, size_t memlen)
|
||||
|
||||
static void writedata_free(WriteData *wd)
|
||||
{
|
||||
if (wd->buf) {
|
||||
MEM_freeN(wd->buf);
|
||||
if (wd->buffer.buf) {
|
||||
MEM_freeN(wd->buffer.buf);
|
||||
}
|
||||
MEM_freeN(wd);
|
||||
}
|
||||
@@ -343,9 +515,9 @@ static void writedata_free(WriteData *wd)
|
||||
*/
|
||||
static void mywrite_flush(WriteData *wd)
|
||||
{
|
||||
if (wd->buf_used_len != 0) {
|
||||
writedata_do_write(wd, wd->buf, wd->buf_used_len);
|
||||
wd->buf_used_len = 0;
|
||||
if (wd->buffer.used_len != 0) {
|
||||
writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
|
||||
wd->buffer.used_len = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -369,20 +541,20 @@ static void mywrite(WriteData *wd, const void *adr, size_t len)
|
||||
wd->write_len += len;
|
||||
#endif
|
||||
|
||||
if (wd->buf == NULL) {
|
||||
if (wd->buffer.buf == NULL) {
|
||||
writedata_do_write(wd, adr, len);
|
||||
}
|
||||
else {
|
||||
/* if we have a single big chunk, write existing data in
|
||||
* buffer and write out big chunk in smaller pieces */
|
||||
if (len > MYWRITE_MAX_CHUNK) {
|
||||
if (wd->buf_used_len != 0) {
|
||||
writedata_do_write(wd, wd->buf, wd->buf_used_len);
|
||||
wd->buf_used_len = 0;
|
||||
if (len > wd->buffer.chunk_size) {
|
||||
if (wd->buffer.used_len != 0) {
|
||||
writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
|
||||
wd->buffer.used_len = 0;
|
||||
}
|
||||
|
||||
do {
|
||||
size_t writelen = MIN2(len, MYWRITE_MAX_CHUNK);
|
||||
size_t writelen = MIN2(len, wd->buffer.chunk_size);
|
||||
writedata_do_write(wd, adr, writelen);
|
||||
adr = (const char *)adr + writelen;
|
||||
len -= writelen;
|
||||
@@ -392,14 +564,14 @@ static void mywrite(WriteData *wd, const void *adr, size_t len)
|
||||
}
|
||||
|
||||
/* if data would overflow buffer, write out the buffer */
|
||||
if (len + wd->buf_used_len > MYWRITE_BUFFER_SIZE - 1) {
|
||||
writedata_do_write(wd, wd->buf, wd->buf_used_len);
|
||||
wd->buf_used_len = 0;
|
||||
if (len + wd->buffer.used_len > wd->buffer.max_size - 1) {
|
||||
writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
|
||||
wd->buffer.used_len = 0;
|
||||
}
|
||||
|
||||
/* append data at end of buffer */
|
||||
memcpy(&wd->buf[wd->buf_used_len], adr, len);
|
||||
wd->buf_used_len += len;
|
||||
memcpy(&wd->buffer.buf[wd->buffer.used_len], adr, len);
|
||||
wd->buffer.used_len += len;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,9 +602,9 @@ static WriteData *mywrite_begin(WriteWrap *ww, MemFile *compare, MemFile *curren
|
||||
*/
|
||||
static bool mywrite_end(WriteData *wd)
|
||||
{
|
||||
if (wd->buf_used_len != 0) {
|
||||
writedata_do_write(wd, wd->buf, wd->buf_used_len);
|
||||
wd->buf_used_len = 0;
|
||||
if (wd->buffer.used_len != 0) {
|
||||
writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len);
|
||||
wd->buffer.used_len = 0;
|
||||
}
|
||||
|
||||
if (wd->use_memfile) {
|
||||
@@ -1150,7 +1322,6 @@ bool BLO_write_file(Main *mainvar,
|
||||
ReportList *reports)
|
||||
{
|
||||
char tempname[FILE_MAX + 1];
|
||||
eWriteWrapType ww_type;
|
||||
WriteWrap ww;
|
||||
|
||||
eBLO_WritePathRemap remap_mode = params->remap_mode;
|
||||
@@ -1172,14 +1343,7 @@ bool BLO_write_file(Main *mainvar,
|
||||
/* open temporary file, so we preserve the original in case we crash */
|
||||
BLI_snprintf(tempname, sizeof(tempname), "%s@", filepath);
|
||||
|
||||
if (write_flags & G_FILE_COMPRESS) {
|
||||
ww_type = WW_WRAP_ZLIB;
|
||||
}
|
||||
else {
|
||||
ww_type = WW_WRAP_NONE;
|
||||
}
|
||||
|
||||
ww_handle_init(ww_type, &ww);
|
||||
ww_handle_init((write_flags & G_FILE_COMPRESS) ? WW_WRAP_ZSTD : WW_WRAP_NONE, &ww);
|
||||
|
||||
if (ww.open(&ww, tempname) == false) {
|
||||
BKE_reportf(
|
||||
|
@@ -102,7 +102,7 @@
|
||||
/* <cache type>-<resolution X>x<resolution Y>-<rendersize>%(<view_id>)-<frame no>.dcf */
|
||||
#define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf"
|
||||
#define DCACHE_IMAGES_PER_FILE 100
|
||||
#define DCACHE_CURRENT_VERSION 1
|
||||
#define DCACHE_CURRENT_VERSION 2
|
||||
#define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */
|
||||
|
||||
typedef struct DiskCacheHeaderEntry {
|
||||
@@ -496,24 +496,34 @@ static size_t deflate_imbuf_to_file(ImBuf *ibuf,
|
||||
int level,
|
||||
DiskCacheHeaderEntry *header_entry)
|
||||
{
|
||||
if (ibuf->rect) {
|
||||
return BLI_gzip_mem_to_file_at_pos(
|
||||
ibuf->rect, header_entry->size_raw, file, header_entry->offset, level);
|
||||
void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float;
|
||||
|
||||
/* Apply compression if wanted, otherwise just write directly to the file. */
|
||||
if (level > 0) {
|
||||
return BLI_file_zstd_from_mem_at_pos(
|
||||
data, header_entry->size_raw, file, header_entry->offset, level);
|
||||
}
|
||||
|
||||
return BLI_gzip_mem_to_file_at_pos(
|
||||
ibuf->rect_float, header_entry->size_raw, file, header_entry->offset, level);
|
||||
fseek(file, header_entry->offset, SEEK_SET);
|
||||
return fwrite(data, 1, header_entry->size_raw, file);
|
||||
}
|
||||
|
||||
static size_t inflate_file_to_imbuf(ImBuf *ibuf, FILE *file, DiskCacheHeaderEntry *header_entry)
|
||||
{
|
||||
if (ibuf->rect) {
|
||||
return BLI_ungzip_file_to_mem_at_pos(
|
||||
ibuf->rect, header_entry->size_raw, file, header_entry->offset);
|
||||
void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float;
|
||||
char header[4];
|
||||
fseek(file, header_entry->offset, SEEK_SET);
|
||||
if (fread(header, 1, sizeof(header), file) != sizeof(header)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return BLI_ungzip_file_to_mem_at_pos(
|
||||
ibuf->rect_float, header_entry->size_raw, file, header_entry->offset);
|
||||
/* Check if the data is compressed or raw. */
|
||||
if (BLI_file_magic_is_zstd(header)) {
|
||||
return BLI_file_unzstd_to_mem_at_pos(data, header_entry->size_raw, file, header_entry->offset);
|
||||
}
|
||||
|
||||
fseek(file, header_entry->offset, SEEK_SET);
|
||||
return fread(data, 1, header_entry->size_raw, file);
|
||||
}
|
||||
|
||||
static bool seq_disk_cache_read_header(FILE *file, DiskCacheHeader *header)
|
||||
|
@@ -48,10 +48,6 @@ set(INC
|
||||
${CMAKE_BINARY_DIR}/source/blender/makesdna/intern
|
||||
)
|
||||
|
||||
set(INC_SYS
|
||||
${ZLIB_INCLUDE_DIRS}
|
||||
)
|
||||
|
||||
set(SRC
|
||||
intern/wm.c
|
||||
intern/wm_cursors.c
|
||||
|
@@ -28,11 +28,10 @@
|
||||
* winsock stuff.
|
||||
*/
|
||||
#include <errno.h>
|
||||
#include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "zlib.h" /* wm_read_exotic() */
|
||||
|
||||
#ifdef WIN32
|
||||
/* Need to include windows.h so _WIN32_IE is defined. */
|
||||
# include <windows.h>
|
||||
@@ -51,6 +50,7 @@
|
||||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_fileops_types.h"
|
||||
#include "BLI_filereader.h"
|
||||
#include "BLI_linklist.h"
|
||||
#include "BLI_math.h"
|
||||
#include "BLI_system.h"
|
||||
@@ -481,53 +481,64 @@ static void wm_init_userdef(Main *bmain)
|
||||
/* intended to check for non-blender formats but for now it only reads blends */
|
||||
static int wm_read_exotic(const char *name)
|
||||
{
|
||||
int len;
|
||||
gzFile gzfile;
|
||||
char header[7];
|
||||
int retval;
|
||||
|
||||
/* make sure we're not trying to read a directory.... */
|
||||
|
||||
len = strlen(name);
|
||||
if (len > 0 && ELEM(name[len - 1], '/', '\\')) {
|
||||
retval = BKE_READ_EXOTIC_FAIL_PATH;
|
||||
int namelen = strlen(name);
|
||||
if (namelen > 0 && ELEM(name[namelen - 1], '/', '\\')) {
|
||||
return BKE_READ_EXOTIC_FAIL_PATH;
|
||||
}
|
||||
|
||||
/* open the file. */
|
||||
const int filedes = BLI_open(name, O_BINARY | O_RDONLY, 0);
|
||||
if (filedes == -1) {
|
||||
return BKE_READ_EXOTIC_FAIL_OPEN;
|
||||
}
|
||||
|
||||
FileReader *rawfile = BLI_filereader_new_file(filedes);
|
||||
if (rawfile == NULL) {
|
||||
return BKE_READ_EXOTIC_FAIL_OPEN;
|
||||
}
|
||||
|
||||
/* read the header (7 bytes are enough to identify all known types). */
|
||||
char header[7];
|
||||
if (rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) {
|
||||
rawfile->close(rawfile);
|
||||
return BKE_READ_EXOTIC_FAIL_FORMAT;
|
||||
}
|
||||
rawfile->seek(rawfile, 0, SEEK_SET);
|
||||
|
||||
/* check for uncompressed .blend */
|
||||
if (STREQLEN(header, "BLENDER", 7)) {
|
||||
rawfile->close(rawfile);
|
||||
return BKE_READ_EXOTIC_OK_BLEND;
|
||||
}
|
||||
|
||||
/* check for compressed .blend */
|
||||
FileReader *compressed_file = NULL;
|
||||
if (BLI_file_magic_is_gzip(header)) {
|
||||
/* In earlier versions of Blender (before 3.0), compressed files used Gzip instead of Zstd.
|
||||
* While these files will no longer be written, there still needs to be reading support. */
|
||||
compressed_file = BLI_filereader_new_gzip(rawfile);
|
||||
}
|
||||
else if (BLI_file_magic_is_zstd(header)) {
|
||||
compressed_file = BLI_filereader_new_zstd(rawfile);
|
||||
}
|
||||
|
||||
/* If a compression signature matches, try decompressing the start and check if it's a .blend */
|
||||
if (compressed_file != NULL) {
|
||||
size_t len = compressed_file->read(compressed_file, header, sizeof(header));
|
||||
compressed_file->close(compressed_file);
|
||||
if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) {
|
||||
return BKE_READ_EXOTIC_OK_BLEND;
|
||||
}
|
||||
}
|
||||
else {
|
||||
gzfile = BLI_gzopen(name, "rb");
|
||||
if (gzfile == NULL) {
|
||||
retval = BKE_READ_EXOTIC_FAIL_OPEN;
|
||||
}
|
||||
else {
|
||||
len = gzread(gzfile, header, sizeof(header));
|
||||
gzclose(gzfile);
|
||||
if (len == sizeof(header) && STREQLEN(header, "BLENDER", 7)) {
|
||||
retval = BKE_READ_EXOTIC_OK_BLEND;
|
||||
}
|
||||
else {
|
||||
/* We may want to support loading other file formats
|
||||
* from their header bytes or file extension.
|
||||
* This used to be supported in the code below and may be added
|
||||
* back at some point. */
|
||||
#if 0
|
||||
WM_cursor_wait(true);
|
||||
|
||||
if (is_foo_format(name)) {
|
||||
read_foo(name);
|
||||
retval = BKE_READ_EXOTIC_OK_OTHER;
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
retval = BKE_READ_EXOTIC_FAIL_FORMAT;
|
||||
}
|
||||
#if 0
|
||||
WM_cursor_wait(false);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
rawfile->close(rawfile);
|
||||
}
|
||||
|
||||
return retval;
|
||||
/* Add check for future file formats here. */
|
||||
|
||||
return BKE_READ_EXOTIC_FAIL_FORMAT;
|
||||
}
|
||||
|
||||
/** \} */
|
||||
|
Reference in New Issue
Block a user