Source code
Revision control
Copy as Markdown
Other Tools
/* XzEnc.c -- Xz Encode↩
2018-04-28 : Igor Pavlov : Public domain */↩
↩
#include "Precomp.h"↩
↩
#include <stdlib.h>↩
#include <string.h>↩
↩
#include "7zCrc.h"↩
#include "Bra.h"↩
#include "CpuArch.h"↩
↩
#ifdef USE_SUBBLOCK↩
#include "Bcj3Enc.c"↩
#include "SbFind.c"↩
#include "SbEnc.c"↩
#endif↩
↩
#include "XzEnc.h"↩
↩
// #define _7ZIP_ST↩
↩
#ifndef _7ZIP_ST↩
#include "MtCoder.h"↩
#else↩
#define MTCODER__THREADS_MAX 1↩
#define MTCODER__BLOCKS_MAX 1↩
#endif↩
↩
#define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3)↩
↩
/* max pack size for LZMA2 block + check-64bytrs: */↩
#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64)↩
↩
#define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize))↩
↩
↩
#define XzBlock_ClearFlags(p) (p)->flags = 0;↩
#define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);↩
#define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE;↩
#define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;↩
↩
↩
static SRes WriteBytes(ISeqOutStream *s, const void *buf, size_t size)↩
{↩
return (ISeqOutStream_Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;↩
}↩
↩
static SRes WriteBytesUpdateCrc(ISeqOutStream *s, const void *buf, size_t size, UInt32 *crc)↩
{↩
*crc = CrcUpdate(*crc, buf, size);↩
return WriteBytes(s, buf, size);↩
}↩
↩
↩
static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)↩
{↩
UInt32 crc;↩
Byte header[XZ_STREAM_HEADER_SIZE];↩
memcpy(header, XZ_SIG, XZ_SIG_SIZE);↩
header[XZ_SIG_SIZE] = (Byte)(f >> 8);↩
header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);↩
crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);↩
SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);↩
return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);↩
}↩
↩
↩
static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)↩
{↩
Byte header[XZ_BLOCK_HEADER_SIZE_MAX];↩
↩
unsigned pos = 1;↩
unsigned numFilters, i;↩
header[pos++] = p->flags;↩
↩
if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);↩
if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);↩
numFilters = XzBlock_GetNumFilters(p);↩
↩
for (i = 0; i < numFilters; i++)↩
{↩
const CXzFilter *f = &p->filters[i];↩
pos += Xz_WriteVarInt(header + pos, f->id);↩
pos += Xz_WriteVarInt(header + pos, f->propsSize);↩
memcpy(header + pos, f->props, f->propsSize);↩
pos += f->propsSize;↩
}↩
↩
while ((pos & 3) != 0)↩
header[pos++] = 0;↩
↩
header[0] = (Byte)(pos >> 2);↩
SetUi32(header + pos, CrcCalc(header, pos));↩
return WriteBytes(s, header, pos + 4);↩
}↩
↩
↩
↩
↩
typedef struct↩
{↩
size_t numBlocks;↩
size_t size;↩
size_t allocated;↩
Byte *blocks;↩
} CXzEncIndex;↩
↩
↩
static void XzEncIndex_Construct(CXzEncIndex *p)↩
{↩
p->numBlocks = 0;↩
p->size = 0;↩
p->allocated = 0;↩
p->blocks = NULL;↩
}↩
↩
static void XzEncIndex_Init(CXzEncIndex *p)↩
{↩
p->numBlocks = 0;↩
p->size = 0;↩
}↩
↩
static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc)↩
{↩
if (p->blocks)↩
{↩
ISzAlloc_Free(alloc, p->blocks);↩
p->blocks = NULL;↩
}↩
p->numBlocks = 0;↩
p->size = 0;↩
p->allocated = 0;↩
}↩
↩
↩
static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc)↩
{↩
Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize);↩
if (!blocks)↩
return SZ_ERROR_MEM;↩
if (p->size != 0)↩
memcpy(blocks, p->blocks, p->size);↩
if (p->blocks)↩
ISzAlloc_Free(alloc, p->blocks);↩
p->blocks = blocks;↩
p->allocated = newSize;↩
return SZ_OK;↩
}↩
↩
↩
static SRes XzEncIndex_PreAlloc(CXzEncIndex *p, UInt64 numBlocks, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc)↩
{↩
UInt64 pos;↩
{↩
Byte buf[32];↩
unsigned pos2 = Xz_WriteVarInt(buf, totalSize);↩
pos2 += Xz_WriteVarInt(buf + pos2, unpackSize);↩
pos = numBlocks * pos2;↩
}↩
↩
if (pos <= p->allocated - p->size)↩
return SZ_OK;↩
{↩
UInt64 newSize64 = p->size + pos;↩
size_t newSize = (size_t)newSize64;↩
if (newSize != newSize64)↩
return SZ_ERROR_MEM;↩
return XzEncIndex_ReAlloc(p, newSize, alloc);↩
}↩
}↩
↩
↩
static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc)↩
{↩
Byte buf[32];↩
unsigned pos = Xz_WriteVarInt(buf, totalSize);↩
pos += Xz_WriteVarInt(buf + pos, unpackSize);↩
↩
if (pos > p->allocated - p->size)↩
{↩
size_t newSize = p->allocated * 2 + 16 * 2;↩
if (newSize < p->size + pos)↩
return SZ_ERROR_MEM;↩
RINOK(XzEncIndex_ReAlloc(p, newSize, alloc));↩
}↩
memcpy(p->blocks + p->size, buf, pos);↩
p->size += pos;↩
p->numBlocks++;↩
return SZ_OK;↩
}↩
↩
↩
static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStream *s)↩
{↩
Byte buf[32];↩
UInt64 globalPos;↩
UInt32 crc = CRC_INIT_VAL;↩
unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);↩
↩
globalPos = pos;↩
buf[0] = 0;↩
RINOK(WriteBytesUpdateCrc(s, buf, pos, &crc));↩
RINOK(WriteBytesUpdateCrc(s, p->blocks, p->size, &crc));↩
globalPos += p->size;↩
↩
pos = XZ_GET_PAD_SIZE(globalPos);↩
buf[1] = 0;↩
buf[2] = 0;↩
buf[3] = 0;↩
globalPos += pos;↩
↩
crc = CrcUpdate(crc, buf + 4 - pos, pos);↩
SetUi32(buf + 4, CRC_GET_DIGEST(crc));↩
↩
SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2));↩
buf[8 + 8] = (Byte)(flags >> 8);↩
buf[8 + 9] = (Byte)(flags & 0xFF);↩
SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6));↩
buf[8 + 10] = XZ_FOOTER_SIG_0;↩
buf[8 + 11] = XZ_FOOTER_SIG_1;↩
↩
return WriteBytes(s, buf + 4 - pos, pos + 4 + 12);↩
}↩
↩
↩
↩
/* ---------- CSeqCheckInStream ---------- */↩
↩
typedef struct↩
{↩
ISeqInStream vt;↩
ISeqInStream *realStream;↩
const Byte *data;↩
UInt64 limit;↩
UInt64 processed;↩
int realStreamFinished;↩
CXzCheck check;↩
} CSeqCheckInStream;↩
↩
static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned checkMode)↩
{↩
p->limit = (UInt64)(Int64)-1;↩
p->processed = 0;↩
p->realStreamFinished = 0;↩
XzCheck_Init(&p->check, checkMode);↩
}↩
↩
static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)↩
{↩
XzCheck_Final(&p->check, digest);↩
}↩
↩
static SRes SeqCheckInStream_Read(const ISeqInStream *pp, void *data, size_t *size)↩
{↩
CSeqCheckInStream *p = CONTAINER_FROM_VTBL(pp, CSeqCheckInStream, vt);↩
size_t size2 = *size;↩
SRes res = SZ_OK;↩
↩
if (p->limit != (UInt64)(Int64)-1)↩
{↩
UInt64 rem = p->limit - p->processed;↩
if (size2 > rem)↩
size2 = (size_t)rem;↩
}↩
if (size2 != 0)↩
{↩
if (p->realStream)↩
{↩
res = ISeqInStream_Read(p->realStream, data, &size2);↩
p->realStreamFinished = (size2 == 0) ? 1 : 0;↩
}↩
else↩
memcpy(data, p->data + (size_t)p->processed, size2);↩
XzCheck_Update(&p->check, data, size2);↩
p->processed += size2;↩
}↩
*size = size2;↩
return res;↩
}↩
↩
↩
/* ---------- CSeqSizeOutStream ---------- */↩
↩
typedef struct↩
{↩
ISeqOutStream vt;↩
ISeqOutStream *realStream;↩
Byte *outBuf;↩
size_t outBufLimit;↩
UInt64 processed;↩
} CSeqSizeOutStream;↩
↩
static size_t SeqSizeOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size)↩
{↩
CSeqSizeOutStream *p = CONTAINER_FROM_VTBL(pp, CSeqSizeOutStream, vt);↩
if (p->realStream)↩
size = ISeqOutStream_Write(p->realStream, data, size);↩
else↩
{↩
if (size > p->outBufLimit - (size_t)p->processed)↩
return 0;↩
memcpy(p->outBuf + (size_t)p->processed, data, size);↩
}↩
p->processed += size;↩
return size;↩
}↩
↩
↩
/* ---------- CSeqInFilter ---------- */↩
↩
#define FILTER_BUF_SIZE (1 << 20)↩
↩
typedef struct↩
{↩
ISeqInStream p;↩
ISeqInStream *realStream;↩
IStateCoder StateCoder;↩
Byte *buf;↩
size_t curPos;↩
size_t endPos;↩
int srcWasFinished;↩
} CSeqInFilter;↩
↩
↩
SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc);↩
↩
static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPtr alloc)↩
{↩
if (!p->buf)↩
{↩
p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE);↩
if (!p->buf)↩
return SZ_ERROR_MEM;↩
}↩
p->curPos = p->endPos = 0;↩
p->srcWasFinished = 0;↩
RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, alloc));↩
RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc));↩
p->StateCoder.Init(p->StateCoder.p);↩
return SZ_OK;↩
}↩
↩
↩
static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size)↩
{↩
CSeqInFilter *p = CONTAINER_FROM_VTBL(pp, CSeqInFilter, p);↩
size_t sizeOriginal = *size;↩
if (sizeOriginal == 0)↩
return SZ_OK;↩
*size = 0;↩
↩
for (;;)↩
{↩
if (!p->srcWasFinished && p->curPos == p->endPos)↩
{↩
p->curPos = 0;↩
p->endPos = FILTER_BUF_SIZE;↩
RINOK(ISeqInStream_Read(p->realStream, p->buf, &p->endPos));↩
if (p->endPos == 0)↩
p->srcWasFinished = 1;↩
}↩
{↩
SizeT srcLen = p->endPos - p->curPos;↩
ECoderStatus status;↩
SRes res;↩
*size = sizeOriginal;↩
res = p->StateCoder.Code2(p->StateCoder.p,↩
data, size,↩
p->buf + p->curPos, &srcLen,↩
p->srcWasFinished, CODER_FINISH_ANY,↩
&status);↩
p->curPos += srcLen;↩
if (*size != 0 || srcLen == 0 || res != SZ_OK)↩
return res;↩
}↩
}↩
}↩
↩
static void SeqInFilter_Construct(CSeqInFilter *p)↩
{↩
p->buf = NULL;↩
p->StateCoder.p = NULL;↩
p->p.Read = SeqInFilter_Read;↩
}↩
↩
static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc)↩
{↩
if (p->StateCoder.p)↩
{↩
p->StateCoder.Free(p->StateCoder.p, alloc);↩
p->StateCoder.p = NULL;↩
}↩
if (p->buf)↩
{↩
ISzAlloc_Free(alloc, p->buf);↩
p->buf = NULL;↩
}↩
}↩
↩
↩
/* ---------- CSbEncInStream ---------- */↩
↩
#ifdef USE_SUBBLOCK↩
↩
typedef struct↩
{↩
ISeqInStream vt;↩
ISeqInStream *inStream;↩
CSbEnc enc;↩
} CSbEncInStream;↩
↩
static SRes SbEncInStream_Read(const ISeqInStream *pp, void *data, size_t *size)↩
{↩
CSbEncInStream *p = CONTAINER_FROM_VTBL(pp, CSbEncInStream, vt);↩
size_t sizeOriginal = *size;↩
if (sizeOriginal == 0)↩
return SZ_OK;↩
↩
for (;;)↩
{↩
if (p->enc.needRead && !p->enc.readWasFinished)↩
{↩
size_t processed = p->enc.needReadSizeMax;↩
RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed));↩
p->enc.readPos += processed;↩
if (processed == 0)↩
{↩
p->enc.readWasFinished = True;↩
p->enc.isFinalFinished = True;↩
}↩
p->enc.needRead = False;↩
}↩
↩
*size = sizeOriginal;↩
RINOK(SbEnc_Read(&p->enc, data, size));↩
if (*size != 0 || !p->enc.needRead)↩
return SZ_OK;↩
}↩
}↩
↩
void SbEncInStream_Construct(CSbEncInStream *p, ISzAllocPtr alloc)↩
{↩
SbEnc_Construct(&p->enc, alloc);↩
p->vt.Read = SbEncInStream_Read;↩
}↩
↩
SRes SbEncInStream_Init(CSbEncInStream *p)↩
{↩
return SbEnc_Init(&p->enc);↩
}↩
↩
void SbEncInStream_Free(CSbEncInStream *p)↩
{↩
SbEnc_Free(&p->enc);↩
}↩
↩
#endif↩
↩
↩
↩
/* ---------- CXzProps ---------- */↩
↩
↩
void XzFilterProps_Init(CXzFilterProps *p)↩
{↩
p->id = 0;↩
p->delta = 0;↩
p->ip = 0;↩
p->ipDefined = False;↩
}↩
↩
void XzProps_Init(CXzProps *p)↩
{↩
p->checkId = XZ_CHECK_CRC32;↩
p->blockSize = XZ_PROPS__BLOCK_SIZE__AUTO;↩
p->numBlockThreads_Reduced = -1;↩
p->numBlockThreads_Max = -1;↩
p->numTotalThreads = -1;↩
p->reduceSize = (UInt64)(Int64)-1;↩
p->forceWriteSizesInHeader = 0;↩
// p->forceWriteSizesInHeader = 1;↩
↩
XzFilterProps_Init(&p->filterProps);↩
Lzma2EncProps_Init(&p->lzma2Props);↩
}↩
↩
↩
static void XzEncProps_Normalize_Fixed(CXzProps *p)↩
{↩
UInt64 fileSize;↩
int t1, t1n, t2, t2r, t3;↩
{↩
CLzma2EncProps tp = p->lzma2Props;↩
if (tp.numTotalThreads <= 0)↩
tp.numTotalThreads = p->numTotalThreads;↩
Lzma2EncProps_Normalize(&tp);↩
t1n = tp.numTotalThreads;↩
}↩
↩
t1 = p->lzma2Props.numTotalThreads;↩
t2 = p->numBlockThreads_Max;↩
t3 = p->numTotalThreads;↩
↩
if (t2 > MTCODER__THREADS_MAX)↩
t2 = MTCODER__THREADS_MAX;↩
↩
if (t3 <= 0)↩
{↩
if (t2 <= 0)↩
t2 = 1;↩
t3 = t1n * t2;↩
}↩
else if (t2 <= 0)↩
{↩
t2 = t3 / t1n;↩
if (t2 == 0)↩
{↩
t1 = 1;↩
t2 = t3;↩
}↩
if (t2 > MTCODER__THREADS_MAX)↩
t2 = MTCODER__THREADS_MAX;↩
}↩
else if (t1 <= 0)↩
{↩
t1 = t3 / t2;↩
if (t1 == 0)↩
t1 = 1;↩
}↩
else↩
t3 = t1n * t2;↩
↩
p->lzma2Props.numTotalThreads = t1;↩
↩
t2r = t2;↩
↩
fileSize = p->reduceSize;↩
↩
if ((p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1))↩
p->lzma2Props.lzmaProps.reduceSize = p->blockSize;↩
↩
Lzma2EncProps_Normalize(&p->lzma2Props);↩
↩
t1 = p->lzma2Props.numTotalThreads;↩
↩
{↩
if (t2 > 1 && fileSize != (UInt64)(Int64)-1)↩
{↩
UInt64 numBlocks = fileSize / p->blockSize;↩
if (numBlocks * p->blockSize != fileSize)↩
numBlocks++;↩
if (numBlocks < (unsigned)t2)↩
{↩
t2r = (unsigned)numBlocks;↩
if (t2r == 0)↩
t2r = 1;↩
t3 = t1 * t2r;↩
}↩
}↩
}↩
↩
p->numBlockThreads_Max = t2;↩
p->numBlockThreads_Reduced = t2r;↩
p->numTotalThreads = t3;↩
}↩
↩
↩
static void XzProps_Normalize(CXzProps *p)↩
{↩
/* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties.↩
Lzma2Enc_SetProps() will normalize lzma2Props later. */↩
↩
if (p->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID)↩
{↩
p->lzma2Props.lzmaProps.reduceSize = p->reduceSize;↩
p->numBlockThreads_Reduced = 1;↩
p->numBlockThreads_Max = 1;↩
if (p->lzma2Props.numTotalThreads <= 0)↩
p->lzma2Props.numTotalThreads = p->numTotalThreads;↩
return;↩
}↩
else↩
{↩
CLzma2EncProps *lzma2 = &p->lzma2Props;↩
if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO)↩
{↩
// xz-auto↩
p->lzma2Props.lzmaProps.reduceSize = p->reduceSize;↩
↩
if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID)↩
{↩
// if (xz-auto && lzma2-solid) - we use solid for both↩
p->blockSize = XZ_PROPS__BLOCK_SIZE__SOLID;↩
p->numBlockThreads_Reduced = 1;↩
p->numBlockThreads_Max = 1;↩
if (p->lzma2Props.numTotalThreads <= 0)↩
p->lzma2Props.numTotalThreads = p->numTotalThreads;↩
}↩
else↩
{↩
// if (xz-auto && (lzma2-auto || lzma2-fixed_)↩
// we calculate block size for lzma2 and use that block size for xz, lzma2 uses single-chunk per block↩
CLzma2EncProps tp = p->lzma2Props;↩
if (tp.numTotalThreads <= 0)↩
tp.numTotalThreads = p->numTotalThreads;↩
↩
Lzma2EncProps_Normalize(&tp);↩
↩
p->blockSize = tp.blockSize; // fixed or solid↩
p->numBlockThreads_Reduced = tp.numBlockThreads_Reduced;↩
p->numBlockThreads_Max = tp.numBlockThreads_Max;↩
if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO)↩
lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID↩
if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID)↩
lzma2->lzmaProps.reduceSize = tp.blockSize;↩
lzma2->numBlockThreads_Reduced = 1;↩
lzma2->numBlockThreads_Max = 1;↩
return;↩
}↩
}↩
else↩
{↩
// xz-fixed↩
// we can use xz::reduceSize or xz::blockSize as base for lzmaProps::reduceSize↩
↩
p->lzma2Props.lzmaProps.reduceSize = p->reduceSize;↩
{↩
UInt64 r = p->reduceSize;↩
if (r > p->blockSize || r == (UInt64)(Int64)-1)↩
r = p->blockSize;↩
lzma2->lzmaProps.reduceSize = r;↩
}↩
if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO)↩
lzma2->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID;↩
else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID)↩
lzma2->blockSize = p->blockSize;↩
↩
XzEncProps_Normalize_Fixed(p);↩
}↩
}↩
}↩
↩
↩
/* ---------- CLzma2WithFilters ---------- */↩
↩
typedef struct↩
{↩
CLzma2EncHandle lzma2;↩
CSeqInFilter filter;↩
↩
#ifdef USE_SUBBLOCK↩
CSbEncInStream sb;↩
#endif↩
} CLzma2WithFilters;↩
↩
↩
static void Lzma2WithFilters_Construct(CLzma2WithFilters *p)↩
{↩
p->lzma2 = NULL;↩
SeqInFilter_Construct(&p->filter);↩
↩
#ifdef USE_SUBBLOCK↩
SbEncInStream_Construct(&p->sb, alloc);↩
#endif↩
}↩
↩
↩
static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p, ISzAllocPtr alloc, ISzAllocPtr bigAlloc)↩
{↩
if (!p->lzma2)↩
{↩
p->lzma2 = Lzma2Enc_Create(alloc, bigAlloc);↩
if (!p->lzma2)↩
return SZ_ERROR_MEM;↩
}↩
return SZ_OK;↩
}↩
↩
↩
static void Lzma2WithFilters_Free(CLzma2WithFilters *p, ISzAllocPtr alloc)↩
{↩
#ifdef USE_SUBBLOCK↩
SbEncInStream_Free(&p->sb);↩
#endif↩
↩
SeqInFilter_Free(&p->filter, alloc);↩
if (p->lzma2)↩
{↩
Lzma2Enc_Destroy(p->lzma2);↩
p->lzma2 = NULL;↩
}↩
}↩
↩
↩
typedef struct↩
{↩
UInt64 unpackSize;↩
UInt64 totalSize;↩
size_t headerSize;↩
} CXzEncBlockInfo;↩
↩
↩
static SRes Xz_CompressBlock(↩
CLzma2WithFilters *lzmaf,↩
↩
ISeqOutStream *outStream,↩
Byte *outBufHeader,↩
Byte *outBufData, size_t outBufDataLimit,↩
↩
ISeqInStream *inStream,↩
// UInt64 expectedSize,↩
const Byte *inBuf, // used if (!inStream)↩
size_t inBufSize, // used if (!inStream), it's block size, props->blockSize is ignored↩
↩
const CXzProps *props,↩
ICompressProgress *progress,↩
int *inStreamFinished, /* only for inStream version */↩
CXzEncBlockInfo *blockSizes,↩
ISzAllocPtr alloc,↩
ISzAllocPtr allocBig)↩
{↩
CSeqCheckInStream checkInStream;↩
CSeqSizeOutStream seqSizeOutStream;↩
CXzBlock block;↩
unsigned filterIndex = 0;↩
CXzFilter *filter = NULL;↩
const CXzFilterProps *fp = &props->filterProps;↩
if (fp->id == 0)↩
fp = NULL;↩
↩
*inStreamFinished = False;↩
↩
RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig));↩
↩
RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props));↩
↩
XzBlock_ClearFlags(&block);↩
XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0));↩
↩
if (fp)↩
{↩
filter = &block.filters[filterIndex++];↩
filter->id = fp->id;↩
filter->propsSize = 0;↩
↩
if (fp->id == XZ_ID_Delta)↩
{↩
filter->props[0] = (Byte)(fp->delta - 1);↩
filter->propsSize = 1;↩
}↩
else if (fp->ipDefined)↩
{↩
SetUi32(filter->props, fp->ip);↩
filter->propsSize = 4;↩
}↩
}↩
↩
{↩
CXzFilter *f = &block.filters[filterIndex++];↩
f->id = XZ_ID_LZMA2;↩
f->propsSize = 1;↩
f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);↩
}↩
↩
seqSizeOutStream.vt.Write = SeqSizeOutStream_Write;↩
seqSizeOutStream.realStream = outStream;↩
seqSizeOutStream.outBuf = outBufData;↩
seqSizeOutStream.outBufLimit = outBufDataLimit;↩
seqSizeOutStream.processed = 0;↩
↩
/*↩
if (expectedSize != (UInt64)(Int64)-1)↩
{↩
block.unpackSize = expectedSize;↩
if (props->blockSize != (UInt64)(Int64)-1)↩
if (expectedSize > props->blockSize)↩
block.unpackSize = props->blockSize;↩
XzBlock_SetHasUnpackSize(&block);↩
}↩
*/↩
↩
if (outStream)↩
{↩
RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt));↩
}↩
↩
checkInStream.vt.Read = SeqCheckInStream_Read;↩
SeqCheckInStream_Init(&checkInStream, props->checkId);↩
↩
checkInStream.realStream = inStream;↩
checkInStream.data = inBuf;↩
checkInStream.limit = props->blockSize;↩
if (!inStream)↩
checkInStream.limit = inBufSize;↩
↩
if (fp)↩
{↩
#ifdef USE_SUBBLOCK↩
if (fp->id == XZ_ID_Subblock)↩
{↩
lzmaf->sb.inStream = &checkInStream.vt;↩
RINOK(SbEncInStream_Init(&lzmaf->sb));↩
}↩
else↩
#endif↩
{↩
lzmaf->filter.realStream = &checkInStream.vt;↩
RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc));↩
}↩
}↩
↩
{↩
SRes res;↩
Byte *outBuf = NULL;↩
size_t outSize = 0;↩
Bool useStream = (fp || inStream);↩
// useStream = True;↩
↩
if (!useStream)↩
{↩
XzCheck_Update(&checkInStream.check, inBuf, inBufSize);↩
checkInStream.processed = inBufSize;↩
}↩
↩
if (!outStream)↩
{↩
outBuf = seqSizeOutStream.outBuf; // + (size_t)seqSizeOutStream.processed;↩
outSize = seqSizeOutStream.outBufLimit; // - (size_t)seqSizeOutStream.processed;↩
}↩
↩
res = Lzma2Enc_Encode2(lzmaf->lzma2,↩
outBuf ? NULL : &seqSizeOutStream.vt,↩
outBuf,↩
outBuf ? &outSize : NULL,↩
↩
useStream ?↩
(fp ?↩
(↩
#ifdef USE_SUBBLOCK↩
(fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt:↩
#endif↩
&lzmaf->filter.p) :↩
&checkInStream.vt) : NULL,↩
↩
useStream ? NULL : inBuf,↩
useStream ? 0 : inBufSize,↩
↩
progress);↩
↩
if (outBuf)↩
seqSizeOutStream.processed += outSize;↩
↩
RINOK(res);↩
blockSizes->unpackSize = checkInStream.processed;↩
}↩
{↩
Byte buf[4 + 64];↩
unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed);↩
UInt64 packSize = seqSizeOutStream.processed;↩
↩
buf[0] = 0;↩
buf[1] = 0;↩
buf[2] = 0;↩
buf[3] = 0;↩
↩
SeqCheckInStream_GetDigest(&checkInStream, buf + 4);↩
RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId)));↩
↩
blockSizes->totalSize = seqSizeOutStream.processed - padSize;↩
↩
if (!outStream)↩
{↩
seqSizeOutStream.outBuf = outBufHeader;↩
seqSizeOutStream.outBufLimit = XZ_BLOCK_HEADER_SIZE_MAX;↩
seqSizeOutStream.processed = 0;↩
↩
block.unpackSize = blockSizes->unpackSize;↩
XzBlock_SetHasUnpackSize(&block);↩
↩
block.packSize = packSize;↩
XzBlock_SetHasPackSize(&block);↩
↩
RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt));↩
↩
blockSizes->headerSize = (size_t)seqSizeOutStream.processed;↩
blockSizes->totalSize += seqSizeOutStream.processed;↩
}↩
}↩
↩
if (inStream)↩
*inStreamFinished = checkInStream.realStreamFinished;↩
else↩
{↩
*inStreamFinished = False;↩
if (checkInStream.processed != inBufSize)↩
return SZ_ERROR_FAIL;↩
}↩
↩
return SZ_OK;↩
}↩
↩
↩
↩
typedef struct↩
{↩
ICompressProgress vt;↩
ICompressProgress *progress;↩
UInt64 inOffset;↩
UInt64 outOffset;↩
} CCompressProgress_XzEncOffset;↩
↩
↩
static SRes CompressProgress_XzEncOffset_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize)↩
{↩
const CCompressProgress_XzEncOffset *p = CONTAINER_FROM_VTBL(pp, CCompressProgress_XzEncOffset, vt);↩
inSize += p->inOffset;↩
outSize += p->outOffset;↩
return ICompressProgress_Progress(p->progress, inSize, outSize);↩
}↩
↩
↩
↩
↩
typedef struct↩
{↩
ISzAllocPtr alloc;↩
ISzAllocPtr allocBig;↩
↩
CXzProps xzProps;↩
UInt64 expectedDataSize;↩
↩
CXzEncIndex xzIndex;↩
↩
CLzma2WithFilters lzmaf_Items[MTCODER__THREADS_MAX];↩
↩
size_t outBufSize; /* size of allocated outBufs[i] */↩
Byte *outBufs[MTCODER__BLOCKS_MAX];↩
↩
#ifndef _7ZIP_ST↩
unsigned checkType;↩
ISeqOutStream *outStream;↩
Bool mtCoder_WasConstructed;↩
CMtCoder mtCoder;↩
CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX];↩
#endif↩
↩
} CXzEnc;↩
↩
↩
static void XzEnc_Construct(CXzEnc *p)↩
{↩
unsigned i;↩
↩
XzEncIndex_Construct(&p->xzIndex);↩
↩
for (i = 0; i < MTCODER__THREADS_MAX; i++)↩
Lzma2WithFilters_Construct(&p->lzmaf_Items[i]);↩
↩
#ifndef _7ZIP_ST↩
p->mtCoder_WasConstructed = False;↩
{↩
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)↩
p->outBufs[i] = NULL;↩
p->outBufSize = 0;↩
}↩
#endif↩
}↩
↩
↩
static void XzEnc_FreeOutBufs(CXzEnc *p)↩
{↩
unsigned i;↩
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)↩
if (p->outBufs[i])↩
{↩
ISzAlloc_Free(p->alloc, p->outBufs[i]);↩
p->outBufs[i] = NULL;↩
}↩
p->outBufSize = 0;↩
}↩
↩
↩
static void XzEnc_Free(CXzEnc *p, ISzAllocPtr alloc)↩
{↩
unsigned i;↩
↩
XzEncIndex_Free(&p->xzIndex, alloc);↩
↩
for (i = 0; i < MTCODER__THREADS_MAX; i++)↩
Lzma2WithFilters_Free(&p->lzmaf_Items[i], alloc);↩
↩
#ifndef _7ZIP_ST↩
if (p->mtCoder_WasConstructed)↩
{↩
MtCoder_Destruct(&p->mtCoder);↩
p->mtCoder_WasConstructed = False;↩
}↩
XzEnc_FreeOutBufs(p);↩
#endif↩
}↩
↩
↩
CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig)↩
{↩
CXzEnc *p = (CXzEnc *)ISzAlloc_Alloc(alloc, sizeof(CXzEnc));↩
if (!p)↩
return NULL;↩
XzEnc_Construct(p);↩
XzProps_Init(&p->xzProps);↩
XzProps_Normalize(&p->xzProps);↩
p->expectedDataSize = (UInt64)(Int64)-1;↩
p->alloc = alloc;↩
p->allocBig = allocBig;↩
return p;↩
}↩
↩
↩
void XzEnc_Destroy(CXzEncHandle pp)↩
{↩
CXzEnc *p = (CXzEnc *)pp;↩
XzEnc_Free(p, p->alloc);↩
ISzAlloc_Free(p->alloc, p);↩
}↩
↩
↩
SRes XzEnc_SetProps(CXzEncHandle pp, const CXzProps *props)↩
{↩
CXzEnc *p = (CXzEnc *)pp;↩
p->xzProps = *props;↩
XzProps_Normalize(&p->xzProps);↩
return SZ_OK;↩
}↩
↩
↩
void XzEnc_SetDataSize(CXzEncHandle pp, UInt64 expectedDataSiize)↩
{↩
CXzEnc *p = (CXzEnc *)pp;↩
p->expectedDataSize = expectedDataSiize;↩
}↩
↩
↩
↩
↩
#ifndef _7ZIP_ST↩
↩
static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex,↩
const Byte *src, size_t srcSize, int finished)↩
{↩
CXzEnc *me = (CXzEnc *)pp;↩
SRes res;↩
CMtProgressThunk progressThunk;↩
↩
Byte *dest = me->outBufs[outBufIndex];↩
↩
UNUSED_VAR(finished)↩
↩
{↩
CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex];↩
bInfo->totalSize = 0;↩
bInfo->unpackSize = 0;↩
bInfo->headerSize = 0;↩
}↩
↩
if (!dest)↩
{↩
dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize);↩
if (!dest)↩
return SZ_ERROR_MEM;↩
me->outBufs[outBufIndex] = dest;↩
}↩
↩
MtProgressThunk_CreateVTable(&progressThunk);↩
progressThunk.mtProgress = &me->mtCoder.mtProgress;↩
MtProgressThunk_Init(&progressThunk);↩
↩
{↩
CXzEncBlockInfo blockSizes;↩
int inStreamFinished;↩
↩
res = Xz_CompressBlock(↩
&me->lzmaf_Items[coderIndex],↩
↩
NULL,↩
dest,↩
dest + XZ_BLOCK_HEADER_SIZE_MAX, me->outBufSize - XZ_BLOCK_HEADER_SIZE_MAX,↩
↩
NULL,↩
// srcSize, // expectedSize↩
src, srcSize,↩
↩
&me->xzProps,↩
&progressThunk.vt,↩
&inStreamFinished,↩
&blockSizes,↩
me->alloc,↩
me->allocBig);↩
↩
if (res == SZ_OK)↩
me->EncBlocks[outBufIndex] = blockSizes;↩
↩
return res;↩
}↩
}↩
↩
↩
static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex)↩
{↩
CXzEnc *me = (CXzEnc *)pp;↩
↩
const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex];↩
const Byte *data = me->outBufs[outBufIndex];↩
↩
RINOK(WriteBytes(me->outStream, data, bInfo->headerSize));↩
↩
{↩
UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize);↩
RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize));↩
}↩
↩
return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc);↩
}↩
↩
#endif↩
↩
↩
↩
SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)↩
{↩
CXzEnc *p = (CXzEnc *)pp;↩
↩
const CXzProps *props = &p->xzProps;↩
↩
XzEncIndex_Init(&p->xzIndex);↩
{↩
UInt64 numBlocks = 1;↩
UInt64 blockSize = props->blockSize;↩
↩
if (blockSize != XZ_PROPS__BLOCK_SIZE__SOLID↩
&& props->reduceSize != (UInt64)(Int64)-1)↩
{↩
numBlocks = props->reduceSize / blockSize;↩
if (numBlocks * blockSize != props->reduceSize)↩
numBlocks++;↩
}↩
else↩
blockSize = (UInt64)1 << 62;↩
↩
RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc));↩
}↩
↩
RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream));↩
↩
↩
#ifndef _7ZIP_ST↩
if (props->numBlockThreads_Reduced > 1)↩
{↩
IMtCoderCallback2 vt;↩
↩
if (!p->mtCoder_WasConstructed)↩
{↩
p->mtCoder_WasConstructed = True;↩
MtCoder_Construct(&p->mtCoder);↩
}↩
↩
vt.Code = XzEnc_MtCallback_Code;↩
vt.Write = XzEnc_MtCallback_Write;↩
↩
p->checkType = props->checkId;↩
p->xzProps = *props;↩
↩
p->outStream = outStream;↩
↩
p->mtCoder.allocBig = p->allocBig;↩
p->mtCoder.progress = progress;↩
p->mtCoder.inStream = inStream;↩
p->mtCoder.inData = NULL;↩
p->mtCoder.inDataSize = 0;↩
p->mtCoder.mtCallback = &vt;↩
p->mtCoder.mtCallbackObject = p;↩
↩
if ( props->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID↩
|| props->blockSize == XZ_PROPS__BLOCK_SIZE__AUTO)↩
return SZ_ERROR_FAIL;↩
↩
p->mtCoder.blockSize = (size_t)props->blockSize;↩
if (p->mtCoder.blockSize != props->blockSize)↩
return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */↩
↩
{↩
size_t destBlockSize = XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(p->mtCoder.blockSize);↩
if (destBlockSize < p->mtCoder.blockSize)↩
return SZ_ERROR_PARAM;↩
if (p->outBufSize != destBlockSize)↩
XzEnc_FreeOutBufs(p);↩
p->outBufSize = destBlockSize;↩
}↩
↩
p->mtCoder.numThreadsMax = props->numBlockThreads_Max;↩
p->mtCoder.expectedDataSize = p->expectedDataSize;↩
↩
RINOK(MtCoder_Code(&p->mtCoder));↩
}↩
else↩
#endif↩
{↩
int writeStartSizes;↩
CCompressProgress_XzEncOffset progress2;↩
Byte *bufData = NULL;↩
size_t bufSize = 0;↩
↩
progress2.vt.Progress = CompressProgress_XzEncOffset_Progress;↩
progress2.inOffset = 0;↩
progress2.outOffset = 0;↩
progress2.progress = progress;↩
↩
writeStartSizes = 0;↩
↩
if (props->blockSize != XZ_PROPS__BLOCK_SIZE__SOLID)↩
{↩
writeStartSizes = (props->forceWriteSizesInHeader > 0);↩
↩
if (writeStartSizes)↩
{↩
size_t t2;↩
size_t t = (size_t)props->blockSize;↩
if (t != props->blockSize)↩
return SZ_ERROR_PARAM;↩
t = XZ_GET_MAX_BLOCK_PACK_SIZE(t);↩
if (t < props->blockSize)↩
return SZ_ERROR_PARAM;↩
t2 = XZ_BLOCK_HEADER_SIZE_MAX + t;↩
if (!p->outBufs[0] || t2 != p->outBufSize)↩
{↩
XzEnc_FreeOutBufs(p);↩
p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2);↩
if (!p->outBufs[0])↩
return SZ_ERROR_MEM;↩
p->outBufSize = t2;↩
}↩
bufData = p->outBufs[0] + XZ_BLOCK_HEADER_SIZE_MAX;↩
bufSize = t;↩
}↩
}↩
↩
for (;;)↩
{↩
CXzEncBlockInfo blockSizes;↩
int inStreamFinished;↩
↩
/*↩
UInt64 rem = (UInt64)(Int64)-1;↩
if (props->reduceSize != (UInt64)(Int64)-1↩
&& props->reduceSize >= progress2.inOffset)↩
rem = props->reduceSize - progress2.inOffset;↩
*/↩
↩
blockSizes.headerSize = 0; // for GCC↩
↩
RINOK(Xz_CompressBlock(↩
&p->lzmaf_Items[0],↩
↩
writeStartSizes ? NULL : outStream,↩
writeStartSizes ? p->outBufs[0] : NULL,↩
bufData, bufSize,↩
↩
inStream,↩
// rem,↩
NULL, 0,↩
↩
props,↩
progress ? &progress2.vt : NULL,↩
&inStreamFinished,↩
&blockSizes,↩
p->alloc,↩
p->allocBig));↩
↩
{↩
UInt64 totalPackFull = blockSizes.totalSize + XZ_GET_PAD_SIZE(blockSizes.totalSize);↩
↩
if (writeStartSizes)↩
{↩
RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize));↩
RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize));↩
}↩
↩
RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc));↩
↩
progress2.inOffset += blockSizes.unpackSize;↩
progress2.outOffset += totalPackFull;↩
}↩
↩
if (inStreamFinished)↩
break;↩
}↩
}↩
↩
return XzEncIndex_WriteFooter(&p->xzIndex, (CXzStreamFlags)props->checkId, outStream);↩
}↩
↩
↩
#include "Alloc.h"↩
↩
SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,↩
const CXzProps *props, ICompressProgress *progress)↩
{↩
SRes res;↩
CXzEncHandle xz = XzEnc_Create(&g_Alloc, &g_BigAlloc);↩
if (!xz)↩
return SZ_ERROR_MEM;↩
res = XzEnc_SetProps(xz, props);↩
if (res == SZ_OK)↩
res = XzEnc_Encode(xz, outStream, inStream, progress);↩
XzEnc_Destroy(xz);↩
return res;↩
}↩
↩
↩
SRes Xz_EncodeEmpty(ISeqOutStream *outStream)↩
{↩
SRes res;↩
CXzEncIndex xzIndex;↩
XzEncIndex_Construct(&xzIndex);↩
res = Xz_WriteHeader((CXzStreamFlags)0, outStream);↩
if (res == SZ_OK)↩
res = XzEncIndex_WriteFooter(&xzIndex, (CXzStreamFlags)0, outStream);↩
XzEncIndex_Free(&xzIndex, NULL); // g_Alloc↩
return res;↩
}↩