...
 
Commits (120)
......@@ -188,3 +188,4 @@ a4e32fd539ab41489a51b2aa88bda9a73b839562 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncg
59338f9561099de77c684c00f76507f11e46ebe8 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncgkqlvdUFAl2ty1MQHHJhZkBkdXJpbjQyLmNvbQAKCRC5ydyCSqW91XBUD/wJqwW0cuMCUvuUODLIfWa7ZxNl1mV9eW3tFQEuLGry97s12KDwBe0Erdjj7DASl4/6Xpc4PYxelZwSw4xT1UQg7wd/C3daCq/cDXrAkl7ZNTAHu6iAnHh25mOpIBfhMbh4j3YD0A2OoI17QGScU6S7Uv0Gz1CY20lJmEqsMzuuDPm2zrdPnTWffRUuPgskAg3czaw45Na7nUBeaxN1On0O5WqMYZsCGyi14g5S0Z0LHMKRJzc/s48JUTDjTbbzJ6HBxrxWTW2v8gN2J6QDYykcLBB9kV6laal9jhWs9n/w0yWwHfBfJ+E4EiMXeRdZgGA55OCOuDxnmmONs1/Z0WwPo+vQlowEnjDMT0jPrPePZ5P4BDXZD3tGsmdXDHM7j+VfDyPh1FBFpcaej44t84X1OWtAnLZ3VMPLwobz9MOzz4wr9UuHq23hus0Fen+FJYOAlTx9qPAqBrCTpGl+h1DMKD62D7lF8Z1CxTlqg9PPBB7IZNCXoN7FZ4Wfhv1AarMVNNUgBx6m0r6OScCXrluuFklYDSIZrfgiwosXxsHW27RjxktrV4O+J1GT/chLBJFViTZg/gX/9UC3eLkzp1t6gC6T9SQ+lq0/I+1/rHQkxNaywLycBPOG1yb/59mibEwB9+Mu9anRYKFNHEktNoEmyw5G9UoZhD+1tHt4tkJCwA==
ca3dca416f8d5863ca6f5a4a6a6bb835dcd5feeb 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncgkqlvdUFAl3BrQ4QHHJhZkBkdXJpbjQyLmNvbQAKCRC5ydyCSqW91ZXjEACfBdZczf0a4bmeaaxRwxXAniSS4rVkF790g22fsvSZFvQEpmwqNtsvbTt3N1V2QSDSZyhBa+/qfpuZ689VXMlR3rcJOVjo/7193QLXHOPfRn7sDeeCxjsbtXXLbLa8UT56gtT5gUa4i0LC2kHBEi+UhV9EGgSaDTBxWUFJ9RY2sosy1XFiOUlkUoHUbqUF28J3/CxEXzULWkqTOPwh94JYsgXSSS69WNZEfsuEBSPCzn8Gd7z7lWudZ/VTZBTpTji7HQxpFtSZxNzpwmcmVOH9HlEKoA1K4JoR+1TMHqSytQXlz3FMF6c6Z1G+OPpwTGCjGTkB9ZAusP3gU8KIZTTEXthiEluRtnRq1yu4K2LTyY172JPJvANAWpVEvBvn4k5c9tDOEt9RCAPqCrgNGzDTrw02+gZyyNkjcS6hPn+cDJ6OQ1j2eCQtHlqfHLSc7FsRjUSTiKSEUTdWvHbNfOYe6Yth/tnQ7TnpnS9S0eiugFzZs2f8P85Gfa3uTFQIDm67Ud+8Yu1uOxa6bhECLaXEACnLofzz8sioLsJMiOoG2HmwhyPyfZUHXlb2zdsSP3LC+gKN39VvzSxhhjrIUJoM4ulP0GP1/lkMVzOady66iLaEwDvEn4FLmu395SubHwbre1Jx83hiCQpZfPkI0PhKnh4yVm+BRGUpX97rMTGjzw==
a50fecefa691c9b72a99e49aa6fe9dd13943c2bf 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncgkqlvdUFAl3pEYIQHHJhZkBkdXJpbjQyLmNvbQAKCRC5ydyCSqW91duiD/9fwJbyrXXdpoBCeW3pgiz/xKZRQq0N3UqC/5m3PGl2qPfDqTi1GA6J+O24Cpy/FXYLEKlrEG2jy/iBZnGgTpb2sgycHFlWCT7VbuS8SDE3FFloTE8ZOGy5eJRo1UXYu4vsvNtmarN1xJQPrVK4l/Co5XWXFx15H/oMXLaHzS0kzQ/rHsMr7UXM0QwtmLC0S9IMetg5EUQx9GtHHaRnh1PIyP5NxP9VQ9RK4hmT6F2g60bcsMfpgF0I/RgL3tcdUn1RNIZ2OXHBhKYL+xOUe+wadDPIyPDqLXNEqPH7xqi0MQm/jOG++AvUPM7AdVc9Y2eRFOIIBIY0nkU5LL4yVVdqoc8kgwz14xhJXGTpMDRD54F6WrQtxhbHcb+JF7QDe3i9wI1LvurW4IIA5e4DC1q9yKKxNx9cDUOMF5q9ehiW9V120LTXJnYOUwfB7D4bIhe2mpOw8yYABU3gZ0Q6iVBTH+9rZYZ9TETX6vkf/DnJXteo39OhKrZ1Z4Gj6MSAjPJLARnYGnRMgvsyHSbV0TsGA4tdEaBs3dZmUV7maxLbs70sO6r9WwUY37TcYYHGdRplD9AreDLcxvjXA73Iluoy9WBGxRWF8wftQjaE9XR4KkDFrAoqqYZwN2AwHiTjVD1lQx+xvxZeEQ3ZBDprH3Uy6TwqUo5jbvHgR2+HqaZlTg==
b4c82b70418022e67cc0e69b1aa3c3aa43aa1d29 0 iQJEBAABCAAuFiEEK8zhT1xnJaouqK63ucncgkqlvdUFAl4TkWgQHHJhZkBkdXJpbjQyLmNvbQAKCRC5ydyCSqW91aV6D/4xzlluOwsBhLXWUi7bDp4HtYnyDhq4XuDORAMO5mCZ7I7J6uqGoViqH4AhXoo3yPp1cDiRzzl172xpec38uTL8C5zHhARKuAl5Pn1A8rYORvYzT9nsDh4MAtfTokhg81awRzhun9xtPUT2nETAOgampW0g7r241MSR1j0myAkC7zqO3yf+1rYo7kiv7fh+74MkrSn4HEmEaLsI5gW05tFR+ip6vpm6eikFinqeVJegDCuyTPMvH0D9ZeBNlyoOfdEd6DDYsWvWAmLSO9FGbb03R5aOFRp7RmQRFH/qcueeePa/9Z1zO+YyCeBy0wvWCkjfLMY99HhNhdNfy/qC/69V5RGQYvaapy6BEAi4eCH73hsxzCQpKopUl9VrpwhNasJ41KWc90RsPO91bkTdDddF7e2qjq762aNgm7ysEzIHMgSsMgsE9w8hz70RE7bk/gYn26ak3XP4nCOY0OJQ8mgaElN/FP1kxqqT7MM7WeMiNMFTD1gvWwEAu9Y47AwUedkTrykQsAFzc+CyaIaW+/Kuyv0j5E7v8zAcVTTX4xIyqR4yL2Nwe1rYE4MZgs0L9gQ3rcdyft6899gAiiq96MPR3gLJUPbBz2azH/e0CzNXvDJa39jIm2ez0qC7c88NhTKhFjHE9EW5GI3g8mhS5dJXCnUSq4spgtrJdfGenL3vLw==
......@@ -201,3 +201,4 @@ a4e32fd539ab41489a51b2aa88bda9a73b839562 5.1.1
59338f9561099de77c684c00f76507f11e46ebe8 5.2rc0
ca3dca416f8d5863ca6f5a4a6a6bb835dcd5feeb 5.2
a50fecefa691c9b72a99e49aa6fe9dd13943c2bf 5.2.1
b4c82b70418022e67cc0e69b1aa3c3aa43aa1d29 5.2.2
......@@ -3,100 +3,9 @@
mercurial/cext/manifest.c
mercurial/cext/osutil.c
# Vendored code that we should never format:
contrib/python-zstandard/c-ext/bufferutil.c
contrib/python-zstandard/c-ext/compressionchunker.c
contrib/python-zstandard/c-ext/compressiondict.c
contrib/python-zstandard/c-ext/compressionparams.c
contrib/python-zstandard/c-ext/compressionreader.c
contrib/python-zstandard/c-ext/compressionwriter.c
contrib/python-zstandard/c-ext/compressobj.c
contrib/python-zstandard/c-ext/compressor.c
contrib/python-zstandard/c-ext/compressoriterator.c
contrib/python-zstandard/c-ext/constants.c
contrib/python-zstandard/c-ext/decompressionreader.c
contrib/python-zstandard/c-ext/decompressionwriter.c
contrib/python-zstandard/c-ext/decompressobj.c
contrib/python-zstandard/c-ext/decompressor.c
contrib/python-zstandard/c-ext/decompressoriterator.c
contrib/python-zstandard/c-ext/frameparams.c
contrib/python-zstandard/c-ext/python-zstandard.h
contrib/python-zstandard/zstd.c
contrib/python-zstandard/zstd/common/bitstream.h
contrib/python-zstandard/zstd/common/compiler.h
contrib/python-zstandard/zstd/common/cpu.h
contrib/python-zstandard/zstd/common/debug.c
contrib/python-zstandard/zstd/common/debug.h
contrib/python-zstandard/zstd/common/entropy_common.c
contrib/python-zstandard/zstd/common/error_private.c
contrib/python-zstandard/zstd/common/error_private.h
contrib/python-zstandard/zstd/common/fse_decompress.c
contrib/python-zstandard/zstd/common/fse.h
contrib/python-zstandard/zstd/common/huf.h
contrib/python-zstandard/zstd/common/mem.h
contrib/python-zstandard/zstd/common/pool.c
contrib/python-zstandard/zstd/common/pool.h
contrib/python-zstandard/zstd/common/threading.c
contrib/python-zstandard/zstd/common/threading.h
contrib/python-zstandard/zstd/common/xxhash.c
contrib/python-zstandard/zstd/common/xxhash.h
contrib/python-zstandard/zstd/common/zstd_common.c
contrib/python-zstandard/zstd/common/zstd_errors.h
contrib/python-zstandard/zstd/common/zstd_internal.h
contrib/python-zstandard/zstd/compress/fse_compress.c
contrib/python-zstandard/zstd/compress/hist.c
contrib/python-zstandard/zstd/compress/hist.h
contrib/python-zstandard/zstd/compress/huf_compress.c
contrib/python-zstandard/zstd/compress/zstd_compress.c
contrib/python-zstandard/zstd/compress/zstd_compress_internal.h
contrib/python-zstandard/zstd/compress/zstd_compress_literals.c
contrib/python-zstandard/zstd/compress/zstd_compress_literals.h
contrib/python-zstandard/zstd/compress/zstd_compress_sequences.c
contrib/python-zstandard/zstd/compress/zstd_compress_sequences.h
contrib/python-zstandard/zstd/compress/zstd_double_fast.c
contrib/python-zstandard/zstd/compress/zstd_double_fast.h
contrib/python-zstandard/zstd/compress/zstd_fast.c
contrib/python-zstandard/zstd/compress/zstd_fast.h
contrib/python-zstandard/zstd/compress/zstd_lazy.c
contrib/python-zstandard/zstd/compress/zstd_lazy.h
contrib/python-zstandard/zstd/compress/zstd_ldm.c
contrib/python-zstandard/zstd/compress/zstd_ldm.h
contrib/python-zstandard/zstd/compress/zstdmt_compress.c
contrib/python-zstandard/zstd/compress/zstdmt_compress.h
contrib/python-zstandard/zstd/compress/zstd_opt.c
contrib/python-zstandard/zstd/compress/zstd_opt.h
contrib/python-zstandard/zstd/decompress/huf_decompress.c
contrib/python-zstandard/zstd/decompress/zstd_ddict.c
contrib/python-zstandard/zstd/decompress/zstd_ddict.h
contrib/python-zstandard/zstd/decompress/zstd_decompress_block.c
contrib/python-zstandard/zstd/decompress/zstd_decompress_block.h
contrib/python-zstandard/zstd/decompress/zstd_decompress_internal.h
contrib/python-zstandard/zstd/decompress/zstd_decompress.c
contrib/python-zstandard/zstd/deprecated/zbuff_common.c
contrib/python-zstandard/zstd/deprecated/zbuff_compress.c
contrib/python-zstandard/zstd/deprecated/zbuff_decompress.c
contrib/python-zstandard/zstd/deprecated/zbuff.h
contrib/python-zstandard/zstd/dictBuilder/cover.c
contrib/python-zstandard/zstd/dictBuilder/cover.h
contrib/python-zstandard/zstd/dictBuilder/divsufsort.c
contrib/python-zstandard/zstd/dictBuilder/divsufsort.h
contrib/python-zstandard/zstd/dictBuilder/fastcover.c
contrib/python-zstandard/zstd/dictBuilder/zdict.c
contrib/python-zstandard/zstd/dictBuilder/zdict.h
contrib/python-zstandard/zstd/zstd.h
hgext/fsmonitor/pywatchman/bser.c
mercurial/thirdparty/xdiff/xdiff.h
mercurial/thirdparty/xdiff/xdiffi.c
mercurial/thirdparty/xdiff/xdiffi.h
mercurial/thirdparty/xdiff/xemit.c
mercurial/thirdparty/xdiff/xemit.h
mercurial/thirdparty/xdiff/xhistogram.c
mercurial/thirdparty/xdiff/xinclude.h
mercurial/thirdparty/xdiff/xmacros.h
mercurial/thirdparty/xdiff/xmerge.c
mercurial/thirdparty/xdiff/xpatience.c
mercurial/thirdparty/xdiff/xprepare.c
mercurial/thirdparty/xdiff/xprepare.h
mercurial/thirdparty/xdiff/xtypes.h
mercurial/thirdparty/xdiff/xutils.c
mercurial/thirdparty/xdiff/xutils.h
mercurial/thirdparty/zope/interface/_zope_interface_coptimizations.c
syntax: glob
contrib/python-zstandard/**.c
contrib/python-zstandard/**.h
hgext/fsmonitor/pywatchman/**.c
mercurial/thirdparty/**.c
mercurial/thirdparty/**.h
[fix]
clang-format:command = clang-format --style file -i
clang-format:pattern = (**.c or **.cc or **.h) and not "listfile:contrib/clang-format-ignorelist"
clang-format:command = clang-format --style file
clang-format:pattern = set:(**.c or **.cc or **.h) and not "include:contrib/clang-format-ignorelist"
rustfmt:command = rustfmt
rustfmt:pattern = set:**.rs
......
......@@ -10,7 +10,8 @@ RUN apt-get update && apt-get install -y \
dh-python \
less \
python \
python-all-dev \
python-docutils \
python3-all \
python3-all-dev \
python3-docutils \
unzip \
zip
......@@ -68,79 +68,42 @@ begin
for d := 0 to GetArrayLength(pathdir)-1 do begin
updatepath := true;
// Modify WinNT path
if UsingWinNT() = true then begin
// Get current path, split into an array
RegQueryStringValue(regroot, regpath, 'Path', oldpath);
oldpath := oldpath + ';';
i := 0;
while (Pos(';', oldpath) > 0) do begin
SetArrayLength(pathArr, i+1);
pathArr[i] := Copy(oldpath, 0, Pos(';', oldpath)-1);
oldpath := Copy(oldpath, Pos(';', oldpath)+1, Length(oldpath));
i := i + 1;
// Check if current directory matches app dir
if pathdir[d] = pathArr[i-1] then begin
// if uninstalling, remove dir from path
if IsUninstaller() = true then begin
continue;
// if installing, flag that dir already exists in path
end else begin
updatepath := false;
end;
end;
// Get current path, split into an array
RegQueryStringValue(regroot, regpath, 'Path', oldpath);
oldpath := oldpath + ';';
i := 0;
while (Pos(';', oldpath) > 0) do begin
SetArrayLength(pathArr, i+1);
pathArr[i] := Copy(oldpath, 0, Pos(';', oldpath)-1);
oldpath := Copy(oldpath, Pos(';', oldpath)+1, Length(oldpath));
i := i + 1;
// Add current directory to new path
if i = 1 then begin
newpath := pathArr[i-1];
// Check if current directory matches app dir
if pathdir[d] = pathArr[i-1] then begin
// if uninstalling, remove dir from path
if IsUninstaller() = true then begin
continue;
// if installing, flag that dir already exists in path
end else begin
newpath := newpath + ';' + pathArr[i-1];
updatepath := false;
end;
end;
// Append app dir to path if not already included
if (IsUninstaller() = false) AND (updatepath = true) then
newpath := newpath + ';' + pathdir[d];
// Write new path
RegWriteStringValue(regroot, regpath, 'Path', newpath);
// Modify Win9x path
end else begin
// Convert to shortened dirname
pathdir[d] := GetShortName(pathdir[d]);
// If autoexec.bat exists, check if app dir already exists in path
aExecFile := 'C:\AUTOEXEC.BAT';
if FileExists(aExecFile) then begin
LoadStringsFromFile(aExecFile, aExecArr);
for i := 0 to GetArrayLength(aExecArr)-1 do begin
if IsUninstaller() = false then begin
// If app dir already exists while installing, skip add
if (Pos(pathdir[d], aExecArr[i]) > 0) then
updatepath := false;
break;
end else begin
// If app dir exists and = what we originally set, then delete at uninstall
if aExecArr[i] = 'SET PATH=%PATH%;' + pathdir[d] then
aExecArr[i] := '';
end;
end;
end;
// If app dir not found, or autoexec.bat didn't exist, then (create and) append to current path
if (IsUninstaller() = false) AND (updatepath = true) then begin
SaveStringToFile(aExecFile, #13#10 + 'SET PATH=%PATH%;' + pathdir[d], True);
// If uninstalling, write the full autoexec out
// Add current directory to new path
if i = 1 then begin
newpath := pathArr[i-1];
end else begin
SaveStringsToFile(aExecFile, aExecArr, False);
newpath := newpath + ';' + pathArr[i-1];
end;
end;
// Append app dir to path if not already included
if (IsUninstaller() = false) AND (updatepath = true) then
newpath := newpath + ';' + pathdir[d];
// Write new path
RegWriteStringValue(regroot, regpath, 'Path', newpath);
end;
end;
......@@ -207,13 +170,6 @@ begin
end;
function NeedRestart(): Boolean;
var
taskname: String;
begin
taskname := ModPathName;
if IsTaskSelected(taskname) and not UsingWinNT() then begin
Result := True;
end else begin
Result := False;
end;
Result := False;
end;
......@@ -1878,7 +1878,6 @@ def perftemplating(ui, repo, testedtemplate=None, **opts):
def _displaystats(ui, opts, entries, data):
pass
# use a second formatter because the data are quite different, not sure
# how it flies with the templater.
fm = ui.formatter(b'perf-stats', opts)
......@@ -2048,8 +2047,8 @@ def perfhelpermergecopies(ui, repo, revs=[], **opts):
data['p1.time'] = end - begin
begin = util.timer()
p2renames = copies.pathcopies(b, p2)
data['p2.time'] = end - begin
end = util.timer()
data['p2.time'] = end - begin
data['p1.renamedfiles'] = len(p1renames)
data['p2.renamedfiles'] = len(p2renames)
......@@ -2221,9 +2220,6 @@ def perfhelperpathcopies(ui, repo, revs=[], **opts):
fm.end()
if dostats:
# use a second formatter because the data are quite different, not sure
# how it flies with the templater.
fm = ui.formatter(b'perf', opts)
entries = [
('nbrevs', 'number of revision covered'),
('nbmissingfiles', 'number of missing files at head'),
......
......@@ -43,13 +43,18 @@ Actions Blocking Release
* Support modifying compression parameters mid operation when supported by
zstd API.
* Expose ``ZSTD_CLEVEL_DEFAULT`` constant.
* Expose ``ZSTD_SRCSIZEHINT_{MIN,MAX}`` constants.
* Support ``ZSTD_p_forceAttachDict`` compression parameter.
* Support ``ZSTD_c_literalCompressionMode `` compression parameter.
* Support ``ZSTD_dictForceLoad`` dictionary compression parameter.
* Support ``ZSTD_c_targetCBlockSize`` compression parameter.
* Support ``ZSTD_c_literalCompressionMode`` compression parameter.
* Support ``ZSTD_c_srcSizeHint`` compression parameter.
* Use ``ZSTD_CCtx_getParameter()``/``ZSTD_CCtxParam_getParameter()`` for retrieving
compression parameters.
* Consider exposing ``ZSTDMT_toFlushNow()``.
* Expose ``ZDICT_trainFromBuffer_fastCover()``,
``ZDICT_optimizeTrainFromBuffer_fastCover``.
* Expose ``ZSTD_Sequence`` struct and related ``ZSTD_getSequences()`` API.
* Expose and enforce ``ZSTD_minCLevel()`` for minimum compression level.
* Consider a ``chunker()`` API for decompression.
* Consider stats for ``chunker()`` API, including finding the last consumed
......@@ -67,6 +72,20 @@ Other Actions Not Blocking Release
* API for ensuring max memory ceiling isn't exceeded.
* Move off nose for testing.
0.13.0 (released 2019-12-28)
============================
Changes
-------
* ``pytest-xdist`` ``pytest`` extension is now installed so tests can be
run in parallel.
* CI now builds ``manylinux2010`` and ``manylinux2014`` binary wheels
instead of a mix of ``manylinux2010`` and ``manylinux1``.
* Official support for Python 3.8 has been added.
* Bundled zstandard library upgraded from 1.4.3 to 1.4.4.
* Python code has been reformatted with black.
0.12.0 (released 2019-09-15)
============================
......
......@@ -20,7 +20,7 @@ https://github.com/indygreg/python-zstandard.
Requirements
============
This extension is designed to run with Python 2.7, 3.4, 3.5, 3.6, and 3.7
This extension is designed to run with Python 2.7, 3.5, 3.6, 3.7, and 3.8
on common platforms (Linux, Windows, and OS X). On PyPy (both PyPy2 and PyPy3) we support version 6.0.0 and above.
x86 and x86_64 are well-tested on Windows. Only x86_64 is well-tested on Linux and macOS.
......
......@@ -16,7 +16,7 @@
#include <zdict.h>
/* Remember to change the string in zstandard/__init__ as well */
#define PYTHON_ZSTANDARD_VERSION "0.12.0"
#define PYTHON_ZSTANDARD_VERSION "0.13.0"
typedef enum {
compressorobj_flush_finish,
......
......@@ -16,80 +16,82 @@ import tempfile
HERE = os.path.abspath(os.path.dirname(__file__))
SOURCES = ['zstd/%s' % p for p in (
'common/debug.c',
'common/entropy_common.c',
'common/error_private.c',
'common/fse_decompress.c',
'common/pool.c',
'common/threading.c',
'common/xxhash.c',
'common/zstd_common.c',
'compress/fse_compress.c',
'compress/hist.c',
'compress/huf_compress.c',
'compress/zstd_compress.c',
'compress/zstd_compress_literals.c',
'compress/zstd_compress_sequences.c',
'compress/zstd_double_fast.c',
'compress/zstd_fast.c',
'compress/zstd_lazy.c',
'compress/zstd_ldm.c',
'compress/zstd_opt.c',
'compress/zstdmt_compress.c',
'decompress/huf_decompress.c',
'decompress/zstd_ddict.c',
'decompress/zstd_decompress.c',
'decompress/zstd_decompress_block.c',
'dictBuilder/cover.c',
'dictBuilder/fastcover.c',
'dictBuilder/divsufsort.c',
'dictBuilder/zdict.c',
)]
SOURCES = [
"zstd/%s" % p
for p in (
"common/debug.c",
"common/entropy_common.c",
"common/error_private.c",
"common/fse_decompress.c",
"common/pool.c",
"common/threading.c",
"common/xxhash.c",
"common/zstd_common.c",
"compress/fse_compress.c",
"compress/hist.c",
"compress/huf_compress.c",
"compress/zstd_compress.c",
"compress/zstd_compress_literals.c",
"compress/zstd_compress_sequences.c",
"compress/zstd_double_fast.c",
"compress/zstd_fast.c",
"compress/zstd_lazy.c",
"compress/zstd_ldm.c",
"compress/zstd_opt.c",
"compress/zstdmt_compress.c",
"decompress/huf_decompress.c",
"decompress/zstd_ddict.c",
"decompress/zstd_decompress.c",
"decompress/zstd_decompress_block.c",
"dictBuilder/cover.c",
"dictBuilder/fastcover.c",
"dictBuilder/divsufsort.c",
"dictBuilder/zdict.c",
)
]
# Headers whose preprocessed output will be fed into cdef().
HEADERS = [os.path.join(HERE, 'zstd', *p) for p in (
('zstd.h',),
('dictBuilder', 'zdict.h'),
)]
INCLUDE_DIRS = [os.path.join(HERE, d) for d in (
'zstd',
'zstd/common',
'zstd/compress',
'zstd/decompress',
'zstd/dictBuilder',
)]
HEADERS = [
os.path.join(HERE, "zstd", *p) for p in (("zstd.h",), ("dictBuilder", "zdict.h"),)
]
INCLUDE_DIRS = [
os.path.join(HERE, d)
for d in (
"zstd",
"zstd/common",
"zstd/compress",
"zstd/decompress",
"zstd/dictBuilder",
)
]
# cffi can't parse some of the primitives in zstd.h. So we invoke the
# preprocessor and feed its output into cffi.
compiler = distutils.ccompiler.new_compiler()
# Needed for MSVC.
if hasattr(compiler, 'initialize'):
if hasattr(compiler, "initialize"):
compiler.initialize()
# Distutils doesn't set compiler.preprocessor, so invoke the preprocessor
# manually.
if compiler.compiler_type == 'unix':
args = list(compiler.executables['compiler'])
args.extend([
'-E',
'-DZSTD_STATIC_LINKING_ONLY',
'-DZDICT_STATIC_LINKING_ONLY',
])
elif compiler.compiler_type == 'msvc':
if compiler.compiler_type == "unix":
args = list(compiler.executables["compiler"])
args.extend(
["-E", "-DZSTD_STATIC_LINKING_ONLY", "-DZDICT_STATIC_LINKING_ONLY",]
)
elif compiler.compiler_type == "msvc":
args = [compiler.cc]
args.extend([
'/EP',
'/DZSTD_STATIC_LINKING_ONLY',
'/DZDICT_STATIC_LINKING_ONLY',
])
args.extend(
["/EP", "/DZSTD_STATIC_LINKING_ONLY", "/DZDICT_STATIC_LINKING_ONLY",]
)
else:
raise Exception('unsupported compiler type: %s' % compiler.compiler_type)
raise Exception("unsupported compiler type: %s" % compiler.compiler_type)
def preprocess(path):
with open(path, 'rb') as fh:
with open(path, "rb") as fh:
lines = []
it = iter(fh)
......@@ -104,32 +106,44 @@ def preprocess(path):
# We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline
# #define in zstdmt_compress.h and results in a compiler warning. So drop
# the inline #define.
if l.startswith((b'#include <stddef.h>',
b'#include "zstd.h"',
b'#define ZSTD_STATIC_LINKING_ONLY')):
if l.startswith(
(
b"#include <stddef.h>",
b'#include "zstd.h"',
b"#define ZSTD_STATIC_LINKING_ONLY",
)
):
continue
# The preprocessor environment on Windows doesn't define include
# paths, so the #include of limits.h fails. We work around this
# by removing that import and defining INT_MAX ourselves. This is
# a bit hacky. But it gets the job done.
# TODO make limits.h work on Windows so we ensure INT_MAX is
# correct.
if l.startswith(b"#include <limits.h>"):
l = b"#define INT_MAX 2147483647\n"
# ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't
# important so just filter it out.
if l.startswith(b'ZSTDLIB_API'):
l = l[len(b'ZSTDLIB_API '):]
if l.startswith(b"ZSTDLIB_API"):
l = l[len(b"ZSTDLIB_API ") :]
lines.append(l)
fd, input_file = tempfile.mkstemp(suffix='.h')
os.write(fd, b''.join(lines))
fd, input_file = tempfile.mkstemp(suffix=".h")
os.write(fd, b"".join(lines))
os.close(fd)
try:
env = dict(os.environ)
if getattr(compiler, '_paths', None):
env['PATH'] = compiler._paths
process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE,
env=env)
if getattr(compiler, "_paths", None):
env["PATH"] = compiler._paths
process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE, env=env)
output = process.communicate()[0]
ret = process.poll()
if ret:
raise Exception('preprocessor exited with error')
raise Exception("preprocessor exited with error")
return output
finally:
......@@ -141,16 +155,16 @@ def normalize_output(output):
for line in output.splitlines():
# CFFI's parser doesn't like __attribute__ on UNIX compilers.
if line.startswith(b'__attribute__ ((visibility ("default"))) '):
line = line[len(b'__attribute__ ((visibility ("default"))) '):]
line = line[len(b'__attribute__ ((visibility ("default"))) ') :]
if line.startswith(b'__attribute__((deprecated('):
if line.startswith(b"__attribute__((deprecated("):
continue
elif b'__declspec(deprecated(' in line:
elif b"__declspec(deprecated(" in line:
continue
lines.append(line)
return b'\n'.join(lines)
return b"\n".join(lines)
ffi = cffi.FFI()
......@@ -159,18 +173,22 @@ ffi = cffi.FFI()
# *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning
# when cffi uses the function. Since we statically link against zstd, even
# if we use the deprecated functions it shouldn't be a huge problem.
ffi.set_source('_zstd_cffi', '''
ffi.set_source(
"_zstd_cffi",
"""
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define ZSTD_STATIC_LINKING_ONLY
#include <zstd.h>
#define ZDICT_STATIC_LINKING_ONLY
#define ZDICT_DISABLE_DEPRECATE_WARNINGS
#include <zdict.h>
''', sources=SOURCES,
include_dirs=INCLUDE_DIRS,
extra_compile_args=['-DZSTD_MULTITHREAD'])
""",
sources=SOURCES,
include_dirs=INCLUDE_DIRS,
extra_compile_args=["-DZSTD_MULTITHREAD"],
)
DEFINE = re.compile(b'^\\#define ([a-zA-Z0-9_]+) ')
DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ")
sources = []
......@@ -181,27 +199,27 @@ for header in HEADERS:
# #define's are effectively erased as part of going through preprocessor.
# So perform a manual pass to re-add those to the cdef source.
with open(header, 'rb') as fh:
with open(header, "rb") as fh:
for line in fh:
line = line.strip()
m = DEFINE.match(line)
if not m:
continue
if m.group(1) == b'ZSTD_STATIC_LINKING_ONLY':
if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY":
continue
# The parser doesn't like some constants with complex values.
if m.group(1) in (b'ZSTD_LIB_VERSION', b'ZSTD_VERSION_STRING'):
if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"):
continue
# The ... is magic syntax by the cdef parser to resolve the
# value at compile time.
sources.append(m.group(0) + b' ...')
sources.append(m.group(0) + b" ...")
cdeflines = b'\n'.join(sources).splitlines()
cdeflines = b"\n".join(sources).splitlines()
cdeflines = [l for l in cdeflines if l.strip()]
ffi.cdef(b'\n'.join(cdeflines).decode('latin1'))
ffi.cdef(b"\n".join(cdeflines).decode("latin1"))
if __name__ == '__main__':
if __name__ == "__main__":
ffi.compile()
......@@ -16,7 +16,7 @@ from setuptools import setup
# (like memoryview).
# Need feature in 1.11 for ffi.gc() to declare size of objects so we avoid
# garbage collection pitfalls.
MINIMUM_CFFI_VERSION = '1.11'
MINIMUM_CFFI_VERSION = "1.11"
try:
import cffi
......@@ -26,9 +26,11 @@ try:
# out the CFFI version here and reject CFFI if it is too old.
cffi_version = LooseVersion(cffi.__version__)
if cffi_version < LooseVersion(MINIMUM_CFFI_VERSION):
print('CFFI 1.11 or newer required (%s found); '
'not building CFFI backend' % cffi_version,
file=sys.stderr)
print(
"CFFI 1.11 or newer required (%s found); "
"not building CFFI backend" % cffi_version,
file=sys.stderr,
)
cffi = None
except ImportError:
......@@ -40,73 +42,77 @@ SUPPORT_LEGACY = False
SYSTEM_ZSTD = False
WARNINGS_AS_ERRORS = False
if os.environ.get('ZSTD_WARNINGS_AS_ERRORS', ''):
if os.environ.get("ZSTD_WARNINGS_AS_ERRORS", ""):
WARNINGS_AS_ERRORS = True
if '--legacy' in sys.argv:
if "--legacy" in sys.argv:
SUPPORT_LEGACY = True
sys.argv.remove('--legacy')
sys.argv.remove("--legacy")
if '--system-zstd' in sys.argv:
if "--system-zstd" in sys.argv:
SYSTEM_ZSTD = True
sys.argv.remove('--system-zstd')
sys.argv.remove("--system-zstd")
if '--warnings-as-errors' in sys.argv:
if "--warnings-as-errors" in sys.argv:
WARNINGS_AS_ERRORS = True
sys.argv.remove('--warning-as-errors')
sys.argv.remove("--warning-as-errors")
# Code for obtaining the Extension instance is in its own module to
# facilitate reuse in other projects.
extensions = [
setup_zstd.get_c_extension(name='zstd',
support_legacy=SUPPORT_LEGACY,
system_zstd=SYSTEM_ZSTD,
warnings_as_errors=WARNINGS_AS_ERRORS),
setup_zstd.get_c_extension(
name="zstd",
support_legacy=SUPPORT_LEGACY,
system_zstd=SYSTEM_ZSTD,
warnings_as_errors=WARNINGS_AS_ERRORS,
),
]
install_requires = []
if cffi:
import make_cffi
extensions.append(make_cffi.ffi.distutils_extension())
install_requires.append('cffi>=%s' % MINIMUM_CFFI_VERSION)
install_requires.append("cffi>=%s" % MINIMUM_CFFI_VERSION)
version = None
with open('c-ext/python-zstandard.h', 'r') as fh:
with open("c-ext/python-zstandard.h", "r") as fh:
for line in fh:
if not line.startswith('#define PYTHON_ZSTANDARD_VERSION'):
if not line.startswith("#define PYTHON_ZSTANDARD_VERSION"):
continue
version = line.split()[2][1:-1]
break
if not version:
raise Exception('could not resolve package version; '
'this should never happen')
raise Exception("could not resolve package version; " "this should never happen")
setup(
name='zstandard',
name="zstandard",
version=version,
description='Zstandard bindings for Python',
long_description=open('README.rst', 'r').read(),
url='https://github.com/indygreg/python-zstandard',
author='Gregory Szorc',
author_email='gregory.szorc@gmail.com',
license='BSD',
description="Zstandard bindings for Python",
long_description=open("README.rst", "r").read(),
url="https://github.com/indygreg/python-zstandard",
author="Gregory Szorc",
author_email="gregory.szorc@gmail.com",
license="BSD",
classifiers=[
'Development Status :: 4 - Beta',
'Intended Audience :: Developers',
'License :: OSI Approved :: BSD License',
'Programming Language :: C',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"License :: OSI Approved :: BSD License",
"Programming Language :: C",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
],
keywords='zstandard zstd compression',
packages=['zstandard'],
keywords="zstandard zstd compression",
packages=["zstandard"],
ext_modules=extensions,
test_suite='tests',
test_suite="tests",
install_requires=install_requires,
tests_require=["hypothesis"],
)
......@@ -10,97 +10,110 @@ import os
from distutils.extension import Extension
zstd_sources = ['zstd/%s' % p for p in (
'common/debug.c',
'common/entropy_common.c',
'common/error_private.c',
'common/fse_decompress.c',
'common/pool.c',
'common/threading.c',
'common/xxhash.c',
'common/zstd_common.c',
'compress/fse_compress.c',
'compress/hist.c',
'compress/huf_compress.c',
'compress/zstd_compress_literals.c',
'compress/zstd_compress_sequences.c',
'compress/zstd_compress.c',
'compress/zstd_double_fast.c',
'compress/zstd_fast.c',
'compress/zstd_lazy.c',
'compress/zstd_ldm.c',
'compress/zstd_opt.c',
'compress/zstdmt_compress.c',
'decompress/huf_decompress.c',
'decompress/zstd_ddict.c',
'decompress/zstd_decompress.c',
'decompress/zstd_decompress_block.c',
'dictBuilder/cover.c',
'dictBuilder/divsufsort.c',
'dictBuilder/fastcover.c',
'dictBuilder/zdict.c',
)]
zstd_sources_legacy = ['zstd/%s' % p for p in (
'deprecated/zbuff_common.c',
'deprecated/zbuff_compress.c',
'deprecated/zbuff_decompress.c',
'legacy/zstd_v01.c',
'legacy/zstd_v02.c',
'legacy/zstd_v03.c',
'legacy/zstd_v04.c',
'legacy/zstd_v05.c',
'legacy/zstd_v06.c',
'legacy/zstd_v07.c'
)]
zstd_sources = [
"zstd/%s" % p
for p in (
"common/debug.c",
"common/entropy_common.c",
"common/error_private.c",
"common/fse_decompress.c",
"common/pool.c",
"common/threading.c",
"common/xxhash.c",
"common/zstd_common.c",
"compress/fse_compress.c",
"compress/hist.c",
"compress/huf_compress.c",
"compress/zstd_compress_literals.c",
"compress/zstd_compress_sequences.c",
"compress/zstd_compress.c",
"compress/zstd_double_fast.c",
"compress/zstd_fast.c",
"compress/zstd_lazy.c",
"compress/zstd_ldm.c",
"compress/zstd_opt.c",
"compress/zstdmt_compress.c",
"decompress/huf_decompress.c",
"decompress/zstd_ddict.c",
"decompress/zstd_decompress.c",
"decompress/zstd_decompress_block.c",
"dictBuilder/cover.c",
"dictBuilder/divsufsort.c",
"dictBuilder/fastcover.c",
"dictBuilder/zdict.c",
)
]
zstd_sources_legacy = [
"zstd/%s" % p
for p in (
"deprecated/zbuff_common.c",
"deprecated/zbuff_compress.c",
"deprecated/zbuff_decompress.c",
"legacy/zstd_v01.c",
"legacy/zstd_v02.c",
"legacy/zstd_v03.c",
"legacy/zstd_v04.c",
"legacy/zstd_v05.c",
"legacy/zstd_v06.c",
"legacy/zstd_v07.c",
)
]
zstd_includes = [
'zstd',
'zstd/common',
'zstd/compress',
'zstd/decompress',
'zstd/dictBuilder',
"zstd",
"zstd/common",
"zstd/compress",
"zstd/decompress",
"zstd/dictBuilder",
]
zstd_includes_legacy = [
'zstd/deprecated',
'zstd/legacy',
"zstd/deprecated",
"zstd/legacy",
]
ext_includes = [
'c-ext',
'zstd/common',
"c-ext",
"zstd/common",
]
ext_sources = [
'zstd/common/pool.c',
'zstd/common/threading.c',
'zstd.c',
'c-ext/bufferutil.c',
'c-ext/compressiondict.c',
'c-ext/compressobj.c',
'c-ext/compressor.c',
'c-ext/compressoriterator.c',
'c-ext/compressionchunker.c',
'c-ext/compressionparams.c',
'c-ext/compressionreader.c',
'c-ext/compressionwriter.c',
'c-ext/constants.c',
'c-ext/decompressobj.c',
'c-ext/decompressor.c',
'c-ext/decompressoriterator.c',
'c-ext/decompressionreader.c',
'c-ext/decompressionwriter.c',
'c-ext/frameparams.c',
"zstd/common/error_private.c",
"zstd/common/pool.c",
"zstd/common/threading.c",
"zstd/common/zstd_common.c",
"zstd.c",
"c-ext/bufferutil.c",
"c-ext/compressiondict.c",
"c-ext/compressobj.c",
"c-ext/compressor.c",
"c-ext/compressoriterator.c",
"c-ext/compressionchunker.c",
"c-ext/compressionparams.c",
"c-ext/compressionreader.c",
"c-ext/compressionwriter.c",
"c-ext/constants.c",
"c-ext/decompressobj.c",
"c-ext/decompressor.c",
"c-ext/decompressoriterator.c",
"c-ext/decompressionreader.c",
"c-ext/decompressionwriter.c",
"c-ext/frameparams.c",
]
zstd_depends = [
'c-ext/python-zstandard.h',
"c-ext/python-zstandard.h",
]
def get_c_extension(support_legacy=False, system_zstd=False, name='zstd',
warnings_as_errors=False, root=None):
def get_c_extension(
support_legacy=False,
system_zstd=False,
name="zstd",
warnings_as_errors=False,
root=None,
):
"""Obtain a distutils.extension.Extension for the C extension.
``support_legacy`` controls whether to compile in legacy zstd format support.
......@@ -125,17 +138,16 @@ def get_c_extension(support_legacy=False, system_zstd=False, name='zstd',
if not system_zstd:
sources.update([os.path.join(actual_root, p) for p in zstd_sources])
if support_legacy:
sources.update([os.path.join(actual_root, p)
for p in zstd_sources_legacy])
sources.update([os.path.join(actual_root, p) for p in zstd_sources_legacy])
sources = list(sources)
include_dirs = set([os.path.join(actual_root, d) for d in ext_includes])
if not system_zstd:
include_dirs.update([os.path.join(actual_root, d)
for d in zstd_includes])
include_dirs.update([os.path.join(actual_root, d) for d in zstd_includes])
if support_legacy:
include_dirs.update([os.path.join(actual_root, d)
for d in zstd_includes_legacy])
include_dirs.update(
[os.path.join(actual_root, d) for d in zstd_includes_legacy]
)
include_dirs = list(include_dirs)
depends = [os.path.join(actual_root, p) for p in zstd_depends]
......@@ -143,41 +155,40 @@ def get_c_extension(support_legacy=False, system_zstd=False, name='zstd',
compiler = distutils.ccompiler.new_compiler()
# Needed for MSVC.
if hasattr(compiler, 'initialize'):
if hasattr(compiler, "initialize"):
compiler.initialize()
if compiler.compiler_type == 'unix':
compiler_type = 'unix'
elif compiler.compiler_type == 'msvc':
compiler_type = 'msvc'
elif compiler.compiler_type == 'mingw32':
compiler_type = 'mingw32'
if compiler.compiler_type == "unix":
compiler_type = "unix"
elif compiler.compiler_type == "msvc":
compiler_type = "msvc"
elif compiler.compiler_type == "mingw32":
compiler_type = "mingw32"
else:
raise Exception('unhandled compiler type: %s' %
compiler.compiler_type)
raise Exception("unhandled compiler type: %s" % compiler.compiler_type)
extra_args = ['-DZSTD_MULTITHREAD']
extra_args = ["-DZSTD_MULTITHREAD"]
if not system_zstd:
extra_args.append('-DZSTDLIB_VISIBILITY=')
extra_args.append('-DZDICTLIB_VISIBILITY=')
extra_args.append('-DZSTDERRORLIB_VISIBILITY=')
extra_args.append("-DZSTDLIB_VISIBILITY=")
extra_args.append("-DZDICTLIB_VISIBILITY=")
extra_args.append("-DZSTDERRORLIB_VISIBILITY=")
if compiler_type == 'unix':
extra_args.append('-fvisibility=hidden')
if compiler_type == "unix":
extra_args.append("-fvisibility=hidden")
if not system_zstd and support_legacy:
extra_args.append('-DZSTD_LEGACY_SUPPORT=1')
extra_args.append("-DZSTD_LEGACY_SUPPORT=1")
if warnings_as_errors:
if compiler_type in ('unix', 'mingw32'):
extra_args.append('-Werror')
elif compiler_type == 'msvc':
extra_args.append('/WX')
if compiler_type in ("unix", "mingw32"):
extra_args.append("-Werror")
elif compiler_type == "msvc":
extra_args.append("/WX")
else:
assert False
libraries = ['zstd'] if system_zstd else []
libraries = ["zstd"] if system_zstd else []
# Python 3.7 doesn't like absolute paths. So normalize to relative.
sources = [os.path.relpath(p, root) for p in sources]
......@@ -185,8 +196,11 @@ def get_c_extension(support_legacy=False, system_zstd=False, name='zstd',
depends = [os.path.relpath(p, root) for p in depends]
# TODO compile with optimizations.
return Extension(name, sources,
include_dirs=include_dirs,
depends=depends,
extra_compile_args=extra_args,
libraries=libraries)
return Extension(
name,
sources,
include_dirs=include_dirs,
depends=depends,
extra_compile_args=extra_args,
libraries=libraries,
)
......@@ -3,6 +3,7 @@ import inspect
import io
import os
import types
import unittest
try:
import hypothesis
......@@ -10,39 +11,46 @@ except ImportError:
hypothesis = None
class TestCase(unittest.TestCase):
if not getattr(unittest.TestCase, "assertRaisesRegex", False):
assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
def make_cffi(cls):
"""Decorator to add CFFI versions of each test method."""
# The module containing this class definition should
# `import zstandard as zstd`. Otherwise things may blow up.
mod = inspect.getmodule(cls)
if not hasattr(mod, 'zstd'):
if not hasattr(mod, "zstd"):
raise Exception('test module does not contain "zstd" symbol')
if not hasattr(mod.zstd, 'backend'):
raise Exception('zstd symbol does not have "backend" attribute; did '
'you `import zstandard as zstd`?')
if not hasattr(mod.zstd, "backend"):
raise Exception(
'zstd symbol does not have "backend" attribute; did '
"you `import zstandard as zstd`?"
)
# If `import zstandard` already chose the cffi backend, there is nothing
# for us to do: we only add the cffi variation if the default backend
# is the C extension.
if mod.zstd.backend == 'cffi':
if mod.zstd.backend == "cffi":
return cls
old_env = dict(os.environ)
os.environ['PYTHON_ZSTANDARD_IMPORT_POLICY'] = 'cffi'
os.environ["PYTHON_ZSTANDARD_IMPORT_POLICY"] = "cffi"
try:
try:
mod_info = imp.find_module('zstandard')
mod = imp.load_module('zstandard_cffi', *mod_info)
mod_info = imp.find_module("zstandard")
mod = imp.load_module("zstandard_cffi", *mod_info)
except ImportError:
return cls
finally:
os.environ.clear()
os.environ.update(old_env)
if mod.backend != 'cffi':
raise Exception('got the zstandard %s backend instead of cffi' % mod.backend)
if mod.backend != "cffi":
raise Exception("got the zstandard %s backend instead of cffi" % mod.backend)
# If CFFI version is available, dynamically construct test methods
# that use it.
......@@ -52,27 +60,31 @@ def make_cffi(cls):
if not inspect.ismethod(fn) and not inspect.isfunction(fn):
continue
if not fn.__name__.startswith('test_'):
if not fn.__name__.startswith("test_"):
continue
name = '%s_cffi' % fn.__name__
name = "%s_cffi" % fn.__name__
# Replace the "zstd" symbol with the CFFI module instance. Then copy
# the function object and install it in a new attribute.
if isinstance(fn, types.FunctionType):
globs = dict(fn.__globals__)
globs['zstd'] = mod
new_fn = types.FunctionType(fn.__code__, globs, name,
fn.__defaults__, fn.__closure__)
globs["zstd"] = mod
new_fn = types.FunctionType(
fn.__code__, globs, name, fn.__defaults__, fn.__closure__
)
new_method = new_fn
else:
globs = dict(fn.__func__.func_globals)
globs['zstd'] = mod
new_fn = types.FunctionType(fn.__func__.func_code, globs, name,
fn.__func__.func_defaults,
fn.__func__.func_closure)
new_method = types.UnboundMethodType(new_fn, fn.im_self,
fn.im_class)
globs["zstd"] = mod
new_fn = types.FunctionType(
fn.__func__.func_code,
globs,
name,
fn.__func__.func_defaults,
fn.__func__.func_closure,
)
new_method = types.UnboundMethodType(new_fn, fn.im_self, fn.im_class)
setattr(cls, name, new_method)
......@@ -84,6 +96,7 @@ class NonClosingBytesIO(io.BytesIO):
This allows us to access written data after close().
"""
def __init__(self, *args, **kwargs):
super(NonClosingBytesIO, self).__init__(*args, **kwargs)
self._saved_buffer = None
......@@ -135,7 +148,7 @@ def random_input_data():
dirs[:] = list(sorted(dirs))
for f in sorted(files):
try:
with open(os.path.join(root, f), 'rb') as fh:
with open(os.path.join(root, f), "rb") as fh:
data = fh.read()
if data:
_source_files.append(data)
......@@ -154,11 +167,11 @@ def random_input_data():
def generate_samples():
inputs = [
b'foo',
b'bar',
b'abcdef',
b'sometext',
b'baz',
b"foo",
b"bar",
b"abcdef",
b"sometext",
b"baz",
]
samples = []
......@@ -173,13 +186,12 @@ def generate_samples():
if hypothesis:
default_settings = hypothesis.settings(deadline=10000)
hypothesis.settings.register_profile('default', default_settings)
hypothesis.settings.register_profile("default", default_settings)
ci_settings = hypothesis.settings(deadline=20000, max_examples=1000)
hypothesis.settings.register_profile('ci', ci_settings)
hypothesis.settings.register_profile("ci", ci_settings)
expensive_settings = hypothesis.settings(deadline=None, max_examples=10000)
hypothesis.settings.register_profile('expensive', expensive_settings)
hypothesis.settings.register_profile("expensive", expensive_settings)
hypothesis.settings.load_profile(
os.environ.get('HYPOTHESIS_PROFILE', 'default'))
hypothesis.settings.load_profile(os.environ.get("HYPOTHESIS_PROFILE", "default"))
......@@ -3,104 +3,114 @@ import unittest
import zstandard as zstd
ss = struct.Struct('=QQ')
from .common import TestCase
ss = struct.Struct("=QQ")
class TestBufferWithSegments(unittest.TestCase):
class TestBufferWithSegments(TestCase):
def test_arguments(self):
if not hasattr(zstd, 'BufferWithSegments'):
self.skipTest('BufferWithSegments not available')
if not hasattr(zstd, "BufferWithSegments"):
self.skipTest("BufferWithSegments not available")
with self.assertRaises(TypeError):
zstd.BufferWithSegments()
with self.assertRaises(TypeError):
zstd.BufferWithSegments(b'foo')
zstd.BufferWithSegments(b"foo")
# Segments data should be a multiple of 16.
with self.assertRaisesRegexp(ValueError, 'segments array size is not a multiple of 16'):
zstd.BufferWithSegments(b'foo', b'\x00\x00')
with self.assertRaisesRegex(
ValueError, "segments array size is not a multiple of 16"
):
zstd.BufferWithSegments(b"foo", b"\x00\x00")
def test_invalid_offset(self):
if not hasattr(zstd, 'BufferWithSegments'):
self.skipTest('BufferWithSegments not available')
if not hasattr(zstd, "BufferWithSegments"):
self.skipTest("BufferWithSegments not available")
with self.assertRaisesRegexp(ValueError, 'offset within segments array references memory'):
zstd.BufferWithSegments(b'foo', ss.pack(0, 4))
with self.assertRaisesRegex(
ValueError, "offset within segments array references memory"
):
zstd.BufferWithSegments(b"foo", ss.pack(0, 4))
def test_invalid_getitem(self):
if not hasattr(zstd, 'BufferWithSegments'):
self.skipTest('BufferWithSegments not available')
if not hasattr(zstd, "BufferWithSegments"):
self.skipTest("BufferWithSegments not available")
b = zstd.BufferWithSegments(b'foo', ss.pack(0, 3))
b = zstd.BufferWithSegments(b"foo", ss.pack(0, 3))
with self.assertRaisesRegexp(IndexError, 'offset must be non-negative'):
with self.assertRaisesRegex(IndexError, "offset must be non-negative"):
test = b[-10]
with self.assertRaisesRegexp(IndexError, 'offset must be less than 1'):
with self.assertRaisesRegex(IndexError, "offset must be less than 1"):
test = b[1]
with self.assertRaisesRegexp(IndexError, 'offset must be less than 1'):
with self.assertRaisesRegex(IndexError, "offset must be less than 1"):
test = b[2]
def test_single(self):
if not hasattr(zstd, 'BufferWithSegments'):
self.skipTest('BufferWithSegments not available')
if not hasattr(zstd, "BufferWithSegments"):
self.skipTest("BufferWithSegments not available")
b = zstd.BufferWithSegments(b'foo', ss.pack(0, 3))
b = zstd.BufferWithSegments(b"foo", ss.pack(0, 3))
self.assertEqual(len(b), 1)
self.assertEqual(b.size, 3)
self.assertEqual(b.tobytes(), b'foo')
self.assertEqual(b.tobytes(), b"foo")
self.assertEqual(len(b[0]), 3)
self.assertEqual(b[0].offset, 0)
self.assertEqual(b[0].tobytes(), b'foo')
self.assertEqual(b[0].tobytes(), b"foo")
def test_multiple(self):
if not hasattr(zstd, 'BufferWithSegments'):
self.skipTest('BufferWithSegments not available')
if not hasattr(zstd, "BufferWithSegments"):
self.skipTest("BufferWithSegments not available")
b = zstd.BufferWithSegments(b'foofooxfooxy', b''.join([ss.pack(0, 3),
ss.pack(3, 4),
ss.pack(7, 5)]))
b = zstd.BufferWithSegments(
b"foofooxfooxy", b"".join([ss.pack(0, 3), ss.pack(3, 4), ss.pack(7, 5)])
)
self.assertEqual(len(b), 3)
self.assertEqual(b.size, 12)
self.assertEqual(b.tobytes(), b'foofooxfooxy')
self.assertEqual(b.tobytes(), b"foofooxfooxy")
self.assertEqual(b[0].tobytes(), b'foo')
self.assertEqual(b[1].tobytes(), b'foox')
self.assertEqual(b[2].tobytes(), b'fooxy')
self.assertEqual(b[0].tobytes(), b"foo")
self.assertEqual(b[1].tobytes(), b"foox")
self.assertEqual(b[2].tobytes(), b"fooxy")
class TestBufferWithSegmentsCollection(unittest.TestCase):
class TestBufferWithSegmentsCollection(TestCase):
def test_empty_constructor(self):
if not hasattr(zstd, 'BufferWithSegmentsCollection'):
self.skipTest('BufferWithSegmentsCollection not available')
if not hasattr(zstd, "BufferWithSegmentsCollection"):
self.skipTest("BufferWithSegmentsCollection not available")
with self.assertRaisesRegexp(ValueError, 'must pass at least 1 argument'):
with self.assertRaisesRegex(ValueError, "must pass at least 1 argument"):
zstd.BufferWithSegmentsCollection()
def test_argument_validation(self):
if not hasattr(zstd, 'BufferWithSegmentsCollection'):
self.skipTest('BufferWithSegmentsCollection not available')
if not hasattr(zstd, "BufferWithSegmentsCollection"):
self.skipTest("BufferWithSegmentsCollection not available")
with self.assertRaisesRegexp(TypeError, 'arguments must be BufferWithSegments'):
with self.assertRaisesRegex(TypeError, "arguments must be BufferWithSegments"):
zstd.BufferWithSegmentsCollection(None)
with self.assertRaisesRegexp(TypeError, 'arguments must be BufferWithSegments'):
zstd.BufferWithSegmentsCollection(zstd.BufferWithSegments(b'foo', ss.pack(0, 3)),