...
 
Commits (10)
......@@ -9,7 +9,6 @@ from __future__ import absolute_import
import collections
import contextlib
import hashlib
import os
from mercurial.i18n import _
......@@ -28,7 +27,10 @@ from mercurial import (
scmutil,
util,
)
from mercurial.utils import stringutil
from mercurial.utils import (
hashutil,
stringutil,
)
from . import (
error as faerror,
......@@ -148,7 +150,7 @@ def hashdiffopts(diffopts):
diffoptstr = stringutil.pprint(
sorted((k, getattr(diffopts, k)) for k in mdiff.diffopts.defaults)
)
return node.hex(hashlib.sha1(diffoptstr).digest())[:6]
return node.hex(hashutil.sha1(diffoptstr).digest())[:6]
_defaultdiffopthash = hashdiffopts(mdiff.defaultopts)
......
......@@ -108,7 +108,6 @@ created.
from __future__ import absolute_import
import codecs
import hashlib
import os
import stat
import sys
......@@ -132,7 +131,10 @@ from mercurial import (
util,
)
from mercurial import match as matchmod
from mercurial.utils import stringutil
from mercurial.utils import (
hashutil,
stringutil,
)
from . import (
pywatchman,
......@@ -235,7 +237,7 @@ def _hashignore(ignore):
copy.
"""
sha1 = hashlib.sha1()
sha1 = hashutil.sha1()
sha1.update(pycompat.byterepr(ignore))
return pycompat.sysbytes(sha1.hexdigest())
......
......@@ -6,7 +6,6 @@
from __future__ import absolute_import
import abc
import hashlib
import os
import subprocess
import tempfile
......@@ -16,7 +15,10 @@ from mercurial import (
node,
pycompat,
)
from mercurial.utils import procutil
from mercurial.utils import (
hashutil,
procutil,
)
NamedTemporaryFile = tempfile.NamedTemporaryFile
......@@ -87,7 +89,7 @@ class filebundlestore(object):
return os.path.join(self._dirpath(filename), filename)
def write(self, data):
filename = node.hex(hashlib.sha1(data).digest())
filename = node.hex(hashutil.sha1(data).digest())
dirpath = self._dirpath(filename)
if not os.path.exists(dirpath):
......
......@@ -10,7 +10,6 @@
from __future__ import absolute_import
import errno
import hashlib
import os
import shutil
......@@ -29,6 +28,7 @@ from mercurial import (
scmutil,
util,
)
from mercurial.utils import hashutil
from ..convert import (
convcmd,
......@@ -273,7 +273,7 @@ def _lfconvert_addchangeset(
)
# largefile was modified, update standins
m = hashlib.sha1(b'')
m = hashutil.sha1(b'')
m.update(ctx[f].data())
hash = node.hex(m.digest())
if f not in lfiletohash or lfiletohash[f] != hash:
......
......@@ -11,7 +11,6 @@ from __future__ import absolute_import
import contextlib
import copy
import hashlib
import os
import stat
......@@ -32,6 +31,7 @@ from mercurial import (
util,
vfs as vfsmod,
)
from mercurial.utils import hashutil
shortname = b'.hglf'
shortnameslash = shortname + b'/'
......@@ -432,7 +432,7 @@ def writestandin(repo, standin, hash, executable):
def copyandhash(instream, outfile):
'''Read bytes from instream (iterable) and write them to outfile,
computing the SHA-1 hash of the data along the way. Return the hash.'''
hasher = hashlib.sha1(b'')
hasher = hashutil.sha1(b'')
for data in instream:
hasher.update(data)
outfile.write(data)
......@@ -472,7 +472,7 @@ def urljoin(first, second, *arg):
def hexsha1(fileobj):
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
object data"""
h = hashlib.sha1()
h = hashutil.sha1()
for chunk in util.filechunkiter(fileobj):
h.update(chunk)
return hex(h.digest())
......
......@@ -2,7 +2,6 @@ from __future__ import absolute_import
import collections
import errno
import hashlib
import mmap
import os
import struct
......@@ -20,6 +19,7 @@ from mercurial import (
util,
vfs as vfsmod,
)
from mercurial.utils import hashutil
from . import shallowutil
osutil = policy.importmod('osutil')
......@@ -392,7 +392,7 @@ class mutablebasepack(versionmixin):
)
self.packfp = os.fdopen(self.packfp, 'wb+')
self.idxfp = os.fdopen(self.idxfp, 'wb+')
self.sha = hashlib.sha1()
self.sha = hashutil.sha1()
self._closed = False
# The opener provides no way of doing permission fixup on files created
......
from __future__ import absolute_import
import errno
import hashlib
import os
import shutil
import stat
......@@ -15,6 +14,7 @@ from mercurial import (
pycompat,
util,
)
from mercurial.utils import hashutil
from . import (
constants,
shallowutil,
......@@ -166,7 +166,7 @@ class basestore(object):
# Start with a full manifest, since it'll cover the majority of files
for filename in self.repo[b'tip'].manifest():
sha = hashlib.sha1(filename).digest()
sha = hashutil.sha1(filename).digest()
if sha in missingfilename:
filenames[filename] = sha
missingfilename.discard(sha)
......@@ -178,7 +178,7 @@ class basestore(object):
break
files = cl.readfiles(cl.node(rev))
for filename in files:
sha = hashlib.sha1(filename).digest()
sha = hashutil.sha1(filename).digest()
if sha in missingfilename:
filenames[filename] = sha
missingfilename.discard(sha)
......
......@@ -6,7 +6,6 @@
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
import hashlib
import os
import zlib
......@@ -21,6 +20,7 @@ from mercurial import (
pycompat,
revlog,
)
from mercurial.utils import hashutil
from . import (
constants,
datapack,
......@@ -61,7 +61,7 @@ def debugremotefilelog(ui, path, **opts):
def buildtemprevlog(repo, file):
# get filename key
filekey = nodemod.hex(hashlib.sha1(file).digest())
filekey = nodemod.hex(hashutil.sha1(file).digest())
filedir = os.path.join(repo.path, b'store/data', filekey)
# sort all entries based on linkrev
......@@ -421,7 +421,7 @@ def dumpdeltachain(ui, deltachain, **opts):
% (
hashformatter(node),
hashformatter(deltabasenode),
nodemod.hex(hashlib.sha1(delta).digest()),
nodemod.hex(hashutil.sha1(delta).digest()),
len(delta),
)
)
......
......@@ -7,7 +7,6 @@
from __future__ import absolute_import
import hashlib
import io
import os
import threading
......@@ -25,7 +24,10 @@ from mercurial import (
util,
wireprotov1peer,
)
from mercurial.utils import procutil
from mercurial.utils import (
hashutil,
procutil,
)
from . import (
constants,
......@@ -45,12 +47,12 @@ _lfsmod = None
def getcachekey(reponame, file, id):
pathhash = node.hex(hashlib.sha1(file).digest())
pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
def getlocalkey(file, id):
pathhash = node.hex(hashlib.sha1(file).digest())
pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(pathhash, id)
......
from __future__ import absolute_import
import hashlib
import struct
from mercurial.node import hex, nullid
......@@ -8,6 +7,7 @@ from mercurial import (
pycompat,
util,
)
from mercurial.utils import hashutil
from . import (
basepack,
constants,
......@@ -197,7 +197,7 @@ class historypack(basepack.basepack):
def _findsection(self, name):
params = self.params
namehash = hashlib.sha1(name).digest()
namehash = hashutil.sha1(name).digest()
fanoutkey = struct.unpack(
params.fanoutstruct, namehash[: params.fanoutprefix]
)[0]
......@@ -499,7 +499,7 @@ class mutablehistorypack(basepack.mutablebasepack):
# Record metadata for the index
self.files[filename] = (sectionstart, sectionlen)
node = hashlib.sha1(filename).digest()
node = hashutil.sha1(filename).digest()
self.entries[node] = node
def close(self, ledger=None):
......@@ -517,7 +517,7 @@ class mutablehistorypack(basepack.mutablebasepack):
nodeindexlength = self.NODEINDEXENTRYLENGTH
files = (
(hashlib.sha1(filename).digest(), filename, offset, size)
(hashutil.sha1(filename).digest(), filename, offset, size)
for filename, (offset, size) in pycompat.iteritems(self.files)
)
files = sorted(files)
......
......@@ -8,7 +8,6 @@ from __future__ import absolute_import
import collections
import errno
import hashlib
import os
import stat
import struct
......@@ -24,6 +23,7 @@ from mercurial import (
util,
)
from mercurial.utils import (
hashutil,
storageutil,
stringutil,
)
......@@ -39,12 +39,12 @@ def isenabled(repo):
def getcachekey(reponame, file, id):
pathhash = node.hex(hashlib.sha1(file).digest())
pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
def getlocalkey(file, id):
pathhash = node.hex(hashlib.sha1(file).digest())
pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(pathhash, id)
......
......@@ -45,7 +45,6 @@ option to ``sqlite`` to enable new repositories to use SQLite for storage.
from __future__ import absolute_import
import hashlib
import sqlite3
import struct
import threading
......@@ -75,7 +74,10 @@ from mercurial.interfaces import (
repository,
util as interfaceutil,
)
from mercurial.utils import storageutil
from mercurial.utils import (
hashutil,
storageutil,
)
try:
from mercurial import zstd
......@@ -807,7 +809,7 @@ class sqlitefilestore(object):
self._db, pathid, node, {}, {-1: None}, zstddctx=self._dctx
)
deltahash = hashlib.sha1(fulltext).digest()
deltahash = hashutil.sha1(fulltext).digest()
if self._compengine == b'zstd':
deltablob = self._cctx.compress(fulltext)
......@@ -837,7 +839,7 @@ class sqlitefilestore(object):
# Now create the tombstone delta and replace the delta on the censored
# node.
deltahash = hashlib.sha1(tombstone).digest()
deltahash = hashutil.sha1(tombstone).digest()
tombstonedeltaid = insertdelta(
self._db, COMPRESSION_NONE, deltahash, tombstone
)
......@@ -1004,7 +1006,7 @@ class sqlitefilestore(object):
# us to de-duplicate. The table is configured to ignore conflicts
# and it is faster to just insert and silently noop than to look
# first.
deltahash = hashlib.sha1(delta).digest()
deltahash = hashutil.sha1(delta).digest()
if self._compengine == b'zstd':
deltablob = self._cctx.compress(delta)
......
......@@ -38,6 +38,7 @@ typedef struct {
} nodetreenode;
typedef struct {
int abi_version;
int (*index_parents)(PyObject *, int, int *);
} Revlog_CAPI;
......@@ -3037,6 +3038,9 @@ static PyTypeObject rustlazyancestorsType = {
#endif /* WITH_RUST */
static Revlog_CAPI CAPI = {
/* increment the abi_version field upon each change in the Revlog_CAPI
struct or in the ABI of the listed functions */
1,
HgRevlogIndex_GetParents,
};
......
......@@ -41,7 +41,6 @@ Config
from __future__ import absolute_import
import hashlib
import inspect
import os
import re
......@@ -67,6 +66,7 @@ from . import (
)
from .utils import (
hashutil,
procutil,
stringutil,
)
......@@ -74,7 +74,7 @@ from .utils import (
def _hashlist(items):
"""return sha1 hexdigest for a list"""
return node.hex(hashlib.sha1(stringutil.pprint(items)).digest())
return node.hex(hashutil.sha1(stringutil.pprint(items)).digest())
# sensitive config sections affecting confighash
......
......@@ -8,7 +8,6 @@
from __future__ import absolute_import
import collections
import hashlib
from .i18n import _
from .node import (
......@@ -40,7 +39,10 @@ from . import (
wireprototypes,
)
from .interfaces import repository
from .utils import stringutil
from .utils import (
hashutil,
stringutil,
)
urlerr = util.urlerr
urlreq = util.urlreq
......@@ -2705,7 +2707,7 @@ def check_heads(repo, their_heads, context):
Used by peer for unbundling.
"""
heads = repo.heads()
heads_hash = hashlib.sha1(b''.join(sorted(heads))).digest()
heads_hash = hashutil.sha1(b''.join(sorted(heads))).digest()
if not (
their_heads == [b'force']
or their_heads == heads
......
......@@ -9,7 +9,6 @@
from __future__ import absolute_import
import errno
import hashlib
import os
import shutil
import stat
......@@ -48,7 +47,7 @@ from . import (
verify as verifymod,
vfs as vfsmod,
)
from .utils import hashutil
from .interfaces import repository as repositorymod
release = lock.release
......@@ -738,7 +737,7 @@ def clone(
)
elif sharenamemode == b'remote':
sharepath = os.path.join(
sharepool, node.hex(hashlib.sha1(source).digest())
sharepool, node.hex(hashutil.sha1(source).digest())
)
else:
raise error.Abort(
......
......@@ -8,7 +8,6 @@
from __future__ import absolute_import
import errno
import hashlib
import os
import random
import sys
......@@ -74,6 +73,7 @@ from .interfaces import (
)
from .utils import (
hashutil,
procutil,
stringutil,
)
......@@ -2007,7 +2007,7 @@ class localrepository(object):
)
idbase = b"%.40f#%f" % (random.random(), time.time())
ha = hex(hashlib.sha1(idbase).digest())
ha = hex(hashutil.sha1(idbase).digest())
txnid = b'TXN:' + ha
self.hook(b'pretxnopen', throw=True, txnname=desc, txnid=txnid)
......
......@@ -8,7 +8,6 @@
from __future__ import absolute_import
import errno
import hashlib
import shutil
import stat
import struct
......@@ -39,6 +38,7 @@ from . import (
util,
worker,
)
from .utils import hashutil
_pack = struct.pack
_unpack = struct.unpack
......@@ -512,7 +512,7 @@ class mergestate(object):
"""hash the path of a local file context for storage in the .hg/merge
directory."""
return hex(hashlib.sha1(path).digest())
return hex(hashutil.sha1(path).digest())
def add(self, fcl, fco, fca, fd):
"""add a new (potentially?) conflicting file the merge state
......
......@@ -70,7 +70,6 @@ comment associated with each format for details.
from __future__ import absolute_import
import errno
import hashlib
import struct
from .i18n import _
......@@ -85,7 +84,10 @@ from . import (
pycompat,
util,
)
from .utils import dateutil
from .utils import (
dateutil,
hashutil,
)
parsers = policy.importmod('parsers')
......@@ -1028,7 +1030,7 @@ def _computecontentdivergentset(repo):
def makefoldid(relation, user):
folddigest = hashlib.sha1(user)
folddigest = hashutil.sha1(user)
for p in relation[0] + relation[1]:
folddigest.update(b'%d' % p.rev())
folddigest.update(p.node())
......
......@@ -12,7 +12,6 @@ import collections
import contextlib
import copy
import errno
import hashlib
import os
import re
import shutil
......@@ -41,6 +40,7 @@ from . import (
)
from .utils import (
dateutil,
hashutil,
procutil,
stringutil,
)
......@@ -2943,7 +2943,7 @@ def trydiff(
if not text:
text = b""
l = len(text)
s = hashlib.sha1(b'blob %d\0' % l)
s = hashutil.sha1(b'blob %d\0' % l)
s.update(text)
return hex(s.digest())
......
......@@ -243,49 +243,50 @@ class phasecache(object):
"""return a smartset for the given phases"""
self.loadphaserevs(repo) # ensure phase's sets are loaded
phases = set(phases)
publicphase = public in phases
if public not in phases:
# fast path: _phasesets contains the interesting sets,
# might only need a union and post-filtering.
if len(phases) == 1:
[p] = phases
revs = self._phasesets[p]
else:
revs = set.union(*[self._phasesets[p] for p in phases])
if publicphase:
# In this case, phases keeps all the *other* phases.
phases = set(allphases).difference(phases)
if not phases:
return smartset.fullreposet(repo)
# fast path: _phasesets contains the interesting sets,
# might only need a union and post-filtering.
revsneedscopy = False
if len(phases) == 1:
[p] = phases
revs = self._phasesets[p]
revsneedscopy = True # Don't modify _phasesets
else:
# revs has the revisions in all *other* phases.
revs = set.union(*[self._phasesets[p] for p in phases])
def _addwdir(wdirsubset, wdirrevs):
if wdirrev in wdirsubset and repo[None].phase() in phases:
if revsneedscopy:
wdirrevs = wdirrevs.copy()
# The working dir would never be in the # cache, but it was in
# the subset being filtered for its phase (or filtered out,
# depending on publicphase), so add it to the output to be
# included (or filtered out).
wdirrevs.add(wdirrev)
return wdirrevs
if not publicphase:
if repo.changelog.filteredrevs:
revs = revs - repo.changelog.filteredrevs
if subset is None:
return smartset.baseset(revs)
else:
if wdirrev in subset and repo[None].phase() in phases:
# The working dir would never be in the cache, but it was
# in the subset being filtered for its phase, so add it to
# the output.
revs.add(wdirrev)
revs = _addwdir(subset, revs)
return subset & smartset.baseset(revs)
else:
# phases keeps all the *other* phases.
phases = set(allphases).difference(phases)
if not phases:
return smartset.fullreposet(repo)
# revs has the revisions in all *other* phases.
if len(phases) == 1:
[p] = phases
revs = self._phasesets[p]
else:
revs = set.union(*[self._phasesets[p] for p in phases])
if subset is None:
subset = smartset.fullreposet(repo)
if wdirrev in subset and repo[None].phase() in phases:
# The working dir is in the subset being filtered, and its
# phase is in the phases *not* being returned, so add it to the
# set of revisions to filter out.
revs.add(wdirrev)
revs = _addwdir(subset, revs)
if not revs:
return subset
......
......@@ -9,7 +9,6 @@
from __future__ import absolute_import
import errno
import hashlib
from .i18n import _
from .node import (
......@@ -29,7 +28,10 @@ from . import (
pycompat,
util,
)
from .utils import stringutil
from .utils import (
hashutil,
stringutil,
)
def backupbundle(
......@@ -45,7 +47,7 @@ def backupbundle(
# Include a hash of all the nodes in the filename for uniqueness
allcommits = repo.set(b'%ln::%ln', bases, heads)
allhashes = sorted(c.hex() for c in allcommits)
totalhash = hashlib.sha1(b''.join(allhashes)).digest()
totalhash = hashutil.sha1(b''.join(allhashes)).digest()
name = b"%s/%s-%s-%s.hg" % (
backupdir,
short(node),
......
......@@ -33,10 +33,10 @@ the concept.
from __future__ import absolute_import
import hashlib
import struct
from .. import error
from ..utils import hashutil
## sidedata type constant
# reserve a block for testing purposes.
......@@ -64,7 +64,7 @@ def sidedatawriteprocessor(rl, text, sidedata):
sidedata.sort()
rawtext = [SIDEDATA_HEADER.pack(len(sidedata))]
for key, value in sidedata:
digest = hashlib.sha1(value).digest()
digest = hashutil.sha1(value).digest()
rawtext.append(SIDEDATA_ENTRY.pack(key, len(value), digest))
for key, value in sidedata:
rawtext.append(value)
......@@ -85,7 +85,7 @@ def sidedatareadprocessor(rl, text):
# read the data associated with that entry
nextdataoffset = dataoffset + size
entrytext = text[dataoffset:nextdataoffset]
readdigest = hashlib.sha1(entrytext).digest()
readdigest = hashutil.sha1(entrytext).digest()
if storeddigest != readdigest:
raise error.SidedataHashError(key, storeddigest, readdigest)
sidedata[key] = entrytext
......
......@@ -9,7 +9,6 @@ from __future__ import absolute_import
import errno
import glob
import hashlib
import os
import posixpath
import re
......@@ -48,6 +47,7 @@ from . import (
)
from .utils import (
hashutil,
procutil,
stringutil,
)
......@@ -366,7 +366,7 @@ def filteredhash(repo, maxrev):
key = None
revs = sorted(r for r in cl.filteredrevs if r <= maxrev)
if revs:
s = hashlib.sha1()
s = hashutil.sha1()
for rev in revs:
s.update(b'%d;' % rev)
key = s.digest()
......@@ -1760,6 +1760,7 @@ class progress(object):
self._updatebar(self.topic, self.pos, b"", self.unit, self.total)
def _printdebug(self, item):
unit = b''
if self.unit:
unit = b' ' + self.unit
if item:
......
......@@ -7,7 +7,6 @@
from __future__ import absolute_import
import hashlib
import os
from .i18n import _
......@@ -24,6 +23,7 @@ from . import (
scmutil,
util,
)
from .utils import hashutil
# Whether sparse features are enabled. This variable is intended to be
# temporary to facilitate porting sparse to core. It should eventually be
......@@ -205,12 +205,12 @@ def configsignature(repo, includetemp=True):
tempsignature = b'0'
if signature is None or (includetemp and tempsignature is None):
signature = hex(hashlib.sha1(repo.vfs.tryread(b'sparse')).digest())
signature = hex(hashutil.sha1(repo.vfs.tryread(b'sparse')).digest())
cache[b'signature'] = signature
if includetemp:
raw = repo.vfs.tryread(b'tempsparse')
tempsignature = hex(hashlib.sha1(raw).digest())
tempsignature = hex(hashutil.sha1(raw).digest())
cache[b'tempsignature'] = tempsignature
return b'%s %s' % (signature, tempsignature)
......
......@@ -24,6 +24,7 @@ from . import (
util,
)
from .utils import (
hashutil,
resourceutil,
stringutil,
)
......@@ -949,7 +950,7 @@ def validatesocket(sock):
# If a certificate fingerprint is pinned, use it and only it to
# validate the remote cert.
peerfingerprints = {
b'sha1': node.hex(hashlib.sha1(peercert).digest()),
b'sha1': node.hex(hashutil.sha1(peercert).digest()),
b'sha256': node.hex(hashlib.sha256(peercert).digest()),
b'sha512': node.hex(hashlib.sha512(peercert).digest()),
}
......
......@@ -9,7 +9,6 @@ from __future__ import absolute_import
import errno
import functools
import hashlib
import os
import stat
......@@ -25,6 +24,7 @@ from . import (
util,
vfs as vfsmod,
)
from .utils import hashutil
parsers = policy.importmod('parsers')
# how much bytes should be read from fncache in one read
......@@ -273,7 +273,7 @@ _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4
def _hashencode(path, dotencode):
digest = node.hex(hashlib.sha1(path).digest())
digest = node.hex(hashutil.sha1(path).digest())
le = lowerencode(path[5:]).split(b'/') # skips prefix 'data/' or 'meta/'
parts = _auxencode(le, dotencode)
basename = parts[-1]
......
......@@ -9,7 +9,6 @@ from __future__ import absolute_import
import copy
import errno
import hashlib
import os
import re
import stat
......@@ -37,6 +36,7 @@ from . import (
)
from .utils import (
dateutil,
hashutil,
procutil,
stringutil,
)
......@@ -61,7 +61,7 @@ def _expandedabspath(path):
def _getstorehashcachename(remotepath):
'''get a unique filename for the store hash cache of a remote repository'''
return node.hex(hashlib.sha1(_expandedabspath(remotepath)).digest())[0:12]
return node.hex(hashutil.sha1(_expandedabspath(remotepath)).digest())[0:12]
class SubrepoAbort(error.Abort):
......@@ -514,7 +514,7 @@ class hgsubrepo(abstractsubrepo):
yield b'# %s\n' % _expandedabspath(remotepath)
vfs = self._repo.vfs
for relname in filelist:
filehash = node.hex(hashlib.sha1(vfs.tryread(relname)).digest())
filehash = node.hex(hashutil.sha1(vfs.tryread(relname)).digest())
yield b'%s = %s\n' % (relname, filehash)
@propertycache
......
MIT License
Copyright (c) 2017:
Marc Stevens
Cryptology Group
Centrum Wiskunde & Informatica
P.O. Box 94079, 1090 GB Amsterdam, Netherlands
marc@marc-stevens.nl
Dan Shumow
Microsoft Research
danshu@microsoft.com
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# sha1collisiondetection
Library and command line tool to detect SHA-1 collisions in files
Copyright 2017 Marc Stevens <marc@marc-stevens.nl>
Distributed under the MIT Software License.
See accompanying file LICENSE.txt or copy at https://opensource.org/licenses/MIT.
## Developers
- Marc Stevens, CWI Amsterdam (https://marc-stevens.nl)
- Dan Shumow, Microsoft Research (https://www.microsoft.com/en-us/research/people/danshu/)
## About
This library and command line tool were designed as near drop-in replacements for common SHA-1 libraries and sha1sum.
They will compute the SHA-1 hash of any given file and additionally will detect cryptanalytic collision attacks against SHA-1 present in each file. It is very fast and takes less than twice the amount of time as regular SHA-1.
More specifically they will detect any cryptanalytic collision attack against SHA-1 using any of the top 32 SHA-1 disturbance vectors with probability 1:
```
I(43,0), I(44,0), I(45,0), I(46,0), I(47,0), I(48,0), I(49,0), I(50,0), I(51,0), I(52,0),
I(46,2), I(47,2), I(48,2), I(49,2), I(50,2), I(51,2),
II(45,0), II(46,0), II(47,0), II(48,0), II(49,0), II(50,0), II(51,0), II(52,0), II(53,0), II(54,0), II(55,0), II(56,0),
II(46,2), II(49,2), II(50,2), II(51,2)
```
The possibility of false positives can be neglected as the probability is smaller than 2^-90.
The library supports both an indicator flag that applications can check and act on, as well as a special _safe-hash_ mode that returns the real SHA-1 hash when no collision was detected and a different _safe_ hash when a collision was detected.
Colliding files will have the same SHA-1 hash, but will have different unpredictable safe-hashes.
This essentially enables protection of applications against SHA-1 collisions with no further changes in the application, e.g., digital signature forgeries based on SHA-1 collisions automatically become invalid.
For the theoretical explanation of collision detection see the award-winning paper on _Counter-Cryptanalysis_:
Counter-cryptanalysis, Marc Stevens, CRYPTO 2013, Lecture Notes in Computer Science, vol. 8042, Springer, 2013, pp. 129-146,
https://marc-stevens.nl/research/papers/C13-S.pdf
## Compiling
Run:
```
make
```
## Command-line usage
There are two programs `bin/sha1dcsum` and `bin/sha1dcsum_partialcoll`.
The first program `bin/sha1dcsum` will detect and warn for files that were generated with a cryptanalytic SHA-1 collision attack like the one documented at https://shattered.io/.
The second program `bin/sha1dcsum_partialcoll` will detect and warn for files that were generated with a cryptanalytic collision attack against reduced-round SHA-1 (of which there are a few examples so far).
Examples:
```
bin/sha1dcsum test/sha1_reducedsha_coll.bin test/shattered-1.pdf
bin/sha1dcsum_partialcoll test/sha1reducedsha_coll.bin test/shattered-1.pdf
pipe_data | bin/sha1dcsum -
```
## Library usage
See the documentation in `lib/sha1.h`. Here is a simple example code snippet:
```
#include <sha1dc/sha1.h>
SHA1_CTX ctx;
unsigned char hash[20];
SHA1DCInit(&ctx);
/** disable safe-hash mode (safe-hash mode is enabled by default) **/
// SHA1DCSetSafeHash(&ctx, 0);
/** disable use of unavoidable attack conditions to speed up detection (enabled by default) **/
// SHA1DCSetUseUBC(&ctx, 0);
SHA1DCUpdate(&ctx, buffer, (unsigned)(size));
int iscoll = SHA1DCFinal(hash,&ctx);
if (iscoll)
printf("collision detected");
else
printf("no collision detected");
```
## Inclusion in other programs
In order to make it easier to include these sources in other project
there are several preprocessor macros that the code uses. Rather than
copy/pasting and customizing or specializing the code, first see if
setting any of these defines appropriately will allow you to avoid
modifying the code yourself.
- SHA1DC_NO_STANDARD_INCLUDES
Skips including standard headers. Use this if your project for
whatever reason wishes to do its own header includes.
- SHA1DC_CUSTOM_INCLUDE_SHA1_C
Includes a custom header at the top of sha1.c. Usually this would be
set in conjunction with SHA1DC_NO_STANDARD_INCLUDES to point to a
header file which includes various standard headers.
- SHA1DC_INIT_SAFE_HASH_DEFAULT
Sets the default for safe_hash in SHA1DCInit(). Valid values are 0
and 1. If unset 1 is the default.
- SHA1DC_CUSTOM_TRAILING_INCLUDE_SHA1_C
Includes a custom trailer in sha1.c. Useful for any extra utility
functions that make use of the functions already defined in sha1.c.
- SHA1DC_CUSTOM_TRAILING_INCLUDE_SHA1_H
Includes a custom trailer in sha1.h. Useful for defining the
prototypes of the functions or code included by
SHA1DC_CUSTOM_TRAILING_INCLUDE_SHA1_C.
- SHA1DC_CUSTOM_INCLUDE_UBC_CHECK_C
Includes a custom header at the top of ubc_check.c.
- SHA1DC_CUSTOM_TRAILING_INCLUDE_UBC_CHECK_C
Includes a custom trailer in ubc_check.c.
- SHA1DC_CUSTOM_TRAILING_INCLUDE_UBC_CHECK_H
Includes a custom trailer in ubc_check.H.
This code will try to auto-detect certain things based on
CPU/platform. Unless you're running on some really obscure CPU or
porting to a new platform you should not need to tweak this. If you do
please open an issue at
https://github.com/cr-marcstevens/sha1collisiondetection
- SHA1DC_FORCE_LITTLEENDIAN / SHA1DC_FORCE_BIGENDIAN
Override the check for processor endianenss and force either
Little-Endian or Big-Endian.
- SHA1DC_FORCE_UNALIGNED_ACCESS
Permit unaligned access. This will fail on e.g. SPARC processors, so
it's only permitted on a whitelist of processors. If your CPU isn't
detected as allowing this, and allows unaligned access, setting this
may improve performance (or make it worse, if the kernel has to
catch and emulate such access on its own).
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "lib/sha1.h"
#if PY_MAJOR_VERSION >= 3
#define IS_PY3K
#endif
/* helper to switch things like string literal depending on Python version */
#ifdef IS_PY3K
#define PY23(py2, py3) py3
#else
#define PY23(py2, py3) py2
#endif
static char sha1dc_doc[] = "Efficient detection of SHA1 collision constructs.";
/* clang-format off */
typedef struct {
PyObject_HEAD
SHA1_CTX ctx;
} pysha1ctx;
/* clang-format on */
static int pysha1ctx_init(pysha1ctx *self, PyObject *args)
{
const char *data = NULL;
Py_ssize_t len;
SHA1DCInit(&(self->ctx));
/* We don't want "safe" sha1s, wherein sha1dc can give you a
different hash for something that's trying to give you a
collision. We just want to detect collisions.
*/
SHA1DCSetSafeHash(&(self->ctx), 0);
if (!PyArg_ParseTuple(args, PY23("|s#", "|y#"), &data, &len)) {
return -1;
}
if (data) {
SHA1DCUpdate(&(self->ctx), data, len);
}
return 0;
}
static void pysha1ctx_dealloc(pysha1ctx *self)
{
PyObject_Del(self);
}
static PyObject *pysha1ctx_update(pysha1ctx *self, PyObject *args)
{
const char *data;
Py_ssize_t len;
if (!PyArg_ParseTuple(args, PY23("s#", "y#"), &data, &len)) {
return NULL;
}
SHA1DCUpdate(&(self->ctx), data, len);
Py_RETURN_NONE;
}
/* it is intentional that this take a ctx by value, as that clones the
context so we can keep using .update() without poisoning the state
with padding.
*/
static int finalize(SHA1_CTX ctx, unsigned char *hash_out)
{
if (SHA1DCFinal(hash_out, &ctx)) {
PyErr_SetString(PyExc_OverflowError,
"sha1 collision attack detected");
return 0;
}
return 1;
}
static PyObject *pysha1ctx_digest(pysha1ctx *self)
{
unsigned char hash[20];
if (!finalize(self->ctx, hash)) {
return NULL;
}
return PyBytes_FromStringAndSize((char *)hash, 20);
}
static PyObject *pysha1ctx_hexdigest(pysha1ctx *self)
{
unsigned char hash[20];
if (!finalize(self->ctx, hash)) {
return NULL;
}
char hexhash[40];
static const char hexdigit[] = "0123456789abcdef";
for (int i = 0; i < 20; ++i) {
hexhash[i * 2] = hexdigit[hash[i] >> 4];
hexhash[i * 2 + 1] = hexdigit[hash[i] & 15];
}
return PyString_FromStringAndSize(hexhash, 40);
}
static PyTypeObject sha1ctxType;
static PyObject *pysha1ctx_copy(pysha1ctx *self)
{
pysha1ctx *clone = (pysha1ctx *)PyObject_New(pysha1ctx, &sha1ctxType);
if (!clone) {
return NULL;
}
clone->ctx = self->ctx;
return (PyObject *)clone;
}
static PyMethodDef pysha1ctx_methods[] = {
{"update", (PyCFunction)pysha1ctx_update, METH_VARARGS,
"Update this hash object's state with the provided bytes."},
{"digest", (PyCFunction)pysha1ctx_digest, METH_NOARGS,
"Return the digest value as a string of binary data."},
{"hexdigest", (PyCFunction)pysha1ctx_hexdigest, METH_NOARGS,
"Return the digest value as a string of hexadecimal digits."},
{"copy", (PyCFunction)pysha1ctx_copy, METH_NOARGS,
"Return a copy of the hash object."},
{NULL},
};
/* clang-format off */
static PyTypeObject sha1ctxType = {
PyVarObject_HEAD_INIT(NULL, 0) /* header */
"sha1dc.sha1", /* tp_name */
sizeof(pysha1ctx), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)pysha1ctx_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT, /* tp_flags */
"sha1 implementation that looks for collisions", /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
pysha1ctx_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
(initproc)pysha1ctx_init, /* tp_init */
0, /* tp_alloc */
};
/* clang-format on */
static PyMethodDef methods[] = {
{NULL, NULL},
};
static void module_init(PyObject *mod)
{
sha1ctxType.tp_new = PyType_GenericNew;
if (PyType_Ready(&sha1ctxType) < 0) {
return;
}
Py_INCREF(&sha1ctxType);
PyModule_AddObject(mod, "sha1", (PyObject *)&sha1ctxType);
}
#ifdef IS_PY3K
static struct PyModuleDef sha1dc_module = {PyModuleDef_HEAD_INIT, "sha1dc",
sha1dc_doc, -1, methods};
PyMODINIT_FUNC PyInit_sha1dc(void)
{
PyObject *mod = PyModule_Create(&sha1dc_module);
module_init(mod);
return mod;
}
#else
PyMODINIT_FUNC initsha1dc(void)
{
PyObject *mod = Py_InitModule3("sha1dc", methods, sha1dc_doc);
module_init(mod);
}
#endif
/***
* Copyright 2017 Marc Stevens <marc@marc-stevens.nl>, Dan Shumow <danshu@microsoft.com>
* Distributed under the MIT Software License.
* See accompanying file LICENSE.txt or copy at
* https://opensource.org/licenses/MIT
***/
#ifndef SHA1DC_SHA1_H
#define SHA1DC_SHA1_H
#if defined(__cplusplus)
extern "C" {
#endif
#ifndef SHA1DC_NO_STANDARD_INCLUDES
#include <stdint.h>
#endif
/* sha-1 compression function that takes an already expanded message, and additionally store intermediate states */
/* only stores states ii (the state between step ii-1 and step ii) when DOSTORESTATEii is defined in ubc_check.h */
void sha1_compression_states(uint32_t[5], const uint32_t[16], uint32_t[80], uint32_t[80][5]);
/*
// Function type for sha1_recompression_step_T (uint32_t ihvin[5], uint32_t ihvout[5], const uint32_t me2[80], const uint32_t state[5]).
// Where 0 <= T < 80
// me2 is an expanded message (the expansion of an original message block XOR'ed with a disturbance vector's message block difference.)
// state is the internal state (a,b,c,d,e) before step T of the SHA-1 compression function while processing the original message block.
// The function will return:
// ihvin: The reconstructed input chaining value.
// ihvout: The reconstructed output chaining value.
*/
typedef void(*sha1_recompression_type)(uint32_t*, uint32_t*, const uint32_t*, const uint32_t*);
/* A callback function type that can be set to be called when a collision block has been found: */
/* void collision_block_callback(uint64_t byteoffset, const uint32_t ihvin1[5], const uint32_t ihvin2[5], const uint32_t m1[80], const uint32_t m2[80]) */
typedef void(*collision_block_callback)(uint64_t, const uint32_t*, const uint32_t*, const uint32_t*, const uint32_t*);
/* The SHA-1 context. */
typedef struct {
uint64_t total;
uint32_t ihv[5];
unsigned char buffer[64];
int found_collision;
int safe_hash;
int detect_coll;
int ubc_check;
int reduced_round_coll;
collision_block_callback callback;
uint32_t ihv1[5];
uint32_t ihv2[5];
uint32_t m1[80];
uint32_t m2[80];
uint32_t states[80][5];
} SHA1_CTX;
/* Initialize SHA-1 context. */
void SHA1DCInit(SHA1_CTX*);
/*
Function to enable safe SHA-1 hashing:
Collision attacks are thwarted by hashing a detected near-collision block 3 times.
Think of it as extending SHA-1 from 80-steps to 240-steps for such blocks:
The best collision attacks against SHA-1 have complexity about 2^60,
thus for 240-steps an immediate lower-bound for the best cryptanalytic attacks would be 2^180.
An attacker would be better off using a generic birthday search of complexity 2^80.
Enabling safe SHA-1 hashing will result in the correct SHA-1 hash for messages where no collision attack was detected,
but it will result in a different SHA-1 hash for messages where a collision attack was detected.
This will automatically invalidate SHA-1 based digital signature forgeries.
Enabled by default.
*/
void SHA1DCSetSafeHash(SHA1_CTX*, int);
/*
Function to disable or enable the use of Unavoidable Bitconditions (provides a significant speed up).
Enabled by default
*/
void SHA1DCSetUseUBC(SHA1_CTX*, int);
/*
Function to disable or enable the use of Collision Detection.
Enabled by default.
*/
void SHA1DCSetUseDetectColl(SHA1_CTX*, int);
/* function to disable or enable the detection of reduced-round SHA-1 collisions */
/* disabled by default */
void SHA1DCSetDetectReducedRoundCollision(SHA1_CTX*, int);
/* function to set a callback function, pass NULL to disable */
/* by default no callback set */
void SHA1DCSetCallback(SHA1_CTX*, collision_block_callback);
/* update SHA-1 context with buffer contents */
void SHA1DCUpdate(SHA1_CTX*, const char*, size_t);
/* obtain SHA-1 hash from SHA-1 context */
/* returns: 0 = no collision detected, otherwise = collision found => warn user for active attack */
int SHA1DCFinal(unsigned char[20], SHA1_CTX*);
#if defined(__cplusplus)
}
#endif
#ifdef SHA1DC_CUSTOM_TRAILING_INCLUDE_SHA1_H
#include SHA1DC_CUSTOM_TRAILING_INCLUDE_SHA1_H
#endif
#endif
/***
* Copyright 2017 Marc Stevens <marc@marc-stevens.nl>, Dan Shumow <danshu@microsoft.com>
* Distributed under the MIT Software License.
* See accompanying file LICENSE.txt or copy at
* https://opensource.org/licenses/MIT
***/
/*
// this file was generated by the 'parse_bitrel' program in the tools section
// using the data files from directory 'tools/data/3565'
//
// sha1_dvs contains a list of SHA-1 Disturbance Vectors (DV) to check
// dvType, dvK and dvB define the DV: I(K,B) or II(K,B) (see the paper)
// dm[80] is the expanded message block XOR-difference defined by the DV
// testt is the step to do the recompression from for collision detection
// maski and maskb define the bit to check for each DV in the dvmask returned by ubc_check
//
// ubc_check takes as input an expanded message block and verifies the unavoidable bitconditions for all listed DVs
// it returns a dvmask where each bit belonging to a DV is set if all unavoidable bitconditions for that DV have been met
// thus one needs to do the recompression check for each DV that has its bit set
*/
#ifndef SHA1DC_UBC_CHECK_H
#define SHA1DC_UBC_CHECK_H
#if defined(__cplusplus)
extern "C" {
#endif
#ifndef SHA1DC_NO_STANDARD_INCLUDES
#include <stdint.h>
#endif
#define DVMASKSIZE 1
typedef struct { int dvType; int dvK; int dvB; int testt; int maski; int maskb; uint32_t dm[80]; } dv_info_t;
extern dv_info_t sha1_dvs[];
void ubc_check(const uint32_t W[80], uint32_t dvmask[DVMASKSIZE]);
#define DOSTORESTATE58
#define DOSTORESTATE65
#define CHECK_DVMASK(_DVMASK) (0 != _DVMASK[0])
#if defined(__cplusplus)
}
#endif
#ifdef SHA1DC_CUSTOM_TRAILING_INCLUDE_UBC_CHECK_H
#include SHA1DC_CUSTOM_TRAILING_INCLUDE_UBC_CHECK_H
#endif
#endif
......@@ -53,6 +53,7 @@ from . import (
)
from .utils import (
compression,
hashutil,
procutil,
stringutil,
)
......@@ -197,7 +198,7 @@ def nouideprecwarn(msg, version, stacklevel=1):
DIGESTS = {
b'md5': hashlib.md5,
b'sha1': hashlib.sha1,
b'sha1': hashutil.sha1,
b'sha512': hashlib.sha512,
}
# List of digest types from strongest to weakest
......
from __future__ import absolute_import
import hashlib
try:
from mercurial.thirdparty import sha1dc
sha1 = sha1dc.sha1
except (ImportError, AttributeError):
sha1 = hashlib.sha1
......@@ -7,7 +7,6 @@
from __future__ import absolute_import
import hashlib
import re
import struct
......@@ -24,8 +23,9 @@ from .. import (
pycompat,
)
from ..interfaces import repository
from ..utils import hashutil
_nullhash = hashlib.sha1(nullid)
_nullhash = hashutil.sha1(nullid)
def hashrevisionsha1(text, p1, p2):
......@@ -48,7 +48,7 @@ def hashrevisionsha1(text, p1, p2):
else:
a = p2
b = p1
s = hashlib.sha1(a)
s = hashutil.sha1(a)
s.update(b)
s.update(text)
return s.digest()
......
......@@ -7,7 +7,6 @@
from __future__ import absolute_import
import hashlib
import sys
import weakref
......@@ -31,6 +30,7 @@ from .interfaces import (
repository,
util as interfaceutil,
)
from .utils import hashutil
urlreq = util.urlreq
......@@ -489,7 +489,7 @@ class wirepeer(repository.peer):
if heads != [b'force'] and self.capable(b'unbundlehash'):
heads = wireprototypes.encodelist(
[b'hashed', hashlib.sha1(b''.join(sorted(heads))).digest()]
[b'hashed', hashutil.sha1(b''.join(sorted(heads))).digest()]
)
else:
heads = wireprototypes.encodelist(heads)
......
......@@ -8,7 +8,6 @@ from __future__ import absolute_import
import collections
import contextlib
import hashlib
from .i18n import _
from .node import (
......@@ -31,6 +30,7 @@ from . import (
from .interfaces import util as interfaceutil