...
 
Commits (44)
......@@ -2,7 +2,7 @@
clang-format:command = clang-format --style file
clang-format:pattern = set:(**.c or **.cc or **.h) and not "include:contrib/clang-format-ignorelist"
rustfmt:command = rustfmt
rustfmt:command = $(rustup which --toolchain nightly rustfmt)
rustfmt:pattern = set:**.rs
black:command = black --config=black.toml -
......
......@@ -9,7 +9,6 @@ from __future__ import absolute_import
import collections
import contextlib
import hashlib
import os
from mercurial.i18n import _
......@@ -28,7 +27,10 @@ from mercurial import (
scmutil,
util,
)
from mercurial.utils import stringutil
from mercurial.utils import (
hashutil,
stringutil,
)
from . import (
error as faerror,
......@@ -148,7 +150,7 @@ def hashdiffopts(diffopts):
diffoptstr = stringutil.pprint(
sorted((k, getattr(diffopts, k)) for k in mdiff.diffopts.defaults)
)
return node.hex(hashlib.sha1(diffoptstr).digest())[:6]
return node.hex(hashutil.sha1(diffoptstr).digest())[:6]
_defaultdiffopthash = hashdiffopts(mdiff.defaultopts)
......
......@@ -108,7 +108,6 @@ created.
from __future__ import absolute_import
import codecs
import hashlib
import os
import stat
import sys
......@@ -132,7 +131,10 @@ from mercurial import (
util,
)
from mercurial import match as matchmod
from mercurial.utils import stringutil
from mercurial.utils import (
hashutil,
stringutil,
)
from . import (
pywatchman,
......@@ -235,7 +237,7 @@ def _hashignore(ignore):
copy.
"""
sha1 = hashlib.sha1()
sha1 = hashutil.sha1()
sha1.update(pycompat.byterepr(ignore))
return pycompat.sysbytes(sha1.hexdigest())
......
......@@ -105,11 +105,11 @@ class client(object):
)
return self._watchmanclient.query(*watchmanargs)
except pywatchman.CommandError as ex:
if b'unable to resolve root' in ex.msg:
if 'unable to resolve root' in ex.msg:
raise WatchmanNoRoot(
self._root, stringutil.forcebytestr(ex.msg)
)
raise Unavailable(ex.msg)
raise Unavailable(stringutil.forcebytestr(ex.msg))
except pywatchman.WatchmanError as ex:
raise Unavailable(stringutil.forcebytestr(ex))
......
......@@ -6,7 +6,6 @@
from __future__ import absolute_import
import abc
import hashlib
import os
import subprocess
import tempfile
......@@ -16,7 +15,10 @@ from mercurial import (
node,
pycompat,
)
from mercurial.utils import procutil
from mercurial.utils import (
hashutil,
procutil,
)
NamedTemporaryFile = tempfile.NamedTemporaryFile
......@@ -87,7 +89,7 @@ class filebundlestore(object):
return os.path.join(self._dirpath(filename), filename)
def write(self, data):
filename = node.hex(hashlib.sha1(data).digest())
filename = node.hex(hashutil.sha1(data).digest())
dirpath = self._dirpath(filename)
if not os.path.exists(dirpath):
......
......@@ -10,7 +10,6 @@
from __future__ import absolute_import
import errno
import hashlib
import os
import shutil
......@@ -29,6 +28,7 @@ from mercurial import (
scmutil,
util,
)
from mercurial.utils import hashutil
from ..convert import (
convcmd,
......@@ -273,7 +273,7 @@ def _lfconvert_addchangeset(
)
# largefile was modified, update standins
m = hashlib.sha1(b'')
m = hashutil.sha1(b'')
m.update(ctx[f].data())
hash = node.hex(m.digest())
if f not in lfiletohash or lfiletohash[f] != hash:
......
......@@ -11,7 +11,6 @@ from __future__ import absolute_import
import contextlib
import copy
import hashlib
import os
import stat
......@@ -32,6 +31,7 @@ from mercurial import (
util,
vfs as vfsmod,
)
from mercurial.utils import hashutil
shortname = b'.hglf'
shortnameslash = shortname + b'/'
......@@ -432,7 +432,7 @@ def writestandin(repo, standin, hash, executable):
def copyandhash(instream, outfile):
'''Read bytes from instream (iterable) and write them to outfile,
computing the SHA-1 hash of the data along the way. Return the hash.'''
hasher = hashlib.sha1(b'')
hasher = hashutil.sha1(b'')
for data in instream:
hasher.update(data)
outfile.write(data)
......@@ -472,7 +472,7 @@ def urljoin(first, second, *arg):
def hexsha1(fileobj):
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
object data"""
h = hashlib.sha1()
h = hashutil.sha1()
for chunk in util.filechunkiter(fileobj):
h.update(chunk)
return hex(h.digest())
......
......@@ -402,3 +402,25 @@ def debuglfsupload(ui, repo, **opts):
revs = opts.get('rev', [])
pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs))
wrapper.uploadblobs(repo, pointers)
@eh.wrapcommand(
b'verify',
opts=[(b'', b'no-lfs', None, _(b'skip missing lfs blob content'))],
)
def verify(orig, ui, repo, **opts):
skipflags = repo.ui.configint(b'verify', b'skipflags')
no_lfs = opts.pop('no_lfs')
if skipflags:
# --lfs overrides the config bit, if set.
if no_lfs is False:
skipflags &= ~repository.REVISION_FLAG_EXTSTORED
else:
skipflags = 0
if no_lfs is True:
skipflags |= repository.REVISION_FLAG_EXTSTORED
with ui.configoverride({(b'verify', b'skipflags'): skipflags}):
return orig(ui, repo, **opts)
......@@ -155,15 +155,29 @@ class local(object):
return self.vfs(oid, b'rb')
def download(self, oid, src):
def download(self, oid, src, content_length):
"""Read the blob from the remote source in chunks, verify the content,
and write to this local blobstore."""
sha256 = hashlib.sha256()
size = 0
with self.vfs(oid, b'wb', atomictemp=True) as fp:
for chunk in util.filechunkiter(src, size=1048576):
fp.write(chunk)
sha256.update(chunk)
size += len(chunk)
# If the server advertised a length longer than what we actually
# received, then we should expect that the server crashed while
# producing the response (but the server has no way of telling us
# that), and we really don't need to try to write the response to
# the localstore, because it's not going to match the expected.
if content_length is not None and int(content_length) != size:
msg = (
b"Response length (%s) does not match Content-Length "
b"header (%d): likely server-side crash"
)
raise LfsRemoteError(_(msg) % (size, int(content_length)))
realoid = node.hex(sha256.digest())
if realoid != oid:
......@@ -489,27 +503,30 @@ class _gitlfsremote(object):
for k, v in headers:
request.add_header(pycompat.strurl(k), pycompat.strurl(v))
response = b''
try:
with contextlib.closing(self.urlopener.open(request)) as req:
with contextlib.closing(self.urlopener.open(request)) as res:
contentlength = res.info().get(b"content-length")
ui = self.ui # Shorten debug lines
if self.ui.debugflag:
ui.debug(b'Status: %d\n' % req.status)
ui.debug(b'Status: %d\n' % res.status)
# lfs-test-server and hg serve return headers in different
# order
headers = pycompat.bytestr(req.info()).strip()
headers = pycompat.bytestr(res.info()).strip()
ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines())))
if action == b'download':
# If downloading blobs, store downloaded data to local
# blobstore
localstore.download(oid, req)
localstore.download(oid, res, contentlength)
else:
blocks = []
while True:
data = req.read(1048576)
data = res.read(1048576)
if not data:
break
response += data
blocks.append(data)
response = b"".join(blocks)
if response:
ui.debug(b'lfs %s response: %s' % (action, response))
except util.urlerr.httperror as ex:
......@@ -588,7 +605,9 @@ class _gitlfsremote(object):
else:
oids = transfer(sorted(objects, key=lambda o: o.get(b'oid')))
with self.ui.makeprogress(topic, total=total) as progress:
with self.ui.makeprogress(
topic, unit=_(b"bytes"), total=total
) as progress:
progress.update(0)
processed = 0
blobs = 0
......@@ -635,7 +654,7 @@ class _dummyremote(object):
def readbatch(self, pointers, tostore):
for p in _deduplicate(pointers):
with self.vfs(p.oid(), b'rb') as fp:
tostore.download(p.oid(), fp)
tostore.download(p.oid(), fp, None)
class _nullremote(object):
......
......@@ -327,7 +327,7 @@ def _processbasictransfer(repo, req, res, checkperm):
statusmessage = hgwebcommon.statusmessage
try:
localstore.download(oid, req.bodyfh)
localstore.download(oid, req.bodyfh, req.headers[b'Content-Length'])
res.status = statusmessage(HTTP_OK if existed else HTTP_CREATED)
except blobstore.LfsCorruptionError:
_logexception(req)
......
......@@ -225,6 +225,25 @@ def filelogsize(orig, self, rev):
return orig(self, rev)
@eh.wrapfunction(revlog, b'_verify_revision')
def _verify_revision(orig, rl, skipflags, state, node):
if _islfs(rl, node=node):
rawtext = rl.rawdata(node)
metadata = pointer.deserialize(rawtext)
# Don't skip blobs that are stored locally, as local verification is
# relatively cheap and there's no other way to verify the raw data in
# the revlog.
if rl.opener.lfslocalblobstore.has(metadata.oid()):
skipflags &= ~revlog.REVIDX_EXTSTORED
elif skipflags & revlog.REVIDX_EXTSTORED:
# The wrapped method will set `skipread`, but there's enough local
# info to check renames.
state[b'safe_renamed'].add(node)
orig(rl, skipflags, state, node)
@eh.wrapfunction(context.basefilectx, b'cmp')
def filectxcmp(orig, self, fctx):
"""returns True if text is different than fctx"""
......
......@@ -1065,6 +1065,7 @@ def phabsend(ui, repo, *revs, **opts):
opts = pycompat.byteskwargs(opts)
revs = list(revs) + opts.get(b'rev', [])
revs = scmutil.revrange(repo, revs)
revs.sort() # ascending order to preserve topological parent/child in phab
if not revs:
raise error.Abort(_(b'phabsend requires at least one changeset'))
......
......@@ -2,7 +2,6 @@ from __future__ import absolute_import
import collections
import errno
import hashlib
import mmap
import os
import struct
......@@ -20,6 +19,7 @@ from mercurial import (
util,
vfs as vfsmod,
)
from mercurial.utils import hashutil
from . import shallowutil
osutil = policy.importmod('osutil')
......@@ -392,7 +392,7 @@ class mutablebasepack(versionmixin):
)
self.packfp = os.fdopen(self.packfp, 'wb+')
self.idxfp = os.fdopen(self.idxfp, 'wb+')
self.sha = hashlib.sha1()
self.sha = hashutil.sha1()
self._closed = False
# The opener provides no way of doing permission fixup on files created
......
from __future__ import absolute_import
import errno
import hashlib
import os
import shutil
import stat
......@@ -15,6 +14,7 @@ from mercurial import (
pycompat,
util,
)
from mercurial.utils import hashutil
from . import (
constants,
shallowutil,
......@@ -166,7 +166,7 @@ class basestore(object):
# Start with a full manifest, since it'll cover the majority of files
for filename in self.repo[b'tip'].manifest():
sha = hashlib.sha1(filename).digest()
sha = hashutil.sha1(filename).digest()
if sha in missingfilename:
filenames[filename] = sha
missingfilename.discard(sha)
......@@ -178,7 +178,7 @@ class basestore(object):
break
files = cl.readfiles(cl.node(rev))
for filename in files:
sha = hashlib.sha1(filename).digest()
sha = hashutil.sha1(filename).digest()
if sha in missingfilename:
filenames[filename] = sha
missingfilename.discard(sha)
......
......@@ -6,7 +6,6 @@
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
import hashlib
import os
import zlib
......@@ -21,6 +20,7 @@ from mercurial import (
pycompat,
revlog,
)
from mercurial.utils import hashutil
from . import (
constants,
datapack,
......@@ -61,7 +61,7 @@ def debugremotefilelog(ui, path, **opts):
def buildtemprevlog(repo, file):
# get filename key
filekey = nodemod.hex(hashlib.sha1(file).digest())
filekey = nodemod.hex(hashutil.sha1(file).digest())
filedir = os.path.join(repo.path, b'store/data', filekey)
# sort all entries based on linkrev
......@@ -421,7 +421,7 @@ def dumpdeltachain(ui, deltachain, **opts):
% (
hashformatter(node),
hashformatter(deltabasenode),
nodemod.hex(hashlib.sha1(delta).digest()),
nodemod.hex(hashutil.sha1(delta).digest()),
len(delta),
)
)
......
......@@ -7,7 +7,6 @@
from __future__ import absolute_import
import hashlib
import io
import os
import threading
......@@ -25,7 +24,10 @@ from mercurial import (
util,
wireprotov1peer,
)
from mercurial.utils import procutil
from mercurial.utils import (
hashutil,
procutil,
)
from . import (
constants,
......@@ -45,12 +47,12 @@ _lfsmod = None
def getcachekey(reponame, file, id):
pathhash = node.hex(hashlib.sha1(file).digest())
pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
def getlocalkey(file, id):
pathhash = node.hex(hashlib.sha1(file).digest())
pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(pathhash, id)
......
from __future__ import absolute_import
import hashlib
import struct
from mercurial.node import hex, nullid
......@@ -8,6 +7,7 @@ from mercurial import (
pycompat,
util,
)
from mercurial.utils import hashutil
from . import (
basepack,
constants,
......@@ -197,7 +197,7 @@ class historypack(basepack.basepack):
def _findsection(self, name):
params = self.params
namehash = hashlib.sha1(name).digest()
namehash = hashutil.sha1(name).digest()
fanoutkey = struct.unpack(
params.fanoutstruct, namehash[: params.fanoutprefix]
)[0]
......@@ -499,7 +499,7 @@ class mutablehistorypack(basepack.mutablebasepack):
# Record metadata for the index
self.files[filename] = (sectionstart, sectionlen)
node = hashlib.sha1(filename).digest()
node = hashutil.sha1(filename).digest()
self.entries[node] = node
def close(self, ledger=None):
......@@ -517,7 +517,7 @@ class mutablehistorypack(basepack.mutablebasepack):
nodeindexlength = self.NODEINDEXENTRYLENGTH
files = (
(hashlib.sha1(filename).digest(), filename, offset, size)
(hashutil.sha1(filename).digest(), filename, offset, size)
for filename, (offset, size) in pycompat.iteritems(self.files)
)
files = sorted(files)
......
......@@ -8,7 +8,6 @@ from __future__ import absolute_import
import collections
import errno
import hashlib
import os
import stat
import struct
......@@ -24,6 +23,7 @@ from mercurial import (
util,
)
from mercurial.utils import (
hashutil,
storageutil,
stringutil,
)
......@@ -39,12 +39,12 @@ def isenabled(repo):
def getcachekey(reponame, file, id):
pathhash = node.hex(hashlib.sha1(file).digest())
pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
def getlocalkey(file, id):
pathhash = node.hex(hashlib.sha1(file).digest())
pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(pathhash, id)
......
......@@ -45,7 +45,6 @@ option to ``sqlite`` to enable new repositories to use SQLite for storage.
from __future__ import absolute_import
import hashlib
import sqlite3
import struct
import threading
......@@ -75,7 +74,10 @@ from mercurial.interfaces import (
repository,
util as interfaceutil,
)
from mercurial.utils import storageutil
from mercurial.utils import (
hashutil,
storageutil,
)
try:
from mercurial import zstd
......@@ -807,7 +809,7 @@ class sqlitefilestore(object):
self._db, pathid, node, {}, {-1: None}, zstddctx=self._dctx
)
deltahash = hashlib.sha1(fulltext).digest()
deltahash = hashutil.sha1(fulltext).digest()
if self._compengine == b'zstd':
deltablob = self._cctx.compress(fulltext)
......@@ -837,7 +839,7 @@ class sqlitefilestore(object):
# Now create the tombstone delta and replace the delta on the censored
# node.
deltahash = hashlib.sha1(tombstone).digest()
deltahash = hashutil.sha1(tombstone).digest()
tombstonedeltaid = insertdelta(
self._db, COMPRESSION_NONE, deltahash, tombstone
)
......@@ -1004,7 +1006,7 @@ class sqlitefilestore(object):
# us to de-duplicate. The table is configured to ignore conflicts
# and it is faster to just insert and silently noop than to look
# first.
deltahash = hashlib.sha1(delta).digest()
deltahash = hashutil.sha1(delta).digest()
if self._compengine == b'zstd':
deltablob = self._cctx.compress(delta)
......
......@@ -38,6 +38,7 @@ typedef struct {
} nodetreenode;
typedef struct {
int abi_version;
int (*index_parents)(PyObject *, int, int *);
} Revlog_CAPI;
......@@ -3037,6 +3038,9 @@ static PyTypeObject rustlazyancestorsType = {
#endif /* WITH_RUST */
static Revlog_CAPI CAPI = {
/* increment the abi_version field upon each change in the Revlog_CAPI
struct or in the ABI of the listed functions */
1,
HgRevlogIndex_GetParents,
};
......
......@@ -41,7 +41,6 @@ Config
from __future__ import absolute_import
import hashlib
import inspect
import os
import re
......@@ -67,6 +66,7 @@ from . import (
)
from .utils import (
hashutil,
procutil,
stringutil,
)
......@@ -74,7 +74,7 @@ from .utils import (
def _hashlist(items):
"""return sha1 hexdigest for a list"""
return node.hex(hashlib.sha1(stringutil.pprint(items)).digest())
return node.hex(hashutil.sha1(stringutil.pprint(items)).digest())
# sensitive config sections affecting confighash
......
......@@ -1083,6 +1083,46 @@ class dirstate(object):
results[next(iv)] = st
return results
def _rust_status(self, matcher, list_clean):
# Force Rayon (Rust parallelism library) to respect the number of
# workers. This is a temporary workaround until Rust code knows
# how to read the config file.
numcpus = self._ui.configint(b"worker", b"numcpus")
if numcpus is not None:
encoding.environ.setdefault(b'RAYON_NUM_THREADS', b'%d' % numcpus)
workers_enabled = self._ui.configbool(b"worker", b"enabled", True)
if not workers_enabled:
encoding.environ[b"RAYON_NUM_THREADS"] = b"1"
(
lookup,
modified,
added,
removed,
deleted,
unknown,
clean,
) = rustmod.status(
self._map._rustmap,
matcher,
self._rootdir,
bool(list_clean),
self._lastnormaltime,
self._checkexec,
)
status = scmutil.status(
modified=modified,
added=added,
removed=removed,
deleted=deleted,
unknown=unknown,
ignored=[],
clean=clean,
)
return (lookup, status)
def status(self, match, subrepos, ignored, clean, unknown):
'''Determine the status of the working copy relative to the
dirstate and return a pair of (unsure, status), where status is of type
......@@ -1127,46 +1167,7 @@ class dirstate(object):
use_rust = False
if use_rust:
# Force Rayon (Rust parallelism library) to respect the number of
# workers. This is a temporary workaround until Rust code knows
# how to read the config file.
numcpus = self._ui.configint(b"worker", b"numcpus")
if numcpus is not None:
encoding.environ.setdefault(
b'RAYON_NUM_THREADS', b'%d' % numcpus
)
workers_enabled = self._ui.configbool(b"worker", b"enabled", True)
if not workers_enabled:
encoding.environ[b"RAYON_NUM_THREADS"] = b"1"
(
lookup,
modified,
added,
removed,
deleted,
unknown,
clean,
) = rustmod.status(
dmap._rustmap,
match,
self._rootdir,
bool(listclean),
self._lastnormaltime,
self._checkexec,
)
status = scmutil.status(
modified=modified,
added=added,
removed=removed,
deleted=deleted,
unknown=unknown,
ignored=ignored,
clean=clean,
)
return (lookup, status)
return self._rust_status(match, listclean)
def noop(f):
pass
......
......@@ -8,7 +8,6 @@
from __future__ import absolute_import
import collections
import hashlib
from .i18n import _
from .node import (
......@@ -40,7 +39,10 @@ from . import (
wireprototypes,
)
from .interfaces import repository
from .utils import stringutil
from .utils import (
hashutil,
stringutil,
)
urlerr = util.urlerr
urlreq = util.urlreq
......@@ -2705,7 +2707,7 @@ def check_heads(repo, their_heads, context):
Used by peer for unbundling.
"""
heads = repo.heads()
heads_hash = hashlib.sha1(b''.join(sorted(heads))).digest()
heads_hash = hashutil.sha1(b''.join(sorted(heads))).digest()
if not (
their_heads == [b'force']
or their_heads == heads
......
......@@ -9,7 +9,6 @@
from __future__ import absolute_import
import errno
import hashlib
import os
import shutil
import stat
......@@ -48,7 +47,7 @@ from . import (
verify as verifymod,
vfs as vfsmod,
)
from .utils import hashutil
from .interfaces import repository as repositorymod
release = lock.release
......@@ -738,7 +737,7 @@ def clone(
)
elif sharenamemode == b'remote':
sharepath = os.path.join(
sharepool, node.hex(hashlib.sha1(source).digest())
sharepool, node.hex(hashutil.sha1(source).digest())
)
else:
raise error.Abort(
......
......@@ -878,7 +878,9 @@ class ifilestorage(ifileindex, ifiledata, ifilemutation):
If individual revisions cannot have their revision content resolved,
the method is expected to set the ``skipread`` key to a set of nodes
that encountered problems.
that encountered problems. If set, the method can also add the node(s)
to ``safe_renamed`` in order to indicate nodes that may perform the
rename checks with currently accessible data.
The method yields objects conforming to the ``iverifyproblem``
interface.
......
......@@ -8,7 +8,6 @@
from __future__ import absolute_import
import errno
import hashlib
import os
import random
import sys
......@@ -74,6 +73,7 @@ from .interfaces import (
)
from .utils import (
hashutil,
procutil,
stringutil,
)
......@@ -1644,7 +1644,7 @@ class localrepository(object):
user aliases, consider calling ``scmutil.revrange()`` or
``repo.anyrevs([expr], user=True)``.
Returns a revset.abstractsmartset, which is a list-like interface
Returns a smartset.abstractsmartset, which is a list-like interface
that contains integer revisions.
'''
tree = revsetlang.spectree(expr, *args)
......@@ -2007,7 +2007,7 @@ class localrepository(object):
)
idbase = b"%.40f#%f" % (random.random(), time.time())
ha = hex(hashlib.sha1(idbase).digest())
ha = hex(hashutil.sha1(idbase).digest())
txnid = b'TXN:' + ha
self.hook(b'pretxnopen', throw=True, txnname=desc, txnid=txnid)
......
......@@ -8,7 +8,6 @@
from __future__ import absolute_import
import errno
import hashlib
import shutil
import stat
import struct
......@@ -39,6 +38,7 @@ from . import (
util,
worker,
)
from .utils import hashutil
_pack = struct.pack
_unpack = struct.unpack
......@@ -512,7 +512,7 @@ class mergestate(object):
"""hash the path of a local file context for storage in the .hg/merge
directory."""
return hex(hashlib.sha1(path).digest())
return hex(hashutil.sha1(path).digest())
def add(self, fcl, fco, fca, fd):
"""add a new (potentially?) conflicting file the merge state
......
......@@ -70,7 +70,6 @@ comment associated with each format for details.
from __future__ import absolute_import
import errno
import hashlib
import struct
from .i18n import _
......@@ -85,7 +84,10 @@ from . import (
pycompat,
util,
)
from .utils import dateutil
from .utils import (
dateutil,
hashutil,
)
parsers = policy.importmod('parsers')
......@@ -1028,7 +1030,7 @@ def _computecontentdivergentset(repo):
def makefoldid(relation, user):
folddigest = hashlib.sha1(user)
folddigest = hashutil.sha1(user)
for p in relation[0] + relation[1]:
folddigest.update(b'%d' % p.rev())
folddigest.update(p.node())
......
......@@ -12,7 +12,6 @@ import collections
import contextlib
import copy
import errno
import hashlib
import os
import re
import shutil
......@@ -41,6 +40,7 @@ from . import (
)
from .utils import (
dateutil,
hashutil,
procutil,
stringutil,
)
......@@ -2943,7 +2943,7 @@ def trydiff(
if not text:
text = b""
l = len(text)
s = hashlib.sha1(b'blob %d\0' % l)
s = hashutil.sha1(b'blob %d\0' % l)
s.update(text)
return hex(s.digest())
......
......@@ -243,49 +243,50 @@ class phasecache(object):
"""return a smartset for the given phases"""
self.loadphaserevs(repo) # ensure phase's sets are loaded
phases = set(phases)
publicphase = public in phases
if public not in phases:
# fast path: _phasesets contains the interesting sets,
# might only need a union and post-filtering.
if len(phases) == 1:
[p] = phases
revs = self._phasesets[p]
else:
revs = set.union(*[self._phasesets[p] for p in phases])
if publicphase:
# In this case, phases keeps all the *other* phases.
phases = set(allphases).difference(phases)
if not phases:
return smartset.fullreposet(repo)
# fast path: _phasesets contains the interesting sets,
# might only need a union and post-filtering.
revsneedscopy = False
if len(phases) == 1:
[p] = phases
revs = self._phasesets[p]
revsneedscopy = True # Don't modify _phasesets
else:
# revs has the revisions in all *other* phases.
revs = set.union(*[self._phasesets[p] for p in phases])
def _addwdir(wdirsubset, wdirrevs):
if wdirrev in wdirsubset and repo[None].phase() in phases:
if revsneedscopy:
wdirrevs = wdirrevs.copy()
# The working dir would never be in the # cache, but it was in
# the subset being filtered for its phase (or filtered out,
# depending on publicphase), so add it to the output to be
# included (or filtered out).
wdirrevs.add(wdirrev)
return wdirrevs
if not publicphase:
if repo.changelog.filteredrevs:
revs = revs - repo.changelog.filteredrevs
if subset is None:
return smartset.baseset(revs)
else:
if wdirrev in subset and repo[None].phase() in phases:
# The working dir would never be in the cache, but it was
# in the subset being filtered for its phase, so add it to
# the output.
revs.add(wdirrev)
revs = _addwdir(subset, revs)
return subset & smartset.baseset(revs)
else:
# phases keeps all the *other* phases.
phases = set(allphases).difference(phases)
if not phases:
return smartset.fullreposet(repo)
# revs has the revisions in all *other* phases.
if len(phases) == 1:
[p] = phases
revs = self._phasesets[p]
else:
revs = set.union(*[self._phasesets[p] for p in phases])
if subset is None:
subset = smartset.fullreposet(repo)
if wdirrev in subset and repo[None].phase() in phases:
# The working dir is in the subset being filtered, and its
# phase is in the phases *not* being returned, so add it to the
# set of revisions to filter out.
revs.add(wdirrev)
revs = _addwdir(subset, revs)
if not revs:
return subset
......
......@@ -9,7 +9,6 @@
from __future__ import absolute_import
import errno
import hashlib
from .i18n import _
from .node import (
......@@ -29,7 +28,10 @@ from . import (
pycompat,
util,
)
from .utils import stringutil
from .utils import (
hashutil,
stringutil,
)
def backupbundle(
......@@ -45,7 +47,7 @@ def backupbundle(
# Include a hash of all the nodes in the filename for uniqueness
allcommits = repo.set(b'%ln::%ln', bases, heads)
allhashes = sorted(c.hex() for c in allcommits)
totalhash = hashlib.sha1(b''.join(allhashes)).digest()
totalhash = hashutil.sha1(b''.join(allhashes)).digest()
name = b"%s/%s-%s-%s.hg" % (
backupdir,
short(node),
......
......@@ -2874,6 +2874,7 @@ class revlog(object):
)
state[b'skipread'] = set()
state[b'safe_renamed'] = set()
for rev in self:
node = self.node(rev)
......
......@@ -33,10 +33,10 @@ the concept.
from __future__ import absolute_import
import hashlib
import struct
from .. import error
from ..utils import hashutil
## sidedata type constant
# reserve a block for testing purposes.
......@@ -64,7 +64,7 @@ def sidedatawriteprocessor(rl, text, sidedata):
sidedata.sort()
rawtext = [SIDEDATA_HEADER.pack(len(sidedata))]
for key, value in sidedata:
digest = hashlib.sha1(value).digest()
digest = hashutil.sha1(value).digest()
rawtext.append(SIDEDATA_ENTRY.pack(key, len(value), digest))
for key, value in sidedata:
rawtext.append(value)
......@@ -85,7 +85,7 @@ def sidedatareadprocessor(rl, text):
# read the data associated with that entry
nextdataoffset = dataoffset + size
entrytext = text[dataoffset:nextdataoffset]
readdigest = hashlib.sha1(entrytext).digest()
readdigest = hashutil.sha1(entrytext).digest()
if storeddigest != readdigest:
raise error.SidedataHashError(key, storeddigest, readdigest)
sidedata[key] = entrytext
......
......@@ -9,7 +9,6 @@ from __future__ import absolute_import
import errno
import glob
import hashlib
import os
import posixpath
import re
......@@ -48,6 +47,7 @@ from . import (
)
from .utils import (
hashutil,
procutil,
stringutil,
)
......@@ -366,7 +366,7 @@ def filteredhash(repo, maxrev):
key = None
revs = sorted(r for r in cl.filteredrevs if r <= maxrev)
if revs:
s = hashlib.sha1()
s = hashutil.sha1()
for rev in revs:
s.update(b'%d;' % rev)
key = s.digest()
......@@ -748,7 +748,7 @@ def revrange(repo, specs, localalias=None):
Specifying a single revset is allowed.
Returns a ``revset.abstractsmartset`` which is a list-like interface over
Returns a ``smartset.abstractsmartset`` which is a list-like interface over
integer revisions.
"""
allspecs = []
......@@ -1760,6 +1760,7 @@ class progress(object):
self._updatebar(self.topic, self.pos, b"", self.unit, self.total)
def _printdebug(self, item):
unit = b''
if self.unit:
unit = b' ' + self.unit
if item:
......
......@@ -7,7 +7,6 @@
from __future__ import absolute_import
import hashlib
import os
from .i18n import _
......@@ -24,6 +23,7 @@ from . import (
scmutil,
util,
)
from .utils import hashutil
# Whether sparse features are enabled. This variable is intended to be
# temporary to facilitate porting sparse to core. It should eventually be
......@@ -205,12 +205,12 @@ def configsignature(repo, includetemp=True):
tempsignature = b'0'
if signature is None or (includetemp and tempsignature is None):
signature = hex(hashlib.sha1(repo.vfs.tryread(b'sparse')).digest())
signature = hex(hashutil.sha1(repo.vfs.tryread(b'sparse')).digest())
cache[b'signature'] = signature
if includetemp:
raw = repo.vfs.tryread(b'tempsparse')
tempsignature = hex(hashlib.sha1(raw).digest())
tempsignature = hex(hashutil.sha1(raw).digest())
cache[b'tempsignature'] = tempsignature
return b'%s %s' % (signature, tempsignature)
......
......@@ -24,6 +24,7 @@ from . import (
util,
)
from .utils import (
hashutil,
resourceutil,
stringutil,
)
......@@ -949,7 +950,7 @@ def validatesocket(sock):
# If a certificate fingerprint is pinned, use it and only it to
# validate the remote cert.
peerfingerprints = {
b'sha1': node.hex(hashlib.sha1(peercert).digest()),
b'sha1': node.hex(hashutil.sha1(peercert).digest()),
b'sha256': node.hex(hashlib.sha256(peercert).digest()),
b'sha512': node.hex(hashlib.sha512(peercert).digest()),
}
......
......@@ -9,7 +9,6 @@ from __future__ import absolute_import
import errno
import functools
import hashlib
import os
import stat
......@@ -25,6 +24,7 @@ from . import (
util,
vfs as vfsmod,
)
from .utils import hashutil
parsers = policy.importmod('parsers')
# how much bytes should be read from fncache in one read
......@@ -273,7 +273,7 @@ _maxshortdirslen = 8 * (_dirprefixlen + 1) - 4
def _hashencode(path, dotencode):
digest = node.hex(hashlib.sha1(path).digest())
digest = node.hex(hashutil.sha1(path).digest())
le = lowerencode(path[5:]).split(b'/') # skips prefix 'data/' or 'meta/'
parts = _auxencode(le, dotencode)
basename = parts[-1]
......
......@@ -9,7 +9,6 @@ from __future__ import absolute_import
import copy
import errno
import hashlib
import os
import re
import stat
......@@ -37,6 +36,7 @@ from . import (
)
from .utils import (
dateutil,
hashutil,
procutil,
stringutil,
)
......@@ -61,7 +61,7 @@ def _expandedabspath(path):
def _getstorehashcachename(remotepath):
'''get a unique filename for the store hash cache of a remote repository'''
return node.hex(hashlib.sha1(_expandedabspath(remotepath)).digest())[0:12]
return node.hex(hashutil.sha1(_expandedabspath(remotepath)).digest())[0:12]
class SubrepoAbort(error.Abort):
......@@ -514,7 +514,7 @@ class hgsubrepo(abstractsubrepo):
yield b'# %s\n' % _expandedabspath(remotepath)
vfs = self._repo.vfs
for relname in filelist:
filehash = node.hex(hashlib.sha1(vfs.tryread(relname)).digest())
filehash = node.hex(hashutil.sha1(vfs.tryread(relname)).digest())
yield b'%s = %s\n' % (relname, filehash)
@propertycache
......
MIT License
Copyright (c) 2017:
Marc Stevens
Cryptology Group
Centrum Wiskunde & Informatica
P.O. Box 94079, 1090 GB Amsterdam, Netherlands
marc@marc-stevens.nl
Dan Shumow
Microsoft Research
danshu@microsoft.com
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# sha1collisiondetection
Library and command line tool to detect SHA-1 collisions in files
Copyright 2017 Marc Stevens <marc@marc-stevens.nl>
Distributed under the MIT Software License.
See accompanying file LICENSE.txt or copy at https://opensource.org/licenses/MIT.
## Developers
- Marc Stevens, CWI Amsterdam (https://marc-stevens.nl)
- Dan Shumow, Microsoft Research (https://www.microsoft.com/en-us/research/people/danshu/)
## About
This library and command line tool were designed as near drop-in replacements for common SHA-1 libraries and sha1sum.
They will compute the SHA-1 hash of any given file and additionally will detect cryptanalytic collision attacks against SHA-1 present in each file. It is very fast and takes less than twice the amount of time as regular SHA-1.
More specifically they will detect any cryptanalytic collision attack against SHA-1 using any of the top 32 SHA-1 disturbance vectors with probability 1:
```
I(43,0), I(44,0), I(45,0), I(46,0), I(47,0), I(48,0), I(49,0), I(50,0), I(51,0), I(52,0),
I(46,2), I(47,2), I(48,2), I(49,2), I(50,2), I(51,2),
II(45,0), II(46,0), II(47,0), II(48,0), II(49,0), II(50,0), II(51,0), II(52,0), II(53,0), II(54,0), II(55,0), II(56,0),
II(46,2), II(49,2), II(50,2), II(51,2)
```
The possibility of false positives can be neglected as the probability is smaller than 2^-90.
The library supports both an indicator flag that applications can check and act on, as well as a special _safe-hash_ mode that returns the real SHA-1 hash when no collision was detected and a different _safe_ hash when a collision was detected.
Colliding files will have the same SHA-1 hash, but will have different unpredictable safe-hashes.
This essentially enables protection of applications against SHA-1 collisions with no further changes in the application, e.g., digital signature forgeries based on SHA-1 collisions automatically become invalid.
For the theoretical explanation of collision detection see the award-winning paper on _Counter-Cryptanalysis_:
Counter-cryptanalysis, Marc Stevens, CRYPTO 2013, Lecture Notes in Computer Science, vol. 8042, Springer, 2013, pp. 129-146,
https://marc-stevens.nl/research/papers/C13-S.pdf
## Compiling
Run:
```
make
```
## Command-line usage
There are two programs `bin/sha1dcsum` and `bin/sha1dcsum_partialcoll`.
The first program `bin/sha1dcsum` will detect and warn for files that were generated with a cryptanalytic SHA-1 collision attack like the one documented at https://shattered.io/.
The second program `bin/sha1dcsum_partialcoll` will detect and warn for files that were generated with a cryptanalytic collision attack against reduced-round SHA-1 (of which there are a few examples so far).
Examples:
```
bin/sha1dcsum test/sha1_reducedsha_coll.bin test/shattered-1.pdf
bin/sha1dcsum_partialcoll test/sha1reducedsha_coll.bin test/shattered-1.pdf
pipe_data | bin/sha1dcsum -
```
## Library usage
See the documentation in `lib/sha1.h`. Here is a simple example code snippet:
```
#include <sha1dc/sha1.h>
SHA1_CTX ctx;
unsigned char hash[20];
SHA1DCInit(&ctx);
/** disable safe-hash mode (safe-hash mode is enabled by default) **/
// SHA1DCSetSafeHash(&ctx, 0);
/** disable use of unavoidable attack conditions to speed up detection (enabled by default) **/
// SHA1DCSetUseUBC(&ctx, 0);
SHA1DCUpdate(&ctx, buffer, (unsigned)(size));
int iscoll = SHA1DCFinal(hash,&ctx);
<