diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..47bd1f0b8 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,83 @@ +# CI on Windows via appveyor +environment: + GIT_DAEMON_PATH: "C:\\Program Files\\Git\\mingw64\\libexec\\git-core" + CYGWIN_GIT_PATH: "C:\\cygwin\\bin;%GIT_DAEMON_PATH%" + CYGWIN64_GIT_PATH: "C:\\cygwin64\\bin;%GIT_DAEMON_PATH%" + + matrix: + ## MINGW + # + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7" + GIT_PATH: "%GIT_DAEMON_PATH%" + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" + GIT_PATH: "%GIT_DAEMON_PATH%" + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + GIT_PATH: "%GIT_DAEMON_PATH%" + - PYTHON: "C:\\Miniconda35-x64" + PYTHON_VERSION: "3.5" + IS_CONDA: "yes" + GIT_PATH: "%GIT_DAEMON_PATH%" + + # ## Cygwin + # # + # - PYTHON: "C:\\Miniconda-x64" + # PYTHON_VERSION: "2.7" + # IS_CONDA: "yes" + # GIT_PATH: "%CYGWIN_GIT_PATH%" + # - PYTHON: "C:\\Python34-x64" + # PYTHON_VERSION: "3.4" + # GIT_PATH: "%CYGWIN_GIT_PATH%" + # - PYTHON: "C:\\Python35-x64" + # PYTHON_VERSION: "3.5" + # GIT_PATH: "%CYGWIN64_GIT_PATH%" + + +install: + - set PATH=%PYTHON%;%PYTHON%\Scripts;%GIT_PATH%;%PATH% + + ## Print architecture, python & git used for debugging. + # + - | + uname -a + where git git-daemon python pip pip3 pip34 + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + + - IF "%IS_CONDA%"=="yes" ( + conda info -a & + conda install --yes --quiet pip + ) + - pip install nose ddt wheel coveralls + - IF "%PYTHON_VERSION%"=="2.7" ( + pip install mock + ) + + ## Copied from `init-tests-after-clone.sh`. + # + - | + git submodule update --init --recursive + git fetch --tags + git tag __testing_point__ + git checkout master || git checkout -b master + git reset --hard HEAD~1 + git reset --hard HEAD~1 + git reset --hard HEAD~1 + git reset --hard __testing_point__ + + ## For commits performed with the default user. + - | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + + - pip install -e . + +build: false + +test_script: + - nosetests --with-coverage + +#on_success: +# - IF "%PYTHON_VERSION%"=="3.4" (coveralls) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..872b8eb4f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +git/test/fixtures/* eol=lf +init-tests-after-clone.sh diff --git a/.travis.yml b/.travis.yml index 31f2c00c7..ab766e7cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ git: install: - git submodule update --init --recursive - git fetch --tags - - pip install coveralls flake8 sphinx + - pip install coveralls flake8 ddt sphinx # generate some reflog as git-python tests need it (in master) - ./init-tests-after-clone.sh @@ -31,8 +31,9 @@ script: # Make sure we limit open handles to see if we are leaking them - ulimit -n 96 - ulimit -n - - nosetests -v --with-coverage - - flake8 - - cd doc && make html + - nosetests --with-coverage + - if [ "$TRAVIS_PYTHON_VERSION" == '3.4' ]; then flake8; fi + - if [ "$TRAVIS_PYTHON_VERSION" == '3.5' ]; then cd doc && make html; fi + - after_success: - coveralls diff --git a/README.md b/README.md index b7e2b139f..6632347f4 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,28 @@ ## GitPython -GitPython is a python library used to interact with git repositories, high-level like git-porcelain, or low-level like git-plumbing. +GitPython is a python library used to interact with git repositories, high-level like git-porcelain, +or low-level like git-plumbing. -It provides abstractions of git objects for easy access of repository data, and additionally allows you to access the git repository more directly using either a pure python implementation, or the faster, but more resource intensive git command implementation. +It provides abstractions of git objects for easy access of repository data, and additionally +allows you to access the git repository more directly using either a pure python implementation, +or the faster, but more resource intensive *git command* implementation. -The object database implementation is optimized for handling large quantities of objects and large datasets, which is achieved by using low-level structures and data streaming. +The object database implementation is optimized for handling large quantities of objects and large datasets, +which is achieved by using low-level structures and data streaming. ### REQUIREMENTS -GitPython needs the `git` executable to be installed on the system and available in your `PATH` for most operations. If it is not in your `PATH`, you can help GitPython find it by setting the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. +GitPython needs the `git` executable to be installed on the system and available +in your `PATH` for most operations. +If it is not in your `PATH`, you can help GitPython find it by setting +the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. * Git (1.7.x or newer) * Python 2.7 to 3.5, while python 2.6 is supported on a *best-effort basis*. -The list of dependencies are listed in `./requirements.txt` and `./test-requirements.txt`. The installer takes care of installing them for you. +The list of dependencies are listed in `./requirements.txt` and `./test-requirements.txt`. +The installer takes care of installing them for you. ### INSTALL @@ -31,7 +39,7 @@ Both commands will install the required package dependencies. A distribution package can be obtained for manual installation at: http://pypi.python.org/pypi/GitPython - + If you like to clone from source, you can do it like so: ```bash @@ -45,7 +53,7 @@ git submodule update --init --recursive #### Leakage of System Resources GitPython is not suited for long-running processes (like daemons) as it tends to -leak system resources. It was written in a time where destructors (as implemented +leak system resources. It was written in a time where destructors (as implemented in the `__del__` method) still ran deterministically. In case you still want to use it in such a context, you will want to search the @@ -61,17 +69,25 @@ as they are kept alive solely by their users, or not. ### RUNNING TESTS -*Important*: Right after cloning this repository, please be sure to have executed the `init-tests-after-clone.sh` script in the repository root. Otherwise you will encounter test failures. +*Important*: Right after cloning this repository, please be sure to have executed +the `./init-tests-after-clone.sh` script in the repository root. Otherwise +you will encounter test failures. + +On *Windows*, make sure you have `git-daemon` in your PATH. For MINGW-git, the `git-daemon.exe` +exists in `Git\mingw64\libexec\git-core\`; CYGWIN has no daemon, but should get along fine +with MINGW's. -The easiest way to run test is by using [tox](https://pypi.python.org/pypi/tox) a wrapper around virtualenv. It will take care of setting up environnements with the proper dependencies installed and execute test commands. To install it simply: +The easiest way to run tests is by using [tox](https://pypi.python.org/pypi/tox) +a wrapper around virtualenv. It will take care of setting up environnements with the proper +dependencies installed and execute test commands. To install it simply: pip install tox Then run: tox - - + + For more fine-grained control, you can use `nose`. ### Contributions @@ -84,7 +100,8 @@ Please have a look at the [contributions file][contributing]. * [Questions and Answers](http://stackexchange.com/filters/167317/gitpython) * Please post on stackoverflow and use the `gitpython` tag * [Issue Tracker](https://github.com/gitpython-developers/GitPython/issues) - * Post reproducible bugs and feature requests as a new issue. Please be sure to provide the following information if posting bugs: + * Post reproducible bugs and feature requests as a new issue. + Please be sure to provide the following information if posting bugs: * GitPython version (e.g. `import git; git.__version__`) * Python version (e.g. `python --version`) * The encountered stack-trace, if applicable @@ -100,7 +117,7 @@ Please have a look at the [contributions file][contributing]. * Finally, set the upcoming version in the `VERSION` file, usually be incrementing the patch level, and possibly by appending `-dev`. Probably you want to `git push` once more. - + ### LICENSE New BSD License. See the LICENSE file. @@ -113,7 +130,8 @@ New BSD License. See the LICENSE file. [![Stories in Ready](https://badge.waffle.io/gitpython-developers/GitPython.png?label=ready&title=Ready)](https://waffle.io/gitpython-developers/GitPython) [![Throughput Graph](https://graphs.waffle.io/gitpython-developers/GitPython/throughput.svg)](https://waffle.io/gitpython-developers/GitPython/metrics/throughput) -Now that there seems to be a massive user base, this should be motivation enough to let git-python return to a proper state, which means +Now that there seems to be a massive user base, this should be motivation enough to let git-python +return to a proper state, which means * no open pull requests * no open issues describing bugs diff --git a/doc/source/conf.py b/doc/source/conf.py index add686d3f..2df3bbb63 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -50,7 +50,8 @@ # built documents. # # The short X.Y version. -VERSION = open(os.path.join(os.path.dirname(__file__),"..", "..", 'VERSION')).readline().strip() +with open(os.path.join(os.path.dirname(__file__),"..", "..", 'VERSION')) as fd: + VERSION = fd.readline().strip() version = VERSION # The full version, including alpha/beta/rc tags. release = VERSION diff --git a/git/cmd.py b/git/cmd.py index ceea24425..f4f5f99a1 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -4,64 +4,53 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -import os -import os.path -import sys -import select -import logging -import threading -import errno -import mmap - -from git.odict import OrderedDict from contextlib import contextmanager +import io +import logging +import os import signal from subprocess import ( call, Popen, PIPE ) +import subprocess +import sys +import threading - -from .util import ( - LazyMixin, - stream_copy, - WaitGroup -) -from .exc import ( - GitCommandError, - GitCommandNotFound -) from git.compat import ( string_types, defenc, force_bytes, PY3, - bchr, # just to satisfy flake8 on py3 unicode, safe_decode, + is_posix, + is_win, ) +from git.exc import CommandError +from git.odict import OrderedDict -execute_kwargs = ('istream', 'with_keep_cwd', 'with_extended_output', - 'with_exceptions', 'as_process', 'stdout_as_string', - 'output_stream', 'with_stdout', 'kill_after_timeout', - 'universal_newlines') +from .exc import ( + GitCommandError, + GitCommandNotFound +) +from .util import ( + LazyMixin, + stream_copy, +) -log = logging.getLogger('git.cmd') -log.addHandler(logging.NullHandler()) -__all__ = ('Git', ) +execute_kwargs = set(('istream', 'with_keep_cwd', 'with_extended_output', + 'with_exceptions', 'as_process', 'stdout_as_string', + 'output_stream', 'with_stdout', 'kill_after_timeout', + 'universal_newlines', 'shell')) -if sys.platform != 'win32': - WindowsError = OSError +log = logging.getLogger('git.cmd') +log.addHandler(logging.NullHandler()) -if PY3: - _bchr = bchr -else: - def _bchr(c): - return c -# get custom byte character handling +__all__ = ('Git',) # ============================================================================== @@ -70,154 +59,63 @@ def _bchr(c): # Documentation ## @{ -def handle_process_output(process, stdout_handler, stderr_handler, finalizer): +def handle_process_output(process, stdout_handler, stderr_handler, finalizer, decode_streams=True): """Registers for notifications to lean that process output is ready to read, and dispatches lines to - the respective line handlers. We are able to handle carriage returns in case progress is sent by that - mean. For performance reasons, we only apply this to stderr. + the respective line handlers. This function returns once the finalizer returns + :return: result of finalizer :param process: subprocess.Popen instance :param stdout_handler: f(stdout_line_string), or None :param stderr_hanlder: f(stderr_line_string), or None - :param finalizer: f(proc) - wait for proc to finish""" - fdmap = {process.stdout.fileno(): (stdout_handler, [b'']), - process.stderr.fileno(): (stderr_handler, [b''])} - - def _parse_lines_from_buffer(buf): - line = b'' - bi = 0 - lb = len(buf) - while bi < lb: - char = _bchr(buf[bi]) - bi += 1 - - if char in (b'\r', b'\n') and line: - yield bi, line - line = b'' - else: - line += char - # END process parsed line - # END while file is not done reading - # end - - def _read_lines_from_fno(fno, last_buf_list): - buf = os.read(fno, mmap.PAGESIZE) - buf = last_buf_list[0] + buf - - bi = 0 - for bi, line in _parse_lines_from_buffer(buf): - yield line - # for each line to parse from the buffer - - # keep remainder - last_buf_list[0] = buf[bi:] - - def _dispatch_single_line(line, handler): - line = line.decode(defenc) - if line and handler: - handler(line) - # end dispatch helper - # end single line helper - - def _dispatch_lines(fno, handler, buf_list): - lc = 0 - for line in _read_lines_from_fno(fno, buf_list): - _dispatch_single_line(line, handler) - lc += 1 - # for each line - return lc - # end - - def _deplete_buffer(fno, handler, buf_list, wg=None): - lc = 0 - while True: - line_count = _dispatch_lines(fno, handler, buf_list) - lc += line_count - if line_count == 0: - break - # end deplete buffer - - if buf_list[0]: - _dispatch_single_line(buf_list[0], handler) - lc += 1 - # end - - if wg: - wg.done() - - return lc - # end - - if hasattr(select, 'poll'): - # poll is preferred, as select is limited to file handles up to 1024 ... . This could otherwise be - # an issue for us, as it matters how many handles our own process has - poll = select.poll() - READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR - CLOSED = select.POLLHUP | select.POLLERR - - poll.register(process.stdout, READ_ONLY) - poll.register(process.stderr, READ_ONLY) - - closed_streams = set() - while True: - # no timeout - - try: - poll_result = poll.poll() - except select.error as e: - if e.args[0] == errno.EINTR: - continue - raise - # end handle poll exception - - for fd, result in poll_result: - if result & CLOSED: - closed_streams.add(fd) - else: - _dispatch_lines(fd, *fdmap[fd]) - # end handle closed stream - # end for each poll-result tuple - - if len(closed_streams) == len(fdmap): - break - # end its all done - # end endless loop - - # Depelete all remaining buffers - for fno, (handler, buf_list) in fdmap.items(): - _deplete_buffer(fno, handler, buf_list) - # end for each file handle - - for fno in fdmap.keys(): - poll.unregister(fno) - # end don't forget to unregister ! - else: - # Oh ... probably we are on windows. select.select() can only handle sockets, we have files - # The only reliable way to do this now is to use threads and wait for both to finish - # Since the finalizer is expected to wait, we don't have to introduce our own wait primitive - # NO: It's not enough unfortunately, and we will have to sync the threads - wg = WaitGroup() - for fno, (handler, buf_list) in fdmap.items(): - wg.add(1) - t = threading.Thread(target=lambda: _deplete_buffer(fno, handler, buf_list, wg)) - t.start() - # end - # NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's - # actually started, which could make the wait() call to just return because the thread is not yet - # active - wg.wait() - # end + :param finalizer: f(proc) - wait for proc to finish + :param decode_streams: + Assume stdout/stderr streams are binary and decode them vefore pushing \ + their contents to handlers. + Set it to False if `universal_newline == True` (then streams are in text-mode) + or if decoding must happen later (i.e. for Diffs). + """ + # Use 2 "pupm" threads and wait for both to finish. + def pump_stream(cmdline, name, stream, is_decode, handler): + try: + for line in stream: + if handler: + if is_decode: + line = line.decode(defenc) + handler(line) + except Exception as ex: + log.error("Pumping %r of cmd(%s) failed due to: %r", name, cmdline, ex) + raise CommandError(['<%s-pump>' % name] + cmdline, ex) + finally: + stream.close() + + cmdline = getattr(process, 'args', '') # PY3+ only + if not isinstance(cmdline, (tuple, list)): + cmdline = cmdline.split() + threads = [] + for name, stream, handler in ( + ('stdout', process.stdout, stdout_handler), + ('stderr', process.stderr, stderr_handler), + ): + t = threading.Thread(target=pump_stream, + args=(cmdline, name, stream, decode_streams, handler)) + t.setDaemon(True) + t.start() + threads.append(t) + + for t in threads: + t.join() return finalizer(process) def dashify(string): return string.replace('_', '-') - + def slots_to_dict(self, exclude=()): return dict((s, getattr(self, s)) for s in self.__slots__ if s not in exclude) - + def dict_to_slots_and__excluded_are_none(self, d, excluded=()): for k, v in d.items(): @@ -227,6 +125,15 @@ def dict_to_slots_and__excluded_are_none(self, d, excluded=()): ## -- End Utilities -- @} +# value of Windows process creation flag taken from MSDN +CREATE_NO_WINDOW = 0x08000000 + +## CREATE_NEW_PROCESS_GROUP is needed to allow killing it afterwards, +# seehttps://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal +PROC_CREATIONFLAGS = (CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP + if is_win + else 0) + class Git(LazyMixin): @@ -246,35 +153,31 @@ class Git(LazyMixin): """ __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version_info", "_git_options", "_environment") - + _excluded_ = ('cat_file_all', 'cat_file_header', '_version_info') - + def __getstate__(self): return slots_to_dict(self, exclude=self._excluded_) - + def __setstate__(self, d): dict_to_slots_and__excluded_are_none(self, d, excluded=self._excluded_) - + # CONFIGURATION # The size in bytes read from stdout when copying git's output to another stream - max_chunk_size = 1024 * 64 + max_chunk_size = io.DEFAULT_BUFFER_SIZE git_exec_name = "git" # default that should work on linux and windows - git_exec_name_win = "git.cmd" # alternate command name, windows only # Enables debugging of GitPython's git commands GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) - # value of Windows process creation flag taken from MSDN - CREATE_NO_WINDOW = 0x08000000 - # Provide the full path to the git executable. Otherwise it assumes git is in the path _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" GIT_PYTHON_GIT_EXECUTABLE = os.environ.get(_git_exec_env_var, git_exec_name) # If True, a shell will be used when executing git commands. - # This should only be desirable on windows, see https://github.com/gitpython-developers/GitPython/pull/126 - # for more information + # This should only be desirable on Windows, see https://github.com/gitpython-developers/GitPython/pull/126 + # and check `git/test_repo.py:TestRepo.test_untracked_files()` TC for an example where it is required. # Override this value using `Git.USE_SHELL = True` USE_SHELL = False @@ -315,9 +218,10 @@ def __del__(self): # try to kill it try: - os.kill(proc.pid, 2) # interrupt signal + proc.terminate() proc.wait() # ensure process goes away - except (OSError, WindowsError): + except OSError as ex: + log.info("Ignored error after process has dies: %r", ex) pass # ignore error when process already died except AttributeError: # try windows @@ -339,7 +243,7 @@ def wait(self, stderr=b''): if stderr is None: stderr = b'' stderr = force_bytes(stderr) - + status = self.proc.wait() def read_all_from_possibly_closed_stream(stream): @@ -447,6 +351,7 @@ def next(self): line = self.readline() if not line: raise StopIteration + return line def __del__(self): @@ -517,6 +422,7 @@ def execute(self, command, kill_after_timeout=None, with_stdout=True, universal_newlines=False, + shell=None, **subprocess_kwargs ): """Handles executing the command on the shell and consumes and returns @@ -574,6 +480,9 @@ def execute(self, command, :param universal_newlines: if True, pipes will be opened as text, and lines are split at all known line endings. + :param shell: + Whether to invoke commands through a shell (see `Popen(..., shell=True)`). + It overrides :attr:`USE_SHELL` if it is not `None`. :param kill_after_timeout: To specify a timeout in seconds for the git command, after which the process should be killed. This will have no effect if as_process is set to True. It is @@ -619,18 +528,18 @@ def execute(self, command, env["LC_ALL"] = "C" env.update(self._environment) - if sys.platform == 'win32': - cmd_not_found_exception = WindowsError + if is_win: + cmd_not_found_exception = OSError if kill_after_timeout: - raise GitCommandError('"kill_after_timeout" feature is not supported on Windows.') + raise GitCommandError(command, '"kill_after_timeout" feature is not supported on Windows.') else: if sys.version_info[0] > 2: - cmd_not_found_exception = FileNotFoundError # NOQA # this is defined, but flake8 doesn't know + cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable else: cmd_not_found_exception = OSError # end handle - creationflags = self.CREATE_NO_WINDOW if sys.platform == 'win32' else 0 + log.debug("Popen(%s, cwd=%s, universal_newlines=%s", command, cwd, universal_newlines) try: proc = Popen(command, env=env, @@ -639,21 +548,22 @@ def execute(self, command, stdin=istream, stderr=PIPE, stdout=PIPE if with_stdout else open(os.devnull, 'wb'), - shell=self.USE_SHELL, - close_fds=(os.name == 'posix'), # unsupported on windows + shell=shell is not None and shell or self.USE_SHELL, + close_fds=(is_posix), # unsupported on windows universal_newlines=universal_newlines, - creationflags=creationflags, + creationflags=PROC_CREATIONFLAGS, **subprocess_kwargs ) except cmd_not_found_exception as err: - raise GitCommandNotFound(str(err)) + raise GitCommandNotFound(command, err) if as_process: return self.AutoInterrupt(proc, command) def _kill_process(pid): """ Callback method to kill a process. """ - p = Popen(['ps', '--ppid', str(pid)], stdout=PIPE, creationflags=creationflags) + p = Popen(['ps', '--ppid', str(pid)], stdout=PIPE, + creationflags=PROC_CREATIONFLAGS) child_pids = [] for line in p.stdout: if len(line.split()) > 0: @@ -679,7 +589,7 @@ def _kill_process(pid): if kill_after_timeout: kill_check = threading.Event() - watchdog = threading.Timer(kill_after_timeout, _kill_process, args=(proc.pid, )) + watchdog = threading.Timer(kill_after_timeout, _kill_process, args=(proc.pid,)) # Wait for the process to return status = 0 @@ -766,10 +676,7 @@ def update_environment(self, **kwargs): for key, value in kwargs.items(): # set value if it is None if value is not None: - if key in self._environment: - old_env[key] = self._environment[key] - else: - old_env[key] = None + old_env[key] = self._environment.get(key) self._environment[key] = value # remove key from environment if its value is None elif key in self._environment: @@ -885,12 +792,8 @@ def _call_process(self, method, *args, **kwargs): :return: Same as ``execute``""" # Handle optional arguments prior to calling transform_kwargs # otherwise these'll end up in args, which is bad. - _kwargs = dict() - for kwarg in execute_kwargs: - try: - _kwargs[kwarg] = kwargs.pop(kwarg) - except KeyError: - pass + _kwargs = {k: v for k, v in kwargs.items() if k in execute_kwargs} + kwargs = {k: v for k, v in kwargs.items() if k not in execute_kwargs} insert_after_this_arg = kwargs.pop('insert_kwargs_after', None) @@ -910,48 +813,17 @@ def _call_process(self, method, *args, **kwargs): args = ext_args[:index + 1] + opt_args + ext_args[index + 1:] # end handle kwargs - def make_call(): - call = [self.GIT_PYTHON_GIT_EXECUTABLE] - - # add the git options, the reset to empty - # to avoid side_effects - call.extend(self._git_options) - self._git_options = () + call = [self.GIT_PYTHON_GIT_EXECUTABLE] - call.extend([dashify(method)]) - call.extend(args) - return call - # END utility to recreate call after changes + # add the git options, the reset to empty + # to avoid side_effects + call.extend(self._git_options) + self._git_options = () - if sys.platform == 'win32': - try: - try: - return self.execute(make_call(), **_kwargs) - except WindowsError: - # did we switch to git.cmd already, or was it changed from default ? permanently fail - if self.GIT_PYTHON_GIT_EXECUTABLE != self.git_exec_name: - raise - # END handle overridden variable - type(self).GIT_PYTHON_GIT_EXECUTABLE = self.git_exec_name_win + call.append(dashify(method)) + call.extend(args) - try: - return self.execute(make_call(), **_kwargs) - finally: - import warnings - msg = "WARNING: Automatically switched to use git.cmd as git executable" - msg += ", which reduces performance by ~70%." - msg += "It is recommended to put git.exe into the PATH or to " - msg += "set the %s " % self._git_exec_env_var - msg += "environment variable to the executable's location" - warnings.warn(msg) - # END print of warning - # END catch first failure - except WindowsError: - raise WindowsError("The system cannot find or execute the file at %r" % self.GIT_PYTHON_GIT_EXECUTABLE) - # END provide better error message - else: - return self.execute(make_call(), **_kwargs) - # END handle windows default installation + return self.execute(call, **_kwargs) def _parse_object_header(self, header_line): """ @@ -1040,6 +912,10 @@ def clear_cache(self): Currently persistent commands will be interrupted. :return: self""" + for cmd in (self.cat_file_all, self.cat_file_header): + if cmd: + cmd.__del__() + self.cat_file_all = None self.cat_file_header = None return self diff --git a/git/compat.py b/git/compat.py index b35724749..441a37617 100644 --- a/git/compat.py +++ b/git/compat.py @@ -7,15 +7,15 @@ """utilities to help provide compatibility with python 3""" # flake8: noqa +import locale +import os import sys from gitdb.utils.compat import ( - PY3, xrange, MAXSIZE, izip, ) - from gitdb.utils.encoding import ( string_types, text_type, @@ -23,7 +23,13 @@ force_text ) + +PY3 = sys.version_info[0] >= 3 +is_win = (os.name == 'nt') +is_posix = (os.name == 'posix') +is_darwin = (os.name == 'darwin') defenc = sys.getdefaultencoding() + if PY3: import io FileType = io.IOBase @@ -57,7 +63,28 @@ def safe_decode(s): return s elif isinstance(s, bytes): return s.decode(defenc, 'replace') - raise TypeError('Expected bytes or text, but got %r' % (s,)) + elif s is not None: + raise TypeError('Expected bytes or text, but got %r' % (s,)) + + +def safe_encode(s): + """Safely decodes a binary string to unicode""" + if isinstance(s, unicode): + return s.encode(defenc) + elif isinstance(s, bytes): + return s + elif s is not None: + raise TypeError('Expected bytes or text, but got %r' % (s,)) + + +def win_encode(s): + """Encode unicodes for process arguments on Windows.""" + if isinstance(s, unicode): + return s.encode(locale.getpreferredencoding(False)) + elif isinstance(s, bytes): + return s + elif s is not None: + raise TypeError('Expected bytes or text, but got %r' % (s,)) def with_metaclass(meta, *bases): @@ -73,9 +100,19 @@ def __new__(cls, name, nbases, d): # we set the __metaclass__ attribute explicitly if not PY3 and '___metaclass__' not in d: d['__metaclass__'] = meta - # end return meta(name, bases, d) - # end - # end metaclass return metaclass(meta.__name__ + 'Helper', None, {}) - # end handle py2 + + +## From https://docs.python.org/3.3/howto/pyporting.html +class UnicodeMixin(object): + + """Mixin class to handle defining the proper __str__/__unicode__ + methods in Python 2 or 3.""" + + if PY3: + def __str__(self): + return self.__unicode__() + else: # Python 2 + def __str__(self): + return self.__unicode__().encode(defenc) diff --git a/git/config.py b/git/config.py index 5bd10975d..ad6192ff2 100644 --- a/git/config.py +++ b/git/config.py @@ -388,23 +388,18 @@ def read(self): while files_to_read: file_path = files_to_read.pop(0) fp = file_path - close_fp = False + file_ok = False - # assume a path if it is not a file-object - if not hasattr(fp, "seek"): + if hasattr(fp, "seek"): + self._read(fp, fp.name) + else: + # assume a path if it is not a file-object try: - fp = open(file_path, 'rb') - close_fp = True + with open(file_path, 'rb') as fp: + file_ok = True + self._read(fp, fp.name) except IOError: continue - # END fp handling - - try: - self._read(fp, fp.name) - finally: - if close_fp: - fp.close() - # END read-handling # Read includes and append those that we didn't handle yet # We expect all paths to be normalized and absolute (and will assure that is the case) @@ -413,7 +408,7 @@ def read(self): if include_path.startswith('~'): include_path = os.path.expanduser(include_path) if not os.path.isabs(include_path): - if not close_fp: + if not file_ok: continue # end ignore relative paths if we don't know the configuration file path assert os.path.isabs(file_path), "Need absolute paths to be sure our cycle checks will work" @@ -477,34 +472,25 @@ def write(self): # end fp = self._file_or_files - close_fp = False # we have a physical file on disk, so get a lock - if isinstance(fp, string_types + (FileType, )): + is_file_lock = isinstance(fp, string_types + (FileType, )) + if is_file_lock: self._lock._obtain_lock() - # END get lock for physical files - - if not hasattr(fp, "seek"): - fp = open(self._file_or_files, "wb") - close_fp = True - else: - fp.seek(0) - # make sure we do not overwrite into an existing file - if hasattr(fp, 'truncate'): - fp.truncate() - # END - # END handle stream or file - - # WRITE DATA try: - self._write(fp) + if not hasattr(fp, "seek"): + with open(self._file_or_files, "wb") as fp: + self._write(fp) + else: + fp.seek(0) + # make sure we do not overwrite into an existing file + if hasattr(fp, 'truncate'): + fp.truncate() + self._write(fp) finally: - if close_fp: - fp.close() - # END data writing - - # we do not release the lock - it will be done automatically once the - # instance vanishes + # we release the lock - it will not vanish automatically in PY3.5+ + if is_file_lock: + self._lock._release_lock() def _assure_writable(self, method_name): if self.read_only: diff --git a/git/diff.py b/git/diff.py index fb8faaf6c..35c7ff86a 100644 --- a/git/diff.py +++ b/git/diff.py @@ -15,6 +15,8 @@ defenc, PY3 ) +from git.cmd import handle_process_output +from git.util import finalize_process __all__ = ('Diffable', 'DiffIndex', 'Diff', 'NULL_TREE') @@ -145,10 +147,10 @@ def diff(self, other=Index, paths=None, create_patch=False, **kwargs): kwargs['as_process'] = True proc = diff_cmd(*self._process_diff_args(args), **kwargs) - diff_method = Diff._index_from_raw_format - if create_patch: - diff_method = Diff._index_from_patch_format - index = diff_method(self.repo, proc.stdout) + diff_method = (Diff._index_from_patch_format + if create_patch + else Diff._index_from_raw_format) + index = diff_method(self.repo, proc) proc.wait() return index @@ -397,13 +399,18 @@ def _pick_best_path(cls, path_match, rename_match, path_fallback_match): return None @classmethod - def _index_from_patch_format(cls, repo, stream): + def _index_from_patch_format(cls, repo, proc): """Create a new DiffIndex from the given text which must be in patch format :param repo: is the repository we are operating on - it is required :param stream: result of 'git diff' as a stream (supporting file protocol) :return: git.DiffIndex """ + + ## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. + text = [] + handle_process_output(proc, text.append, None, finalize_process, decode_streams=False) + # for now, we have to bake the stream - text = stream.read() + text = b''.join(text) index = DiffIndex() previous_header = None for header in cls.re_header.finditer(text): @@ -450,17 +457,19 @@ def _index_from_patch_format(cls, repo, stream): return index @classmethod - def _index_from_raw_format(cls, repo, stream): + def _index_from_raw_format(cls, repo, proc): """Create a new DiffIndex from the given stream which must be in raw format. :return: git.DiffIndex""" # handles # :100644 100644 687099101... 37c5e30c8... M .gitignore + index = DiffIndex() - for line in stream.readlines(): + + def handle_diff_line(line): line = line.decode(defenc) if not line.startswith(":"): - continue - # END its not a valid diff line + return + meta, _, path = line[1:].partition('\t') old_mode, new_mode, a_blob_id, b_blob_id, change_type = meta.split(None, 4) path = path.strip() @@ -489,6 +498,7 @@ def _index_from_raw_format(cls, repo, stream): diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode, new_file, deleted_file, rename_from, rename_to, '', change_type) index.append(diff) - # END for each line + + handle_process_output(proc, handle_diff_line, None, finalize_process, decode_streams=False) return index diff --git a/git/exc.py b/git/exc.py index 34382ecd5..47215c21e 100644 --- a/git/exc.py +++ b/git/exc.py @@ -6,8 +6,7 @@ """ Module containing all exceptions thrown througout the git package, """ from gitdb.exc import * # NOQA - -from git.compat import defenc +from git.compat import UnicodeMixin, safe_decode, string_types class InvalidGitRepositoryError(Exception): @@ -22,29 +21,57 @@ class NoSuchPathError(OSError): """ Thrown if a path could not be access by the system. """ -class GitCommandNotFound(Exception): +class CommandError(UnicodeMixin, Exception): + """Base class for exceptions thrown at every stage of `Popen()` execution. + + :param command: + A non-empty list of argv comprising the command-line. + """ + + #: A unicode print-format with 2 `%s for `` and the rest, + #: e.g. + #: u"'%s' failed%s" + _msg = u"Cmd('%s') failed%s" + + def __init__(self, command, status=None, stderr=None, stdout=None): + if not isinstance(command, (tuple, list)): + command = command.split() + self.command = command + self.status = status + if status: + if isinstance(status, Exception): + status = u"%s('%s')" % (type(status).__name__, safe_decode(str(status))) + else: + try: + status = u'exit code(%s)' % int(status) + except: + s = safe_decode(str(status)) + status = u"'%s'" % s if isinstance(status, string_types) else s + + self._cmd = safe_decode(command[0]) + self._cmdline = u' '.join(safe_decode(i) for i in command) + self._cause = status and u" due to: %s" % status or "!" + self.stdout = stdout and u"\n stdout: '%s'" % safe_decode(stdout) or '' + self.stderr = stderr and u"\n stderr: '%s'" % safe_decode(stderr) or '' + + def __unicode__(self): + return (self._msg + "\n cmdline: %s%s%s") % ( + self._cmd, self._cause, self._cmdline, self.stdout, self.stderr) + + +class GitCommandNotFound(CommandError): """Thrown if we cannot find the `git` executable in the PATH or at the path given by the GIT_PYTHON_GIT_EXECUTABLE environment variable""" - pass + def __init__(self, command, cause): + super(GitCommandNotFound, self).__init__(command, cause) + self._msg = u"Cmd('%s') not found%s" -class GitCommandError(Exception): +class GitCommandError(CommandError): """ Thrown if execution of the git command fails with non-zero status code. """ def __init__(self, command, status, stderr=None, stdout=None): - self.stderr = stderr - self.stdout = stdout - self.status = status - self.command = command - - def __str__(self): - ret = "'%s' returned with exit code %i" % \ - (' '.join(str(i) for i in self.command), self.status) - if self.stderr: - ret += "\nstderr: '%s'" % self.stderr.decode(defenc) - if self.stdout: - ret += "\nstdout: '%s'" % self.stdout.decode(defenc) - return ret + super(GitCommandError, self).__init__(command, status, stderr, stdout) class CheckoutError(Exception): @@ -81,19 +108,13 @@ class UnmergedEntriesError(CacheError): entries in the cache""" -class HookExecutionError(Exception): +class HookExecutionError(CommandError): """Thrown if a hook exits with a non-zero exit code. It provides access to the exit code and the string returned via standard output""" - def __init__(self, command, status, stdout, stderr): - self.command = command - self.status = status - self.stdout = stdout - self.stderr = stderr - - def __str__(self): - return ("'%s' hook returned with exit code %i\nstdout: '%s'\nstderr: '%s'" - % (self.command, self.status, self.stdout, self.stderr)) + def __init__(self, command, status, stderr=None, stdout=None): + super(HookExecutionError, self).__init__(command, status, stderr, stdout) + self._msg = u"Hook('%s') failed%s" class RepositoryDirtyError(Exception): diff --git a/git/index/base.py b/git/index/base.py index 86eda41e6..9b6d28ab1 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -46,7 +46,8 @@ string_types, force_bytes, defenc, - mviter + mviter, + is_win ) from git.util import ( @@ -136,7 +137,7 @@ def _set_cache_(self, attr): # which happens during read-tree. # In this case, we will just read the memory in directly. # Its insanely bad ... I am disappointed ! - allow_mmap = (os.name != 'nt' or sys.version_info[1] > 5) + allow_mmap = (is_win or sys.version_info[1] > 5) stream = file_contents_ro(fd, stream=True, allow_mmap=allow_mmap) try: @@ -213,8 +214,8 @@ def write(self, file_path=None, ignore_extension_data=False): self.entries lfd = LockedFD(file_path or self._file_path) stream = lfd.open(write=True, stream=True) - ok = False + ok = False try: self._serialize(stream, ignore_extension_data) ok = True @@ -601,14 +602,13 @@ def _store_path(self, filepath, fprogress): stream = None if S_ISLNK(st.st_mode): # in PY3, readlink is string, but we need bytes. In PY2, it's just OS encoded bytes, we assume UTF-8 - stream = BytesIO(force_bytes(os.readlink(filepath), encoding=defenc)) + open_stream = lambda: BytesIO(force_bytes(os.readlink(filepath), encoding=defenc)) else: - stream = open(filepath, 'rb') - # END handle stream - fprogress(filepath, False, filepath) - istream = self.repo.odb.store(IStream(Blob.type, st.st_size, stream)) - fprogress(filepath, True, filepath) - stream.close() + open_stream = lambda: open(filepath, 'rb') + with open_stream() as stream: + fprogress(filepath, False, filepath) + istream = self.repo.odb.store(IStream(Blob.type, st.st_size, stream)) + fprogress(filepath, True, filepath) return BaseIndexEntry((stat_mode_to_index_mode(st.st_mode), istream.binsha, 0, to_native_path_linux(filepath))) @@ -1059,7 +1059,7 @@ def handle_stderr(proc, iter_checked_out_files): # END for each possible ending # END for each line if unknown_lines: - raise GitCommandError(("git-checkout-index", ), 128, stderr) + raise GitCommandError(("git-checkout-index",), 128, stderr) if failed_files: valid_files = list(set(iter_checked_out_files) - set(failed_files)) raise CheckoutError( @@ -1090,6 +1090,7 @@ def handle_stderr(proc, iter_checked_out_files): kwargs['as_process'] = True kwargs['istream'] = subprocess.PIPE proc = self.repo.git.checkout_index(args, **kwargs) + # FIXME: Reading from GIL! make_exc = lambda: GitCommandError(("git-checkout-index",) + tuple(args), 128, proc.stderr.read()) checked_out_files = list() diff --git a/git/index/fun.py b/git/index/fun.py index 4dd32b193..74ac929ee 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -14,7 +14,8 @@ import os import subprocess -from git.util import IndexFileSHA1Writer +from git.util import IndexFileSHA1Writer, finalize_process +from git.cmd import PROC_CREATIONFLAGS, handle_process_output from git.exc import ( UnmergedEntriesError, HookExecutionError @@ -40,9 +41,13 @@ from gitdb.base import IStream from gitdb.typ import str_tree_type from git.compat import ( + PY3, defenc, force_text, - force_bytes + force_bytes, + is_posix, + safe_encode, + safe_decode, ) S_IFGITLINK = S_IFLNK | S_IFDIR # a submodule @@ -67,22 +72,28 @@ def run_commit_hook(name, index): return env = os.environ.copy() - env['GIT_INDEX_FILE'] = index.path + env['GIT_INDEX_FILE'] = safe_decode(index.path) if PY3 else safe_encode(index.path) env['GIT_EDITOR'] = ':' - cmd = subprocess.Popen(hp, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=index.repo.working_dir, - close_fds=(os.name == 'posix')) - stdout, stderr = cmd.communicate() - cmd.stdout.close() - cmd.stderr.close() - - if cmd.returncode != 0: - stdout = force_text(stdout, defenc) - stderr = force_text(stderr, defenc) - raise HookExecutionError(hp, cmd.returncode, stdout, stderr) + try: + cmd = subprocess.Popen(hp, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=index.repo.working_dir, + close_fds=is_posix, + creationflags=PROC_CREATIONFLAGS,) + except Exception as ex: + raise HookExecutionError(hp, ex) + else: + stdout = [] + stderr = [] + handle_process_output(cmd, stdout.append, stderr.append, finalize_process) + stdout = ''.join(stdout) + stderr = ''.join(stderr) + if cmd.returncode != 0: + stdout = force_text(stdout, defenc) + stderr = force_text(stderr, defenc) + raise HookExecutionError(hp, cmd.returncode, stdout, stderr) # end handle return code diff --git a/git/index/util.py b/git/index/util.py index 171bd8fcf..0340500cc 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -2,6 +2,7 @@ import struct import tempfile import os +from git.compat import is_win __all__ = ('TemporaryFileSwap', 'post_clear_cache', 'default_index', 'git_working_dir') @@ -29,7 +30,7 @@ def __init__(self, file_path): def __del__(self): if os.path.isfile(self.tmp_file_path): - if os.name == 'nt' and os.path.exists(self.file_path): + if is_win and os.path.exists(self.file_path): os.remove(self.file_path) os.rename(self.tmp_file_path, self.file_path) # END temp file exists diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index eea091f8c..c6c6d6996 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -29,7 +29,8 @@ ) from git.compat import ( string_types, - defenc + defenc, + is_win, ) import stat @@ -289,14 +290,16 @@ def _write_git_file_and_module_config(cls, working_tree_dir, module_abspath): """ git_file = os.path.join(working_tree_dir, '.git') rela_path = os.path.relpath(module_abspath, start=working_tree_dir) - fp = open(git_file, 'wb') - fp.write(("gitdir: %s" % rela_path).encode(defenc)) - fp.close() + if is_win: + if os.path.isfile(git_file): + os.remove(git_file) + with open(git_file, 'wb') as fp: + fp.write(("gitdir: %s" % rela_path).encode(defenc)) - writer = GitConfigParser(os.path.join(module_abspath, 'config'), read_only=False, merge_includes=False) - writer.set_value('core', 'worktree', - to_native_path_linux(os.path.relpath(working_tree_dir, start=module_abspath))) - writer.release() + with GitConfigParser(os.path.join(module_abspath, 'config'), + read_only=False, merge_includes=False) as writer: + writer.set_value('core', 'worktree', + to_native_path_linux(os.path.relpath(working_tree_dir, start=module_abspath))) #{ Edit Interface @@ -848,13 +851,17 @@ def remove(self, module=True, force=False, configuration=True, dry_run=False): # finally delete our own submodule if not dry_run: + self._clear_cache() wtd = mod.working_tree_dir del(mod) # release file-handles (windows) + import gc + gc.collect() rmtree(wtd) # END delete tree if possible # END handle force if not dry_run and os.path.isdir(git_dir): + self._clear_cache() rmtree(git_dir) # end handle separate bare repository # END handle module deletion diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index ec2944c6e..894b26d53 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -134,9 +134,8 @@ def _get_ref_info(cls, repo, ref_path): point to, or None""" tokens = None try: - fp = open(join(repo.git_dir, ref_path), 'rt') - value = fp.read().rstrip() - fp.close() + with open(join(repo.git_dir, ref_path), 'rt') as fp: + value = fp.read().rstrip() # Don't only split on spaces, but on whitespace, which allows to parse lines like # 60b64ef992065e2600bfef6187a97f92398a9144 branch 'master' of git-server:/path/to/repo tokens = value.split() @@ -313,13 +312,17 @@ def set_reference(self, ref, logmsg=None): lfd = LockedFD(fpath) fd = lfd.open(write=True, stream=True) - fd.write(write_value.encode('ascii') + b'\n') - lfd.commit() - + ok = True + try: + fd.write(write_value.encode('ascii') + b'\n') + lfd.commit() + ok = True + finally: + if not ok: + lfd.rollback() # Adjust the reflog if logmsg is not None: self.log_append(oldbinsha, logmsg) - # END handle reflog return self @@ -422,40 +425,36 @@ def delete(cls, repo, path): # check packed refs pack_file_path = cls._get_packed_refs_path(repo) try: - reader = open(pack_file_path, 'rb') - except (OSError, IOError): - pass # it didnt exist at all - else: - new_lines = list() - made_change = False - dropped_last_line = False - for line in reader: - # keep line if it is a comment or if the ref to delete is not - # in the line - # If we deleted the last line and this one is a tag-reference object, - # we drop it as well - line = line.decode(defenc) - if (line.startswith('#') or full_ref_path not in line) and \ - (not dropped_last_line or dropped_last_line and not line.startswith('^')): - new_lines.append(line) - dropped_last_line = False - continue - # END skip comments and lines without our path - - # drop this line - made_change = True - dropped_last_line = True - # END for each line in packed refs - reader.close() + with open(pack_file_path, 'rb') as reader: + new_lines = list() + made_change = False + dropped_last_line = False + for line in reader: + # keep line if it is a comment or if the ref to delete is not + # in the line + # If we deleted the last line and this one is a tag-reference object, + # we drop it as well + line = line.decode(defenc) + if (line.startswith('#') or full_ref_path not in line) and \ + (not dropped_last_line or dropped_last_line and not line.startswith('^')): + new_lines.append(line) + dropped_last_line = False + continue + # END skip comments and lines without our path + + # drop this line + made_change = True + dropped_last_line = True # write the new lines if made_change: # write-binary is required, otherwise windows will # open the file in text mode and change LF to CRLF ! - open(pack_file_path, 'wb').writelines(l.encode(defenc) for l in new_lines) - # END write out file - # END open exception handling - # END handle deletion + with open(pack_file_path, 'wb') as fd: + fd.writelines(l.encode(defenc) for l in new_lines) + + except (OSError, IOError): + pass # it didnt exist at all # delete the reflog reflog_path = RefLog.path(cls(repo, full_ref_path)) @@ -484,7 +483,8 @@ def _create(cls, repo, path, resolve, reference, force, logmsg=None): target_data = target.path if not resolve: target_data = "ref: " + target_data - existing_data = open(abs_ref_path, 'rb').read().decode(defenc).strip() + with open(abs_ref_path, 'rb') as fd: + existing_data = fd.read().decode(defenc).strip() if existing_data != target_data: raise OSError("Reference at %r does already exist, pointing to %r, requested was %r" % (full_ref_path, existing_data, target_data)) @@ -549,7 +549,11 @@ def rename(self, new_path, force=False): if isfile(new_abs_path): if not force: # if they point to the same file, its not an error - if open(new_abs_path, 'rb').read().strip() != open(cur_abs_path, 'rb').read().strip(): + with open(new_abs_path, 'rb') as fd1: + f1 = fd1.read().strip() + with open(cur_abs_path, 'rb') as fd2: + f2 = fd2.read().strip() + if f1 != f2: raise OSError("File at path %r already exists" % new_abs_path) # else: we could remove ourselves and use the otherone, but # but clarity we just continue as usual diff --git a/git/remote.py b/git/remote.py index 4a8a5ee9e..c2ffcc1a6 100644 --- a/git/remote.py +++ b/git/remote.py @@ -6,7 +6,6 @@ # Module implementing a remote object allowing easy access to git remotes import re -import os from .config import ( SectionConstraint, @@ -32,7 +31,7 @@ ) from git.cmd import handle_process_output from gitdb.util import join -from git.compat import (defenc, force_text) +from git.compat import (defenc, force_text, is_win) import logging log = logging.getLogger('git.remote') @@ -113,7 +112,7 @@ def __init__(self, flags, local_ref, remote_ref_string, remote, old_commit=None, self._remote = remote self._old_commit_sha = old_commit self.summary = summary - + @property def old_commit(self): return self._old_commit_sha and self._remote.repo.commit(self._old_commit_sha) or None @@ -377,7 +376,7 @@ def __init__(self, repo, name): self.repo = repo self.name = name - if os.name == 'nt': + if is_win: # some oddity: on windows, python 2.5, it for some reason does not realize # that it has the config_writer property, but instead calls __getattr__ # which will not yield the expected results. 'pinging' the members @@ -635,13 +634,12 @@ def _get_fetch_info_from_stderr(self, proc, progress): # end if progress.error_lines(): stderr_text = '\n'.join(progress.error_lines()) - + finalize_process(proc, stderr=stderr_text) # read head information - fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') - fetch_head_info = [l.decode(defenc) for l in fp.readlines()] - fp.close() + with open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') as fp: + fetch_head_info = [l.decode(defenc) for l in fp.readlines()] l_fil = len(fetch_info_lines) l_fhi = len(fetch_head_info) @@ -657,7 +655,7 @@ def _get_fetch_info_from_stderr(self, proc, progress): fetch_info_lines = fetch_info_lines[:l_fhi] # end truncate correct list # end sanity check + sanitization - + output.extend(FetchInfo._from_line(self.repo, err_line, fetch_line) for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info)) return output @@ -682,7 +680,7 @@ def stdout_handler(line): # END for each line try: - handle_process_output(proc, stdout_handler, progress_handler, finalize_process) + handle_process_output(proc, stdout_handler, progress_handler, finalize_process, decode_streams=False) except Exception: if len(output) == 0: raise @@ -769,17 +767,17 @@ def push(self, refspec=None, progress=None, **kwargs): :param refspec: see 'fetch' method :param progress: Can take one of many value types: - + * None to discard progress information * A function (callable) that is called with the progress infomation. - + Signature: ``progress(op_code, cur_count, max_count=None, message='')``. - + `Click here `_ for a description of all arguments given to the function. * An instance of a class derived from ``git.RemoteProgress`` that overrides the ``update()`` function. - + :note: No further progress information is returned after push returns. :param kwargs: Additional arguments to be passed to git-push :return: diff --git a/git/repo/base.py b/git/repo/base.py index 0e46ee679..947d77d2e 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -56,6 +56,7 @@ PY3, safe_decode, range, + is_win, ) import os @@ -71,7 +72,7 @@ BlameEntry = namedtuple('BlameEntry', ['commit', 'linenos', 'orig_path', 'orig_linenos']) -__all__ = ('Repo', ) +__all__ = ('Repo',) def _expand_path(p): @@ -209,11 +210,13 @@ def __hash__(self): # Description property def _get_description(self): filename = join(self.git_dir, 'description') - return open(filename, 'rb').read().rstrip().decode(defenc) + with open(filename, 'rb') as fp: + return fp.read().rstrip().decode(defenc) def _set_description(self, descr): filename = join(self.git_dir, 'description') - open(filename, 'wb').write((descr + '\n').encode(defenc)) + with open(filename, 'wb') as fp: + fp.write((descr + '\n').encode(defenc)) description = property(_get_description, _set_description, doc="the project's description") @@ -369,7 +372,7 @@ def delete_remote(self, remote): def _get_config_path(self, config_level): # we do not support an absolute path of the gitconfig on windows , # use the global config instead - if sys.platform == "win32" and config_level == "system": + if is_win and config_level == "system": config_level = "global" if config_level == "system": @@ -547,11 +550,8 @@ def _get_alternates(self): alternates_path = join(self.git_dir, 'objects', 'info', 'alternates') if os.path.exists(alternates_path): - try: - f = open(alternates_path, 'rb') + with open(alternates_path, 'rb') as f: alts = f.read().decode(defenc) - finally: - f.close() return alts.strip().splitlines() else: return list() @@ -572,13 +572,8 @@ def _set_alternates(self, alts): if isfile(alternates_path): os.remove(alternates_path) else: - try: - f = open(alternates_path, 'wb') + with open(alternates_path, 'wb') as f: f.write("\n".join(alts).encode(defenc)) - finally: - f.close() - # END file handling - # END alts handling alternates = property(_get_alternates, _set_alternates, doc="Retrieve a list of alternates paths or set a list paths to be used as alternates") @@ -883,7 +878,7 @@ def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): prev_cwd = None prev_path = None odbt = kwargs.pop('odbt', odb_default_type) - if os.name == 'nt': + if is_win: if '~' in path: raise OSError("Git cannot handle the ~ character in path %r correctly" % path) @@ -904,12 +899,8 @@ def _clone(cls, git, url, path, odb_default_type, progress, **kwargs): try: proc = git.clone(url, path, with_extended_output=True, as_process=True, v=True, **add_progress(kwargs, git, progress)) - if progress: - handle_process_output(proc, None, progress.new_message_handler(), finalize_process) - else: - (stdout, stderr) = proc.communicate() - finalize_process(proc, stderr=stderr) - # end handle progress + progress_handler = progress and progress.new_message_handler() or None + handle_process_output(proc, None, progress_handler, finalize_process) finally: if prev_cwd is not None: os.chdir(prev_cwd) diff --git a/git/repo/fun.py b/git/repo/fun.py index 6b06663a0..0483eaa99 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -25,8 +25,8 @@ def touch(filename): - fp = open(filename, "ab") - fp.close() + with open(filename, "ab"): + pass return filename diff --git a/git/test/fixtures/cat_file.py b/git/test/fixtures/cat_file.py index 2f1b915aa..5480e6282 100644 --- a/git/test/fixtures/cat_file.py +++ b/git/test/fixtures/cat_file.py @@ -1,5 +1,6 @@ import sys -for line in open(sys.argv[1]).readlines(): - sys.stdout.write(line) - sys.stderr.write(line) +with open(sys.argv[1]) as fd: + for line in fd.readlines(): + sys.stdout.write(line) + sys.stderr.write(line) diff --git a/git/test/lib/asserts.py b/git/test/lib/asserts.py index 60a888b3b..9edc49e08 100644 --- a/git/test/lib/asserts.py +++ b/git/test/lib/asserts.py @@ -16,7 +16,10 @@ assert_false ) -from mock import patch +try: + from unittest.mock import patch +except ImportError: + from mock import patch __all__ = ['assert_instance_of', 'assert_not_instance_of', 'assert_none', 'assert_not_none', diff --git a/git/test/lib/helper.py b/git/test/lib/helper.py index 8be2881c3..a85ac2fd6 100644 --- a/git/test/lib/helper.py +++ b/git/test/lib/helper.py @@ -4,16 +4,18 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php from __future__ import print_function + import os -import sys from unittest import TestCase import time import tempfile -import shutil import io +import logging from git import Repo, Remote, GitCommandError, Git -from git.compat import string_types +from git.util import rmtree +from git.compat import string_types, is_win +import textwrap osp = os.path.dirname @@ -22,9 +24,12 @@ __all__ = ( 'fixture_path', 'fixture', 'absolute_project_path', 'StringProcessAdapter', - 'with_rw_repo', 'with_rw_and_rw_remote_repo', 'TestBase', 'TestCase', 'GIT_REPO', 'GIT_DAEMON_PORT' + 'with_rw_directory', 'with_rw_repo', 'with_rw_and_rw_remote_repo', 'TestBase', 'TestCase', + 'GIT_REPO', 'GIT_DAEMON_PORT' ) +log = logging.getLogger('git.util') + #{ Routines @@ -34,7 +39,8 @@ def fixture_path(name): def fixture(name): - return open(fixture_path(name), 'rb').read() + with open(fixture_path(name), 'rb') as fd: + return fd.read() def absolute_project_path(): @@ -71,21 +77,36 @@ def _mktemp(*args): prefixing /private/ will lead to incorrect paths on OSX.""" tdir = tempfile.mktemp(*args) # See :note: above to learn why this is comented out. - # if sys.platform == 'darwin': + # if is_darwin: # tdir = '/private' + tdir return tdir -def _rmtree_onerror(osremove, fullpath, exec_info): - """ - Handle the case on windows that read-only files cannot be deleted by - os.remove by setting it to mode 777, then retry deletion. - """ - if os.name != 'nt' or osremove is not os.remove: - raise +def with_rw_directory(func): + """Create a temporary directory which can be written to, remove it if the + test succeeds, but leave it otherwise to aid additional debugging""" - os.chmod(fullpath, 0o777) - os.remove(fullpath) + def wrapper(self): + path = tempfile.mktemp(prefix=func.__name__) + os.mkdir(path) + keep = False + try: + try: + return func(self, path) + except Exception: + log.info.write("Test %s.%s failed, output is at %r\n", + type(self).__name__, func.__name__, path) + keep = True + raise + finally: + # Need to collect here to be sure all handles have been closed. It appears + # a windows-only issue. In fact things should be deleted, as well as + # memory maps closed, once objects go out of scope. For some reason + # though this is not the case here unless we collect explicitly. + import gc + gc.collect() + if not keep: + rmtree(path) def with_rw_repo(working_tree_ref, bare=False): @@ -120,14 +141,17 @@ def repo_creator(self): try: return func(self, rw_repo) except: - print("Keeping repo after failure: %s" % repo_dir, file=sys.stderr) + log.info("Keeping repo after failure: %s", repo_dir) repo_dir = None raise finally: os.chdir(prev_cwd) rw_repo.git.clear_cache() + rw_repo = None + import gc + gc.collect() if repo_dir is not None: - shutil.rmtree(repo_dir, onerror=_rmtree_onerror) + rmtree(repo_dir) # END rm test repo if possible # END cleanup # END rw repo creator @@ -137,6 +161,28 @@ def repo_creator(self): return argument_passer +def launch_git_daemon(temp_dir, ip, port): + if is_win: + ## On MINGW-git, daemon exists in .\Git\mingw64\libexec\git-core\, + # but if invoked as 'git daemon', it detaches from parent `git` cmd, + # and then CANNOT DIE! + # So, invoke it as a single command. + ## Cygwin-git has no daemon. + # + daemon_cmd = ['git-daemon', temp_dir, + '--enable=receive-pack', + '--listen=%s' % ip, + '--port=%s' % port] + gd = Git().execute(daemon_cmd, as_process=True) + else: + gd = Git().daemon(temp_dir, + enable='receive-pack', + listen=ip, + port=port, + as_process=True) + return gd + + def with_rw_and_rw_remote_repo(working_tree_ref): """ Same as with_rw_repo, but also provides a writable remote repository from which the @@ -164,6 +210,7 @@ def case(self, rw_repo, rw_remote_repo) assert isinstance(working_tree_ref, string_types), "Decorator requires ref name for working tree checkout" def argument_passer(func): + def remote_repo_creator(self): remote_repo_dir = _mktemp("remote_repo_%s" % func.__name__) repo_dir = _mktemp("remote_clone_non_bare_repo") @@ -199,62 +246,74 @@ def remote_repo_creator(self): d_remote.config_writer.set('url', remote_repo_url) temp_dir = osp(_mktemp()) - # On windows, this will fail ... we deal with failures anyway and default to telling the user to do it + gd = launch_git_daemon(temp_dir, '127.0.0.1', GIT_DAEMON_PORT) try: - gd = Git().daemon(temp_dir, enable='receive-pack', listen='127.0.0.1', port=GIT_DAEMON_PORT, - as_process=True) # yes, I know ... fortunately, this is always going to work if sleep time is just large enough time.sleep(0.5) - except Exception: - gd = None # end - # try to list remotes to diagnoes whether the server is up - try: - rw_repo.git.ls_remote(d_remote) - except GitCommandError as e: - # We assume in good faith that we didn't start the daemon - but make sure we kill it anyway - # Of course we expect it to work here already, but maybe there are timing constraints - # on some platforms ? - if gd is not None: - os.kill(gd.proc.pid, 15) - print(str(e)) - if os.name == 'nt': - msg = "git-daemon needs to run this test, but windows does not have one. " - msg += 'Otherwise, run: git-daemon "%s"' % temp_dir - raise AssertionError(msg) - else: - msg = 'Please start a git-daemon to run this test, execute: git daemon --enable=receive-pack "%s"' - msg += 'You can also run the daemon on a different port by passing --port=' - msg += 'and setting the environment variable GIT_PYTHON_TEST_GIT_DAEMON_PORT to ' - msg %= temp_dir - raise AssertionError(msg) - # END make assertion - # END catch ls remote error - - # adjust working dir - prev_cwd = os.getcwd() - os.chdir(rw_repo.working_dir) - try: + # try to list remotes to diagnoes whether the server is up + try: + rw_repo.git.ls_remote(d_remote) + except GitCommandError as e: + # We assume in good faith that we didn't start the daemon - but make sure we kill it anyway + # Of course we expect it to work here already, but maybe there are timing constraints + # on some platforms ? + try: + gd.proc.terminate() + except Exception as ex: + log.debug("Ignoring %r while terminating proc after %r.", ex, e) + log.warning('git(%s) ls-remote failed due to:%s', + rw_repo.git_dir, e) + if is_win: + msg = textwrap.dedent(""" + MINGW yet has problems with paths, and `git-daemon.exe` must be in PATH + (look into .\Git\mingw64\libexec\git-core\); + CYGWIN has no daemon, but if one exists, it gets along fine (has also paths problems) + Anyhow, alternatively try starting `git-daemon` manually:""") + else: + msg = "Please try starting `git-daemon` manually:" + + msg += textwrap.dedent(""" + git daemon --enable=receive-pack '%s' + You can also run the daemon on a different port by passing --port=" + and setting the environment variable GIT_PYTHON_TEST_GIT_DAEMON_PORT to + """ % temp_dir) + from nose import SkipTest + raise SkipTest(msg) if is_win else AssertionError(msg) + # END make assertion + # END catch ls remote error + + # adjust working dir + prev_cwd = os.getcwd() + os.chdir(rw_repo.working_dir) + try: return func(self, rw_repo, rw_remote_repo) except: - print("Keeping repos after failure: repo_dir = %s, remote_repo_dir = %s" - % (repo_dir, remote_repo_dir), file=sys.stderr) + log.info("Keeping repos after failure: repo_dir = %s, remote_repo_dir = %s", + repo_dir, remote_repo_dir) repo_dir = remote_repo_dir = None raise + finally: + os.chdir(prev_cwd) + finally: - # gd.proc.kill() ... no idea why that doesn't work - if gd is not None: - os.kill(gd.proc.pid, 15) + try: + gd.proc.kill() + except: + ## Either it has died (and we're here), or it won't die, again here... + pass - os.chdir(prev_cwd) rw_repo.git.clear_cache() rw_remote_repo.git.clear_cache() + rw_repo = rw_remote_repo = None + import gc + gc.collect() if repo_dir: - shutil.rmtree(repo_dir, onerror=_rmtree_onerror) + rmtree(repo_dir) if remote_repo_dir: - shutil.rmtree(remote_repo_dir, onerror=_rmtree_onerror) + rmtree(remote_repo_dir) if gd is not None: gd.proc.wait() @@ -299,6 +358,8 @@ def setUpClass(cls): Dynamically add a read-only repository to our actual type. This way each test type has its own repository """ + import gc + gc.collect() cls.rorepo = Repo(GIT_REPO) @classmethod @@ -313,7 +374,6 @@ def _make_file(self, rela_path, data, repo=None): """ repo = repo or self.rorepo abs_path = os.path.join(repo.working_tree_dir, rela_path) - fp = open(abs_path, "w") - fp.write(data) - fp.close() + with open(abs_path, "w") as fp: + fp.write(data) return abs_path diff --git a/git/test/performance/lib.py b/git/test/performance/lib.py index bb3f7a998..eebbfd76a 100644 --- a/git/test/performance/lib.py +++ b/git/test/performance/lib.py @@ -4,7 +4,6 @@ TestBase ) from gitdb.test.lib import skip_on_travis_ci -import shutil import tempfile import logging @@ -16,6 +15,7 @@ from git import ( Repo ) +from git.util import rmtree #{ Invvariants k_env_git_repo = "GIT_PYTHON_TEST_GIT_REPO_BASE" @@ -86,7 +86,7 @@ def setUp(self): def tearDown(self): super(TestBigRepoRW, self).tearDown() if self.gitrwrepo is not None: - shutil.rmtree(self.gitrwrepo.working_dir) + rmtree(self.gitrwrepo.working_dir) self.gitrwrepo.git.clear_cache() self.gitrwrepo = None self.puregitrwrepo.git.clear_cache() diff --git a/git/test/performance/test_commit.py b/git/test/performance/test_commit.py index b59c747ee..c60dc2fc4 100644 --- a/git/test/performance/test_commit.py +++ b/git/test/performance/test_commit.py @@ -17,6 +17,10 @@ class TestPerformance(TestBigRepoRW): + def tearDown(self): + import gc + gc.collect() + # ref with about 100 commits in its history ref_100 = '0.1.6' diff --git a/git/test/performance/test_streams.py b/git/test/performance/test_streams.py index 4b1738cdf..8194547cb 100644 --- a/git/test/performance/test_streams.py +++ b/git/test/performance/test_streams.py @@ -87,6 +87,9 @@ def test_large_data_streaming(self, rwrepo): % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) # del db file so git has something to do + ostream = None + import gc + gc.collect() os.remove(db_file) # VS. CGIT diff --git a/git/test/test_base.py b/git/test/test_base.py index 7b71a77ee..e5e8f173b 100644 --- a/git/test/test_base.py +++ b/git/test/test_base.py @@ -7,6 +7,7 @@ import os import sys import tempfile +from unittest import skipIf import git.objects.base as base from git.test.lib import ( @@ -23,10 +24,15 @@ ) from git.objects.util import get_object_type_by_name from gitdb.util import hex_to_bin +from git.compat import is_win class TestBase(TestBase): + def tearDown(self): + import gc + gc.collect() + type_tuples = (("blob", "8741fc1d09d61f02ffd8cded15ff603eff1ec070", "blob.py"), ("tree", "3a6a5e3eeed3723c09f1ef0399f81ed6b8d82e79", "directory"), ("commit", "4251bd59fb8e11e40c40548cba38180a9536118c", None), @@ -71,13 +77,11 @@ def test_base_object(self): assert data tmpfilename = tempfile.mktemp(suffix='test-stream') - tmpfile = open(tmpfilename, 'wb+') - assert item == item.stream_data(tmpfile) - tmpfile.seek(0) - assert tmpfile.read() == data - tmpfile.close() + with open(tmpfilename, 'wb+') as tmpfile: + assert item == item.stream_data(tmpfile) + tmpfile.seek(0) + assert tmpfile.read() == data os.remove(tmpfilename) - # END stream to file directly # END for each object type to create # each has a unique sha @@ -112,6 +116,8 @@ def test_with_rw_remote_and_rw_repo(self, rw_repo, rw_remote_repo): assert rw_remote_repo.config_reader("repository").getboolean("core", "bare") assert os.path.isdir(os.path.join(rw_repo.working_tree_dir, 'lib')) + @skipIf(sys.version_info < (3,) and is_win, + "Unicode woes, see https://github.com/gitpython-developers/GitPython/pull/519") @with_rw_repo('0.1.6') def test_add_unicode(self, rw_repo): filename = u"שלום.txt" @@ -125,9 +131,10 @@ def test_add_unicode(self, rw_repo): from nose import SkipTest raise SkipTest("Environment doesn't support unicode filenames") - open(file_path, "wb").write(b'something') + with open(file_path, "wb") as fp: + fp.write(b'something') - if os.name == 'nt': + if is_win: # on windows, there is no way this works, see images on # https://github.com/gitpython-developers/GitPython/issues/147#issuecomment-68881897 # Therefore, it must be added using the python implementation diff --git a/git/test/test_commit.py b/git/test/test_commit.py index c05995033..66d988a3a 100644 --- a/git/test/test_commit.py +++ b/git/test/test_commit.py @@ -19,7 +19,7 @@ Actor, ) from gitdb import IStream -from gitdb.test.lib import with_rw_directory +from git.test.lib import with_rw_directory from git.compat import ( string_types, text_type @@ -34,7 +34,11 @@ import os from datetime import datetime from git.objects.util import tzoffset, utc -from mock import Mock + +try: + from unittest.mock import Mock +except ImportError: + from mock import Mock def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False): @@ -57,14 +61,14 @@ def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False) stream.seek(0) istream = rwrepo.odb.store(IStream(Commit.type, streamlen, stream)) - assert istream.hexsha == cm.hexsha.encode('ascii') + assert_equal(istream.hexsha, cm.hexsha.encode('ascii')) nc = Commit(rwrepo, Commit.NULL_BIN_SHA, cm.tree, cm.author, cm.authored_date, cm.author_tz_offset, cm.committer, cm.committed_date, cm.committer_tz_offset, cm.message, cm.parents, cm.encoding) - assert nc.parents == cm.parents + assert_equal(nc.parents, cm.parents) stream = BytesIO() nc._serialize(stream) ns += 1 @@ -78,7 +82,7 @@ def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False) nc.binsha = rwrepo.odb.store(istream).binsha # if it worked, we have exactly the same contents ! - assert nc.hexsha == cm.hexsha + assert_equal(nc.hexsha, cm.hexsha) # END check commits elapsed = time.time() - st @@ -99,10 +103,10 @@ def test_bake(self): assert_equal("Sebastian Thiel", commit.author.name) assert_equal("byronimo@gmail.com", commit.author.email) - assert commit.author == commit.committer + self.assertEqual(commit.author, commit.committer) assert isinstance(commit.authored_date, int) and isinstance(commit.committed_date, int) assert isinstance(commit.author_tz_offset, int) and isinstance(commit.committer_tz_offset, int) - assert commit.message == "Added missing information to docstrings of commit and stats module\n" + self.assertEqual(commit.message, "Added missing information to docstrings of commit and stats module\n") def test_stats(self): commit = self.rorepo.commit('33ebe7acec14b25c5f84f35a664803fcab2f7781') @@ -125,20 +129,20 @@ def check_entries(d): # assure data is parsed properly michael = Actor._from_string("Michael Trier ") - assert commit.author == michael - assert commit.committer == michael - assert commit.authored_date == 1210193388 - assert commit.committed_date == 1210193388 - assert commit.author_tz_offset == 14400, commit.author_tz_offset - assert commit.committer_tz_offset == 14400, commit.committer_tz_offset - assert commit.message == "initial project\n" + self.assertEqual(commit.author, michael) + self.assertEqual(commit.committer, michael) + self.assertEqual(commit.authored_date, 1210193388) + self.assertEqual(commit.committed_date, 1210193388) + self.assertEqual(commit.author_tz_offset, 14400, commit.author_tz_offset) + self.assertEqual(commit.committer_tz_offset, 14400, commit.committer_tz_offset) + self.assertEqual(commit.message, "initial project\n") def test_unicode_actor(self): # assure we can parse unicode actors correctly name = u"Üäöß ÄußÉ" - assert len(name) == 9 + self.assertEqual(len(name), 9) special = Actor._from_string(u"%s " % name) - assert special.name == name + self.assertEqual(special.name, name) assert isinstance(special.name, text_type) def test_traversal(self): @@ -152,44 +156,44 @@ def test_traversal(self): # basic branch first, depth first dfirst = start.traverse(branch_first=False) bfirst = start.traverse(branch_first=True) - assert next(dfirst) == p0 - assert next(dfirst) == p00 + self.assertEqual(next(dfirst), p0) + self.assertEqual(next(dfirst), p00) - assert next(bfirst) == p0 - assert next(bfirst) == p1 - assert next(bfirst) == p00 - assert next(bfirst) == p10 + self.assertEqual(next(bfirst), p0) + self.assertEqual(next(bfirst), p1) + self.assertEqual(next(bfirst), p00) + self.assertEqual(next(bfirst), p10) # at some point, both iterations should stop - assert list(bfirst)[-1] == first + self.assertEqual(list(bfirst)[-1], first) stoptraverse = self.rorepo.commit("254d04aa3180eb8b8daf7b7ff25f010cd69b4e7d").traverse(as_edge=True) l = list(stoptraverse) - assert len(l[0]) == 2 + self.assertEqual(len(l[0]), 2) # ignore self - assert next(start.traverse(ignore_self=False)) == start + self.assertEqual(next(start.traverse(ignore_self=False)), start) # depth - assert len(list(start.traverse(ignore_self=False, depth=0))) == 1 + self.assertEqual(len(list(start.traverse(ignore_self=False, depth=0))), 1) # prune - assert next(start.traverse(branch_first=1, prune=lambda i, d: i == p0)) == p1 + self.assertEqual(next(start.traverse(branch_first=1, prune=lambda i, d: i == p0)), p1) # predicate - assert next(start.traverse(branch_first=1, predicate=lambda i, d: i == p1)) == p1 + self.assertEqual(next(start.traverse(branch_first=1, predicate=lambda i, d: i == p1)), p1) # traversal should stop when the beginning is reached self.failUnlessRaises(StopIteration, next, first.traverse()) # parents of the first commit should be empty ( as the only parent has a null # sha ) - assert len(first.parents) == 0 + self.assertEqual(len(first.parents), 0) def test_iteration(self): # we can iterate commits all_commits = Commit.list_items(self.rorepo, self.rorepo.head) assert all_commits - assert all_commits == list(self.rorepo.iter_commits()) + self.assertEqual(all_commits, list(self.rorepo.iter_commits())) # this includes merge commits mcomit = self.rorepo.commit('d884adc80c80300b4cc05321494713904ef1df2d') @@ -236,7 +240,7 @@ def test_ambiguous_arg_iteration(self, rw_dir): list(rw_repo.iter_commits(rw_repo.head.ref)) # should fail unless bug is fixed def test_count(self): - assert self.rorepo.tag('refs/tags/0.1.5').commit.count() == 143 + self.assertEqual(self.rorepo.tag('refs/tags/0.1.5').commit.count(), 143) def test_list(self): # This doesn't work anymore, as we will either attempt getattr with bytes, or compare 20 byte string @@ -266,7 +270,7 @@ def test_iter_parents(self): piter = c.iter_parents(skip=skip) first_parent = next(piter) assert first_parent != c - assert first_parent == c.parents[0] + self.assertEqual(first_parent, c.parents[0]) # END for each def test_name_rev(self): @@ -279,7 +283,7 @@ def test_serialization(self, rwrepo): assert_commit_serialization(rwrepo, '0.1.6') def test_serialization_unicode_support(self): - assert Commit.default_encoding.lower() == 'utf-8' + self.assertEqual(Commit.default_encoding.lower(), 'utf-8') # create a commit with unicode in the message, and the author's name # Verify its serialization and deserialization @@ -288,10 +292,10 @@ def test_serialization_unicode_support(self): assert isinstance(cmt.author.name, text_type) # same here cmt.message = u"üäêèß" - assert len(cmt.message) == 5 + self.assertEqual(len(cmt.message), 5) cmt.author.name = u"äüß" - assert len(cmt.author.name) == 3 + self.assertEqual(len(cmt.author.name), 3) cstream = BytesIO() cmt._serialize(cstream) @@ -301,22 +305,24 @@ def test_serialization_unicode_support(self): ncmt = Commit(self.rorepo, cmt.binsha) ncmt._deserialize(cstream) - assert cmt.author.name == ncmt.author.name - assert cmt.message == ncmt.message + self.assertEqual(cmt.author.name, ncmt.author.name) + self.assertEqual(cmt.message, ncmt.message) # actually, it can't be printed in a shell as repr wants to have ascii only # it appears cmt.author.__repr__() def test_invalid_commit(self): cmt = self.rorepo.commit() - cmt._deserialize(open(fixture_path('commit_invalid_data'), 'rb')) + with open(fixture_path('commit_invalid_data'), 'rb') as fd: + cmt._deserialize(fd) - assert cmt.author.name == u'E.Azer Ko�o�o�oculu', cmt.author.name - assert cmt.author.email == 'azer@kodfabrik.com', cmt.author.email + self.assertEqual(cmt.author.name, u'E.Azer Ko�o�o�oculu', cmt.author.name) + self.assertEqual(cmt.author.email, 'azer@kodfabrik.com', cmt.author.email) def test_gpgsig(self): cmt = self.rorepo.commit() - cmt._deserialize(open(fixture_path('commit_with_gpgsig'), 'rb')) + with open(fixture_path('commit_with_gpgsig'), 'rb') as fd: + cmt._deserialize(fd) fixture_sig = """-----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.11 (GNU/Linux) @@ -335,7 +341,7 @@ def test_gpgsig(self): JzJMZDRLQLFvnzqZuCjE =przd -----END PGP SIGNATURE-----""" - assert cmt.gpgsig == fixture_sig + self.assertEqual(cmt.gpgsig, fixture_sig) cmt.gpgsig = "" assert cmt.gpgsig != fixture_sig @@ -343,39 +349,39 @@ def test_gpgsig(self): cstream = BytesIO() cmt._serialize(cstream) assert re.search(r"^gpgsig $", cstream.getvalue().decode('ascii'), re.MULTILINE) - + self.assert_gpgsig_deserialization(cstream) - + cstream.seek(0) cmt.gpgsig = None cmt._deserialize(cstream) - assert cmt.gpgsig == "" + self.assertEqual(cmt.gpgsig, "") cmt.gpgsig = None cstream = BytesIO() cmt._serialize(cstream) assert not re.search(r"^gpgsig ", cstream.getvalue().decode('ascii'), re.MULTILINE) - + def assert_gpgsig_deserialization(self, cstream): assert 'gpgsig' in 'precondition: need gpgsig' - + class RepoMock: def __init__(self, bytestr): self.bytestr = bytestr - + @property def odb(self): class ODBMock: def __init__(self, bytestr): self.bytestr = bytestr - + def stream(self, *args): stream = Mock(spec_set=['read'], return_value=self.bytestr) stream.read.return_value = self.bytestr return ('binsha', 'typename', 'size', stream) - + return ODBMock(self.bytestr) - + repo_mock = RepoMock(cstream.getvalue()) for field in Commit.__slots__: c = Commit(repo_mock, b'x' * 20) @@ -383,9 +389,13 @@ def stream(self, *args): def test_datetimes(self): commit = self.rorepo.commit('4251bd5') - assert commit.authored_date == 1255018625 - assert commit.committed_date == 1255026171 - assert commit.authored_datetime == datetime(2009, 10, 8, 18, 17, 5, tzinfo=tzoffset(-7200)), commit.authored_datetime # noqa - assert commit.authored_datetime == datetime(2009, 10, 8, 16, 17, 5, tzinfo=utc), commit.authored_datetime - assert commit.committed_datetime == datetime(2009, 10, 8, 20, 22, 51, tzinfo=tzoffset(-7200)) - assert commit.committed_datetime == datetime(2009, 10, 8, 18, 22, 51, tzinfo=utc), commit.committed_datetime + self.assertEqual(commit.authored_date, 1255018625) + self.assertEqual(commit.committed_date, 1255026171) + self.assertEqual(commit.authored_datetime, + datetime(2009, 10, 8, 18, 17, 5, tzinfo=tzoffset(-7200)), commit.authored_datetime) # noqa + self.assertEqual(commit.authored_datetime, + datetime(2009, 10, 8, 16, 17, 5, tzinfo=utc), commit.authored_datetime) + self.assertEqual(commit.committed_datetime, + datetime(2009, 10, 8, 20, 22, 51, tzinfo=tzoffset(-7200))) + self.assertEqual(commit.committed_datetime, + datetime(2009, 10, 8, 18, 22, 51, tzinfo=utc), commit.committed_datetime) diff --git a/git/test/test_config.py b/git/test/test_config.py index c0889c1a7..154aaa240 100644 --- a/git/test/test_config.py +++ b/git/test/test_config.py @@ -4,28 +4,45 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -from git.test.lib import ( - TestCase, - fixture_path, - assert_equal, -) -from gitdb.test.lib import with_rw_directory +import glob +import io +import os + from git import ( GitConfigParser ) -from git.compat import ( - string_types, -) -import io -import os +from git.compat import string_types from git.config import cp +from git.test.lib import ( + TestCase, + fixture_path, +) +from git.test.lib import with_rw_directory + +import os.path as osp +from git.util import rmfile + + +_tc_lock_fpaths = osp.join(osp.dirname(__file__), 'fixtures/*.lock') + + +def _rm_lock_files(): + for lfp in glob.glob(_tc_lock_fpaths): + rmfile(lfp) class TestBase(TestCase): + def setUp(self): + _rm_lock_files() + + def tearDown(self): + for lfp in glob.glob(_tc_lock_fpaths): + if osp.isfile(lfp): + raise AssertionError('Previous TC left hanging git-lock file: %s', lfp) def _to_memcache(self, file_path): - fp = open(file_path, "rb") - sio = io.BytesIO(fp.read()) + with open(file_path, "rb") as fp: + sio = io.BytesIO(fp.read()) sio.name = file_path return sio @@ -33,51 +50,49 @@ def test_read_write(self): # writer must create the exact same file as the one read before for filename in ("git_config", "git_config_global"): file_obj = self._to_memcache(fixture_path(filename)) - w_config = GitConfigParser(file_obj, read_only=False) - w_config.read() # enforce reading - assert w_config._sections - w_config.write() # enforce writing - - # we stripped lines when reading, so the results differ - assert file_obj.getvalue() - self.assertEqual(file_obj.getvalue(), self._to_memcache(fixture_path(filename)).getvalue()) - - # creating an additional config writer must fail due to exclusive access - self.failUnlessRaises(IOError, GitConfigParser, file_obj, read_only=False) - - # should still have a lock and be able to make changes - assert w_config._lock._has_lock() - - # changes should be written right away - sname = "my_section" - oname = "mykey" - val = "myvalue" - w_config.add_section(sname) - assert w_config.has_section(sname) - w_config.set(sname, oname, val) - assert w_config.has_option(sname, oname) - assert w_config.get(sname, oname) == val - - sname_new = "new_section" - oname_new = "new_key" - ival = 10 - w_config.set_value(sname_new, oname_new, ival) - assert w_config.get_value(sname_new, oname_new) == ival - - file_obj.seek(0) - r_config = GitConfigParser(file_obj, read_only=True) - assert r_config.has_section(sname) - assert r_config.has_option(sname, oname) - assert r_config.get(sname, oname) == val - w_config.release() + with GitConfigParser(file_obj, read_only=False) as w_config: + w_config.read() # enforce reading + assert w_config._sections + w_config.write() # enforce writing + + # we stripped lines when reading, so the results differ + assert file_obj.getvalue() + self.assertEqual(file_obj.getvalue(), self._to_memcache(fixture_path(filename)).getvalue()) + + # creating an additional config writer must fail due to exclusive access + self.failUnlessRaises(IOError, GitConfigParser, file_obj, read_only=False) + + # should still have a lock and be able to make changes + assert w_config._lock._has_lock() + + # changes should be written right away + sname = "my_section" + oname = "mykey" + val = "myvalue" + w_config.add_section(sname) + assert w_config.has_section(sname) + w_config.set(sname, oname, val) + assert w_config.has_option(sname, oname) + assert w_config.get(sname, oname) == val + + sname_new = "new_section" + oname_new = "new_key" + ival = 10 + w_config.set_value(sname_new, oname_new, ival) + assert w_config.get_value(sname_new, oname_new) == ival + + file_obj.seek(0) + r_config = GitConfigParser(file_obj, read_only=True) + assert r_config.has_section(sname) + assert r_config.has_option(sname, oname) + assert r_config.get(sname, oname) == val # END for each filename @with_rw_directory def test_lock_reentry(self, rw_dir): fpl = os.path.join(rw_dir, 'l') - gcp = GitConfigParser(fpl, read_only=False) - with gcp as cw: - cw.set_value('include', 'some_value', 'a') + with GitConfigParser(fpl, read_only=False) as gcp: + gcp.set_value('include', 'some_value', 'a') # entering again locks the file again... with gcp as cw: cw.set_value('include', 'some_other_value', 'b') @@ -91,21 +106,21 @@ def test_lock_reentry(self, rw_dir): def test_multi_line_config(self): file_obj = self._to_memcache(fixture_path("git_config_with_comments")) - config = GitConfigParser(file_obj, read_only=False) - ev = "ruby -e '\n" - ev += " system %(git), %(merge-file), %(--marker-size=%L), %(%A), %(%O), %(%B)\n" - ev += " b = File.read(%(%A))\n" - ev += " b.sub!(/^<+ .*\\nActiveRecord::Schema\\.define.:version => (\\d+). do\\n=+\\nActiveRecord::Schema\\." - ev += "define.:version => (\\d+). do\\n>+ .*/) do\n" - ev += " %(ActiveRecord::Schema.define(:version => #{[$1, $2].max}) do)\n" - ev += " end\n" - ev += " File.open(%(%A), %(w)) {|f| f.write(b)}\n" - ev += " exit 1 if b.include?(%(<)*%L)'" - assert_equal(config.get('merge "railsschema"', 'driver'), ev) - assert_equal(config.get('alias', 'lg'), - "log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr)%Creset'" - " --abbrev-commit --date=relative") - assert len(config.sections()) == 23 + with GitConfigParser(file_obj, read_only=False) as config: + ev = "ruby -e '\n" + ev += " system %(git), %(merge-file), %(--marker-size=%L), %(%A), %(%O), %(%B)\n" + ev += " b = File.read(%(%A))\n" + ev += " b.sub!(/^<+ .*\\nActiveRecord::Schema\\.define.:version => (\\d+). do\\n=+\\nActiveRecord::Schema\\." # noqa E501 + ev += "define.:version => (\\d+). do\\n>+ .*/) do\n" + ev += " %(ActiveRecord::Schema.define(:version => #{[$1, $2].max}) do)\n" + ev += " end\n" + ev += " File.open(%(%A), %(w)) {|f| f.write(b)}\n" + ev += " exit 1 if b.include?(%(<)*%L)'" + self.assertEqual(config.get('merge "railsschema"', 'driver'), ev) + self.assertEqual(config.get('alias', 'lg'), + "log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr)%Creset'" + " --abbrev-commit --date=relative") + self.assertEqual(len(config.sections()), 23) def test_base(self): path_repo = fixture_path("git_config") @@ -202,22 +217,19 @@ def check_test_value(cr, value): def test_rename(self): file_obj = self._to_memcache(fixture_path('git_config')) - cw = GitConfigParser(file_obj, read_only=False, merge_includes=False) - - self.failUnlessRaises(ValueError, cw.rename_section, "doesntexist", "foo") - self.failUnlessRaises(ValueError, cw.rename_section, "core", "include") + with GitConfigParser(file_obj, read_only=False, merge_includes=False) as cw: + self.failUnlessRaises(ValueError, cw.rename_section, "doesntexist", "foo") + self.failUnlessRaises(ValueError, cw.rename_section, "core", "include") - nn = "bee" - assert cw.rename_section('core', nn) is cw - assert not cw.has_section('core') - assert len(cw.items(nn)) == 4 - cw.release() + nn = "bee" + assert cw.rename_section('core', nn) is cw + assert not cw.has_section('core') + assert len(cw.items(nn)) == 4 def test_complex_aliases(self): file_obj = self._to_memcache(fixture_path('.gitconfig')) - w_config = GitConfigParser(file_obj, read_only=False) - self.assertEqual(w_config.get('alias', 'rbi'), '"!g() { git rebase -i origin/${1:-master} ; } ; g"') - w_config.release() + with GitConfigParser(file_obj, read_only=False) as w_config: + self.assertEqual(w_config.get('alias', 'rbi'), '"!g() { git rebase -i origin/${1:-master} ; } ; g"') self.assertEqual(file_obj.getvalue(), self._to_memcache(fixture_path('.gitconfig')).getvalue()) def test_empty_config_value(self): diff --git a/git/test/test_diff.py b/git/test/test_diff.py index 9fdb26a2c..d34d84e39 100644 --- a/git/test/test_diff.py +++ b/git/test/test_diff.py @@ -15,7 +15,7 @@ ) -from gitdb.test.lib import with_rw_directory +from git.test.lib import with_rw_directory from git import ( Repo, @@ -24,10 +24,16 @@ DiffIndex, NULL_TREE, ) +import ddt +@ddt.ddt class TestDiff(TestBase): + def tearDown(self): + import gc + gc.collect() + def _assert_diff_format(self, diffs): # verify that the format of the diff is sane for diff in diffs: @@ -66,13 +72,14 @@ def test_diff_with_staged_file(self, rw_dir): self.failUnlessRaises(GitCommandError, r.git.cherry_pick, 'master') # Now do the actual testing - this should just work - assert len(r.index.diff(None)) == 2 + self.assertEqual(len(r.index.diff(None)), 2) - assert len(r.index.diff(None, create_patch=True)) == 0, "This should work, but doesn't right now ... it's OK" + self.assertEqual(len(r.index.diff(None, create_patch=True)), 0, + "This should work, but doesn't right now ... it's OK") def test_list_from_string_new_mode(self): output = StringProcessAdapter(fixture('diff_new_mode')) - diffs = Diff._index_from_patch_format(self.rorepo, output.stdout) + diffs = Diff._index_from_patch_format(self.rorepo, output) self._assert_diff_format(diffs) assert_equal(1, len(diffs)) @@ -80,7 +87,7 @@ def test_list_from_string_new_mode(self): def test_diff_with_rename(self): output = StringProcessAdapter(fixture('diff_rename')) - diffs = Diff._index_from_patch_format(self.rorepo, output.stdout) + diffs = Diff._index_from_patch_format(self.rorepo, output) self._assert_diff_format(diffs) assert_equal(1, len(diffs)) @@ -95,42 +102,46 @@ def test_diff_with_rename(self): assert isinstance(str(diff), str) output = StringProcessAdapter(fixture('diff_rename_raw')) - diffs = Diff._index_from_raw_format(self.rorepo, output.stdout) - assert len(diffs) == 1 + diffs = Diff._index_from_raw_format(self.rorepo, output) + self.assertEqual(len(diffs), 1) diff = diffs[0] - assert diff.renamed_file - assert diff.renamed - assert diff.rename_from == 'this' - assert diff.rename_to == 'that' - assert len(list(diffs.iter_change_type('R'))) == 1 + self.assertIsNotNone(diff.renamed_file) + self.assertIsNotNone(diff.renamed) + self.assertEqual(diff.rename_from, 'this') + self.assertEqual(diff.rename_to, 'that') + self.assertEqual(len(list(diffs.iter_change_type('R'))), 1) def test_diff_of_modified_files_not_added_to_the_index(self): output = StringProcessAdapter(fixture('diff_abbrev-40_full-index_M_raw_no-color')) - diffs = Diff._index_from_raw_format(self.rorepo, output.stdout) - - assert len(diffs) == 1, 'one modification' - assert len(list(diffs.iter_change_type('M'))) == 1, 'one modification' - assert diffs[0].change_type == 'M' - assert diffs[0].b_blob is None - - def test_binary_diff(self): - for method, file_name in ((Diff._index_from_patch_format, 'diff_patch_binary'), - (Diff._index_from_raw_format, 'diff_raw_binary')): - res = method(None, StringProcessAdapter(fixture(file_name)).stdout) - assert len(res) == 1 - assert len(list(res.iter_change_type('M'))) == 1 - if res[0].diff: - assert res[0].diff == b"Binary files a/rps and b/rps differ\n", "in patch mode, we get a diff text" - assert str(res[0]), "This call should just work" - # end for each method to test + diffs = Diff._index_from_raw_format(self.rorepo, output) + + self.assertEqual(len(diffs), 1, 'one modification') + self.assertEqual(len(list(diffs.iter_change_type('M'))), 1, 'one modification') + self.assertEqual(diffs[0].change_type, 'M') + self.assertIsNone(diffs[0].b_blob,) + + @ddt.data( + (Diff._index_from_patch_format, 'diff_patch_binary'), + (Diff._index_from_raw_format, 'diff_raw_binary') + ) + def test_binary_diff(self, case): + method, file_name = case + res = method(None, StringProcessAdapter(fixture(file_name))) + self.assertEqual(len(res), 1) + self.assertEqual(len(list(res.iter_change_type('M'))), 1) + if res[0].diff: + self.assertEqual(res[0].diff, + b"Binary files a/rps and b/rps differ\n", + "in patch mode, we get a diff text") + self.assertIsNotNone(str(res[0]), "This call should just work") def test_diff_index(self): output = StringProcessAdapter(fixture('diff_index_patch')) - res = Diff._index_from_patch_format(None, output.stdout) - assert len(res) == 6 + res = Diff._index_from_patch_format(None, output) + self.assertEqual(len(res), 6) for dr in res: - assert dr.diff.startswith(b'@@') - assert str(dr), "Diff to string conversion should be possible" + self.assertTrue(dr.diff.startswith(b'@@'), dr) + self.assertIsNotNone(str(dr), "Diff to string conversion should be possible") # end for each diff dr = res[3] @@ -138,29 +149,29 @@ def test_diff_index(self): def test_diff_index_raw_format(self): output = StringProcessAdapter(fixture('diff_index_raw')) - res = Diff._index_from_raw_format(None, output.stdout) - assert res[0].deleted_file - assert res[0].b_path is None + res = Diff._index_from_raw_format(None, output) + self.assertIsNotNone(res[0].deleted_file) + self.assertIsNone(res[0].b_path,) def test_diff_initial_commit(self): initial_commit = self.rorepo.commit('33ebe7acec14b25c5f84f35a664803fcab2f7781') # Without creating a patch... diff_index = initial_commit.diff(NULL_TREE) - assert diff_index[0].b_path == 'CHANGES' - assert diff_index[0].new_file - assert diff_index[0].diff == '' + self.assertEqual(diff_index[0].b_path, 'CHANGES') + self.assertIsNotNone(diff_index[0].new_file) + self.assertEqual(diff_index[0].diff, '') # ...and with creating a patch diff_index = initial_commit.diff(NULL_TREE, create_patch=True) - assert diff_index[0].a_path is None, repr(diff_index[0].a_path) - assert diff_index[0].b_path == 'CHANGES', repr(diff_index[0].b_path) - assert diff_index[0].new_file - assert diff_index[0].diff == fixture('diff_initial') + self.assertIsNone(diff_index[0].a_path, repr(diff_index[0].a_path)) + self.assertEqual(diff_index[0].b_path, 'CHANGES', repr(diff_index[0].b_path)) + self.assertIsNotNone(diff_index[0].new_file) + self.assertEqual(diff_index[0].diff, fixture('diff_initial')) def test_diff_unsafe_paths(self): output = StringProcessAdapter(fixture('diff_patch_unsafe_paths')) - res = Diff._index_from_patch_format(None, output.stdout) + res = Diff._index_from_patch_format(None, output) # The "Additions" self.assertEqual(res[0].b_path, u'path/ starting with a space') @@ -196,14 +207,14 @@ def test_diff_patch_format(self): for fixture_name in fixtures: diff_proc = StringProcessAdapter(fixture(fixture_name)) - Diff._index_from_patch_format(self.rorepo, diff_proc.stdout) + Diff._index_from_patch_format(self.rorepo, diff_proc) # END for each fixture def test_diff_with_spaces(self): data = StringProcessAdapter(fixture('diff_file_with_spaces')) - diff_index = Diff._index_from_patch_format(self.rorepo, data.stdout) - assert diff_index[0].a_path is None, repr(diff_index[0].a_path) - assert diff_index[0].b_path == u'file with spaces', repr(diff_index[0].b_path) + diff_index = Diff._index_from_patch_format(self.rorepo, data) + self.assertIsNone(diff_index[0].a_path, repr(diff_index[0].a_path)) + self.assertEqual(diff_index[0].b_path, u'file with spaces', repr(diff_index[0].b_path)) def test_diff_interface(self): # test a few variations of the main diff routine @@ -232,12 +243,12 @@ def test_diff_interface(self): diff_set = set() diff_set.add(diff_index[0]) diff_set.add(diff_index[0]) - assert len(diff_set) == 1 - assert diff_index[0] == diff_index[0] - assert not (diff_index[0] != diff_index[0]) + self.assertEqual(len(diff_set), 1) + self.assertEqual(diff_index[0], diff_index[0]) + self.assertFalse(diff_index[0] != diff_index[0]) for dr in diff_index: - assert str(dr), "Diff to string conversion should be possible" + self.assertIsNotNone(str(dr), "Diff to string conversion should be possible") # END diff index checking # END for each patch option # END for each path option @@ -248,11 +259,11 @@ def test_diff_interface(self): # can iterate in the diff index - if not this indicates its not working correctly # or our test does not span the whole range of possibilities for key, value in assertion_map.items(): - assert value, "Did not find diff for %s" % key + self.assertIsNotNone(value, "Did not find diff for %s" % key) # END for each iteration type # test path not existing in the index - should be ignored c = self.rorepo.head.commit cp = c.parents[0] diff_index = c.diff(cp, ["does/not/exist"]) - assert len(diff_index) == 0 + self.assertEqual(len(diff_index), 0) diff --git a/git/test/test_docs.py b/git/test/test_docs.py index b297363dc..8a2dff0f5 100644 --- a/git/test/test_docs.py +++ b/git/test/test_docs.py @@ -7,10 +7,15 @@ import os from git.test.lib import TestBase -from gitdb.test.lib import with_rw_directory +from git.test.lib.helper import with_rw_directory class Tutorials(TestBase): + + def tearDown(self): + import gc + gc.collect() + @with_rw_directory def test_init_repo_object(self, rw_dir): # [1-test_init_repo_object] @@ -48,33 +53,34 @@ def test_init_repo_object(self, rw_dir): # ![5-test_init_repo_object] # [6-test_init_repo_object] - repo.archive(open(join(rw_dir, 'repo.tar'), 'wb')) + with open(join(rw_dir, 'repo.tar'), 'wb') as fp: + repo.archive(fp) # ![6-test_init_repo_object] # repository paths # [7-test_init_repo_object] - assert os.path.isdir(cloned_repo.working_tree_dir) # directory with your work files - assert cloned_repo.git_dir.startswith(cloned_repo.working_tree_dir) # directory containing the git repository - assert bare_repo.working_tree_dir is None # bare repositories have no working tree + assert os.path.isdir(cloned_repo.working_tree_dir) # directory with your work files + assert cloned_repo.git_dir.startswith(cloned_repo.working_tree_dir) # directory containing the git repository + assert bare_repo.working_tree_dir is None # bare repositories have no working tree # ![7-test_init_repo_object] # heads, tags and references # heads are branches in git-speak # [8-test_init_repo_object] - assert repo.head.ref == repo.heads.master # head is a symbolic reference pointing to master - assert repo.tags['0.3.5'] == repo.tag('refs/tags/0.3.5') # you can access tags in various ways too - assert repo.refs.master == repo.heads['master'] # .refs provides access to all refs, i.e. heads ... - + self.assertEqual(repo.head.ref, repo.heads.master) # head is a sym-ref pointing to master + self.assertEqual(repo.tags['0.3.5'], repo.tag('refs/tags/0.3.5')) # you can access tags in various ways too + self.assertEqual(repo.refs.master, repo.heads['master']) # .refs provides all refs, ie heads ... + if 'TRAVIS' not in os.environ: - assert repo.refs['origin/master'] == repo.remotes.origin.refs.master # ... remotes ... - assert repo.refs['0.3.5'] == repo.tags['0.3.5'] # ... and tags + self.assertEqual(repo.refs['origin/master'], repo.remotes.origin.refs.master) # ... remotes ... + self.assertEqual(repo.refs['0.3.5'], repo.tags['0.3.5']) # ... and tags # ![8-test_init_repo_object] # create a new head/branch # [9-test_init_repo_object] new_branch = cloned_repo.create_head('feature') # create a new branch ... assert cloned_repo.active_branch != new_branch # which wasn't checked out yet ... - assert new_branch.commit == cloned_repo.active_branch.commit # and which points to the checked-out commit + self.assertEqual(new_branch.commit, cloned_repo.active_branch.commit) # pointing to the checked-out commit # It's easy to let a branch point to the previous commit, without affecting anything else # Each reference provides access to the git object it points to, usually commits assert new_branch.set_commit('HEAD~1').commit == cloned_repo.active_branch.commit.parents[0] @@ -84,7 +90,7 @@ def test_init_repo_object(self, rw_dir): # [10-test_init_repo_object] past = cloned_repo.create_tag('past', ref=new_branch, message="This is a tag-object pointing to %s" % new_branch.name) - assert past.commit == new_branch.commit # the tag points to the specified commit + self.assertEqual(past.commit, new_branch.commit) # the tag points to the specified commit assert past.tag.message.startswith("This is") # and its object carries the message provided now = cloned_repo.create_tag('now') # This is a tag-reference. It may not carry meta-data @@ -105,7 +111,7 @@ def test_init_repo_object(self, rw_dir): file_count += item.type == 'blob' tree_count += item.type == 'tree' assert file_count and tree_count # we have accumulated all directories and files - assert len(tree.blobs) + len(tree.trees) == len(tree) # a tree is iterable itself to traverse its children + self.assertEqual(len(tree.blobs) + len(tree.trees), len(tree)) # a tree is iterable on its children # ![11-test_init_repo_object] # remotes allow handling push, pull and fetch operations @@ -117,8 +123,8 @@ def update(self, op_code, cur_count, max_count=None, message=''): print(op_code, cur_count, max_count, cur_count / (max_count or 100.0), message or "NO MESSAGE") # end - assert len(cloned_repo.remotes) == 1 # we have been cloned, so there should be one remote - assert len(bare_repo.remotes) == 0 # this one was just initialized + self.assertEqual(len(cloned_repo.remotes), 1) # we have been cloned, so should be one remote + self.assertEqual(len(bare_repo.remotes), 0) # this one was just initialized origin = bare_repo.create_remote('origin', url=cloned_repo.working_tree_dir) assert origin.exists() for fetch_info in origin.fetch(progress=MyProgressPrinter()): @@ -133,8 +139,8 @@ def update(self, op_code, cur_count, max_count=None, message=''): # index # [13-test_init_repo_object] - assert new_branch.checkout() == cloned_repo.active_branch # checking out a branch adjusts the working tree - assert new_branch.commit == past.commit # Now the past is checked out + self.assertEqual(new_branch.checkout(), cloned_repo.active_branch) # checking out branch adjusts the wtree + self.assertEqual(new_branch.commit, past.commit) # Now the past is checked out new_file_path = os.path.join(cloned_repo.working_tree_dir, 'my-new-file') open(new_file_path, 'wb').close() # create new file in working tree @@ -205,7 +211,7 @@ def test_references_and_objects(self, rw_dir): master = head.reference # retrieve the reference the head points to master.commit # from here you use it as any other reference # ![3-test_references_and_objects] - +# # [4-test_references_and_objects] log = master.log() log[0] # first (i.e. oldest) reflog entry @@ -239,17 +245,17 @@ def test_references_and_objects(self, rw_dir): # ![8-test_references_and_objects] # [9-test_references_and_objects] - assert hct.type == 'tree' # preset string type, being a class attribute + self.assertEqual(hct.type, 'tree') # preset string type, being a class attribute assert hct.size > 0 # size in bytes assert len(hct.hexsha) == 40 assert len(hct.binsha) == 20 # ![9-test_references_and_objects] # [10-test_references_and_objects] - assert hct.path == '' # root tree has no path + self.assertEqual(hct.path, '') # root tree has no path assert hct.trees[0].path != '' # the first contained item has one though - assert hct.mode == 0o40000 # trees have the mode of a linux directory - assert hct.blobs[0].mode == 0o100644 # blobs have a specific mode though comparable to a standard linux fs + self.assertEqual(hct.mode, 0o40000) # trees have the mode of a linux directory + self.assertEqual(hct.blobs[0].mode, 0o100644) # blobs have specific mode, comparable to a standard linux fs # ![10-test_references_and_objects] # [11-test_references_and_objects] @@ -306,14 +312,14 @@ def test_references_and_objects(self, rw_dir): # ![18-test_references_and_objects] # [19-test_references_and_objects] - assert tree['smmap'] == tree / 'smmap' # access by index and by sub-path + self.assertEqual(tree['smmap'], tree / 'smmap') # access by index and by sub-path for entry in tree: # intuitive iteration of tree members print(entry) blob = tree.trees[0].blobs[0] # let's get a blob in a sub-tree assert blob.name assert len(blob.path) < len(blob.abspath) - assert tree.trees[0].name + '/' + blob.name == blob.path # this is how the relative blob path is generated - assert tree[blob.path] == blob # you can use paths like 'dir/file' in tree[...] + self.assertEqual(tree.trees[0].name + '/' + blob.name, blob.path) # this is how relative blob path generated + self.assertEqual(tree[blob.path], blob) # you can use paths like 'dir/file' in tree # ![19-test_references_and_objects] # [20-test_references_and_objects] @@ -326,7 +332,7 @@ def test_references_and_objects(self, rw_dir): assert repo.tree() == repo.head.commit.tree past = repo.commit('HEAD~5') assert repo.tree(past) == repo.tree(past.hexsha) - assert repo.tree('v0.8.1').type == 'tree' # yes, you can provide any refspec - works everywhere + self.assertEqual(repo.tree('v0.8.1').type, 'tree') # yes, you can provide any refspec - works everywhere # ![21-test_references_and_objects] # [22-test_references_and_objects] @@ -346,7 +352,7 @@ def test_references_and_objects(self, rw_dir): index.remove(['LICENSE']) # remove an existing one assert os.path.isfile(os.path.join(repo.working_tree_dir, 'LICENSE')) # working tree is untouched - assert index.commit("my commit message").type == 'commit' # commit changed index + self.assertEqual(index.commit("my commit message").type, 'commit') # commit changed index repo.active_branch.commit = repo.commit('HEAD~1') # forget last commit from git import Actor @@ -373,7 +379,7 @@ def test_references_and_objects(self, rw_dir): assert origin == empty_repo.remotes.origin == empty_repo.remotes['origin'] origin.fetch() # assure we actually have data. fetch() returns useful information # Setup a local tracking branch of a remote branch - empty_repo.create_head('master', origin.refs.master) # create local branch "master" from remote branch "master" + empty_repo.create_head('master', origin.refs.master) # create local branch "master" from remote "master" empty_repo.heads.master.set_tracking_branch(origin.refs.master) # set local "master" to track remote "master empty_repo.heads.master.checkout() # checkout local "master" to working tree # Three above commands in one: @@ -443,6 +449,8 @@ def test_references_and_objects(self, rw_dir): git.for_each_ref() # '-' becomes '_' when calling it # ![31-test_references_and_objects] + repo.git.clear_cache() + def test_submodules(self): # [1-test_submodules] repo = self.rorepo @@ -450,19 +458,19 @@ def test_submodules(self): assert len(sms) == 1 sm = sms[0] - assert sm.name == 'gitdb' # git-python has gitdb as single submodule ... - assert sm.children()[0].name == 'smmap' # ... which has smmap as single submodule + self.assertEqual(sm.name, 'gitdb') # git-python has gitdb as single submodule ... + self.assertEqual(sm.children()[0].name, 'smmap') # ... which has smmap as single submodule # The module is the repository referenced by the submodule assert sm.module_exists() # the module is available, which doesn't have to be the case. assert sm.module().working_tree_dir.endswith('gitdb') # the submodule's absolute path is the module's path assert sm.abspath == sm.module().working_tree_dir - assert len(sm.hexsha) == 40 # Its sha defines the commit to checkout + self.assertEqual(len(sm.hexsha), 40) # Its sha defines the commit to checkout assert sm.exists() # yes, this submodule is valid and exists # read its configuration conveniently assert sm.config_reader().get_value('path') == sm.path - assert len(sm.children()) == 1 # query the submodule hierarchy + self.assertEqual(len(sm.children()), 1) # query the submodule hierarchy # ![1-test_submodules] @with_rw_directory diff --git a/git/test/test_exc.py b/git/test/test_exc.py new file mode 100644 index 000000000..7e6b023e5 --- /dev/null +++ b/git/test/test_exc.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +# test_exc.py +# Copyright (C) 2008, 2009, 2016 Michael Trier (mtrier@gmail.com) and contributors +# +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php + + +import re + +import ddt +from git.exc import ( + CommandError, + GitCommandNotFound, + GitCommandError, + HookExecutionError, +) +from git.test.lib import TestBase + +import itertools as itt + + +_cmd_argvs = ( + ('cmd', ), + ('θνιψοδε', ), + ('θνιψοδε', 'normal', 'argvs'), + ('cmd', 'ελληνικα', 'args'), + ('θνιψοδε', 'κι', 'αλλα', 'strange', 'args'), + ('θνιψοδε', 'κι', 'αλλα', 'non-unicode', 'args'), +) +_causes_n_substrings = ( + (None, None), # noqa: E241 + (7, "exit code(7)"), # noqa: E241 + ('Some string', "'Some string'"), # noqa: E241 + ('παλιο string', "'παλιο string'"), # noqa: E241 + (Exception("An exc."), "Exception('An exc.')"), # noqa: E241 + (Exception("Κακια exc."), "Exception('Κακια exc.')"), # noqa: E241 + (object(), " 1 @@ -411,7 +414,7 @@ def test_index_mutation(self, rw_repo): writer.set_value("user", "name", uname) writer.set_value("user", "email", umail) writer.release() - assert writer.get_value("user", "name") == uname + self.assertEqual(writer.get_value("user", "name"), uname) # remove all of the files, provide a wild mix of paths, BaseIndexEntries, # IndexEntries @@ -434,21 +437,21 @@ def mixed_iterator(): # END mixed iterator deleted_files = index.remove(mixed_iterator(), working_tree=False) assert deleted_files - assert self._count_existing(rw_repo, deleted_files) == len(deleted_files) - assert len(index.entries) == 0 + self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) + self.assertEqual(len(index.entries), 0) # reset the index to undo our changes index.reset() - assert len(index.entries) == num_entries + self.assertEqual(len(index.entries), num_entries) # remove with working copy deleted_files = index.remove(mixed_iterator(), working_tree=True) assert deleted_files - assert self._count_existing(rw_repo, deleted_files) == 0 + self.assertEqual(self._count_existing(rw_repo, deleted_files), 0) # reset everything index.reset(working_tree=True) - assert self._count_existing(rw_repo, deleted_files) == len(deleted_files) + self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) # invalid type self.failUnlessRaises(TypeError, index.remove, [1]) @@ -465,14 +468,14 @@ def mixed_iterator(): new_commit = index.commit(commit_message, head=False) assert cur_commit != new_commit - assert new_commit.author.name == uname - assert new_commit.author.email == umail - assert new_commit.committer.name == uname - assert new_commit.committer.email == umail - assert new_commit.message == commit_message - assert new_commit.parents[0] == cur_commit - assert len(new_commit.parents) == 1 - assert cur_head.commit == cur_commit + self.assertEqual(new_commit.author.name, uname) + self.assertEqual(new_commit.author.email, umail) + self.assertEqual(new_commit.committer.name, uname) + self.assertEqual(new_commit.committer.email, umail) + self.assertEqual(new_commit.message, commit_message) + self.assertEqual(new_commit.parents[0], cur_commit) + self.assertEqual(len(new_commit.parents), 1) + self.assertEqual(cur_head.commit, cur_commit) # commit with other actor cur_commit = cur_head.commit @@ -481,15 +484,15 @@ def mixed_iterator(): my_committer = Actor(u"Committing Frèderic Çaufl€", "committer@example.com") commit_actor = index.commit(commit_message, author=my_author, committer=my_committer) assert cur_commit != commit_actor - assert commit_actor.author.name == u"Frèderic Çaufl€" - assert commit_actor.author.email == "author@example.com" - assert commit_actor.committer.name == u"Committing Frèderic Çaufl€" - assert commit_actor.committer.email == "committer@example.com" - assert commit_actor.message == commit_message - assert commit_actor.parents[0] == cur_commit - assert len(new_commit.parents) == 1 - assert cur_head.commit == commit_actor - assert cur_head.log()[-1].actor == my_committer + self.assertEqual(commit_actor.author.name, u"Frèderic Çaufl€") + self.assertEqual(commit_actor.author.email, "author@example.com") + self.assertEqual(commit_actor.committer.name, u"Committing Frèderic Çaufl€") + self.assertEqual(commit_actor.committer.email, "committer@example.com") + self.assertEqual(commit_actor.message, commit_message) + self.assertEqual(commit_actor.parents[0], cur_commit) + self.assertEqual(len(new_commit.parents), 1) + self.assertEqual(cur_head.commit, commit_actor) + self.assertEqual(cur_head.log()[-1].actor, my_committer) # commit with author_date and commit_date cur_commit = cur_head.commit @@ -498,25 +501,25 @@ def mixed_iterator(): new_commit = index.commit(commit_message, author_date="2006-04-07T22:13:13", commit_date="2005-04-07T22:13:13") assert cur_commit != new_commit print(new_commit.authored_date, new_commit.committed_date) - assert new_commit.message == commit_message - assert new_commit.authored_date == 1144447993 - assert new_commit.committed_date == 1112911993 + self.assertEqual(new_commit.message, commit_message) + self.assertEqual(new_commit.authored_date, 1144447993) + self.assertEqual(new_commit.committed_date, 1112911993) # same index, no parents commit_message = "index without parents" commit_no_parents = index.commit(commit_message, parent_commits=list(), head=True) - assert commit_no_parents.message == commit_message - assert len(commit_no_parents.parents) == 0 - assert cur_head.commit == commit_no_parents + self.assertEqual(commit_no_parents.message, commit_message) + self.assertEqual(len(commit_no_parents.parents), 0) + self.assertEqual(cur_head.commit, commit_no_parents) # same index, multiple parents commit_message = "Index with multiple parents\n commit with another line" commit_multi_parent = index.commit(commit_message, parent_commits=(commit_no_parents, new_commit)) - assert commit_multi_parent.message == commit_message - assert len(commit_multi_parent.parents) == 2 - assert commit_multi_parent.parents[0] == commit_no_parents - assert commit_multi_parent.parents[1] == new_commit - assert cur_head.commit == commit_multi_parent + self.assertEqual(commit_multi_parent.message, commit_message) + self.assertEqual(len(commit_multi_parent.parents), 2) + self.assertEqual(commit_multi_parent.parents[0], commit_no_parents) + self.assertEqual(commit_multi_parent.parents[1], new_commit) + self.assertEqual(cur_head.commit, commit_multi_parent) # re-add all files in lib # get the lib folder back on disk, but get an index without it @@ -535,17 +538,17 @@ def mixed_iterator(): entries = index.reset(new_commit).add([os.path.join('lib', 'git', '*.py')], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) - assert len(entries) == 14 + self.assertEqual(len(entries), 14) # same file entries = index.reset(new_commit).add( [os.path.join(rw_repo.working_tree_dir, 'lib', 'git', 'head.py')] * 2, fprogress=self._fprogress_add) self._assert_entries(entries) - assert entries[0].mode & 0o644 == 0o644 + self.assertEqual(entries[0].mode & 0o644, 0o644) # would fail, test is too primitive to handle this case # self._assert_fprogress(entries) self._reset_progress() - assert len(entries) == 2 + self.assertEqual(len(entries), 2) # missing path self.failUnlessRaises(OSError, index.reset(new_commit).add, ['doesnt/exist/must/raise']) @@ -555,7 +558,8 @@ def mixed_iterator(): entries = index.reset(new_commit).add([old_blob], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) - assert index.entries[(old_blob.path, 0)].hexsha == old_blob.hexsha and len(entries) == 1 + self.assertEqual(index.entries[(old_blob.path, 0)].hexsha, old_blob.hexsha) + self.assertEqual(len(entries), 1) # mode 0 not allowed null_hex_sha = Diff.NULL_HEX_SHA @@ -570,23 +574,25 @@ def mixed_iterator(): [BaseIndexEntry((0o10644, null_bin_sha, 0, new_file_relapath))], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) - assert len(entries) == 1 and entries[0].hexsha != null_hex_sha + self.assertEqual(len(entries), 1) + self.assertNotEquals(entries[0].hexsha, null_hex_sha) # add symlink - if sys.platform != "win32": + if not is_win: for target in ('/etc/nonexisting', '/etc/passwd', '/etc'): basename = "my_real_symlink" - + link_file = os.path.join(rw_repo.working_tree_dir, basename) os.symlink(target, link_file) entries = index.reset(new_commit).add([link_file], fprogress=self._fprogress_add) self._assert_entries(entries) self._assert_fprogress(entries) - assert len(entries) == 1 and S_ISLNK(entries[0].mode) - assert S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode) + self.assertEqual(len(entries), 1) + self.assertTrue(S_ISLNK(entries[0].mode)) + self.assertTrue(S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode)) # we expect only the target to be written - assert index.repo.odb.stream(entries[0].binsha).read().decode('ascii') == target + self.assertEqual(index.repo.odb.stream(entries[0].binsha).read().decode('ascii'), target) os.remove(link_file) # end for each target @@ -601,7 +607,8 @@ def mixed_iterator(): self._assert_entries(entries) self._assert_fprogress(entries) assert entries[0].hexsha != null_hex_sha - assert len(entries) == 1 and S_ISLNK(entries[0].mode) + self.assertEqual(len(entries), 1) + self.assertTrue(S_ISLNK(entries[0].mode)) # assure this also works with an alternate method full_index_entry = IndexEntry.from_base(BaseIndexEntry((0o120000, entries[0].binsha, 0, entries[0].path))) @@ -626,12 +633,13 @@ def mixed_iterator(): index.checkout(fake_symlink_path) # on windows we will never get symlinks - if os.name == 'nt': + if is_win: # simlinks should contain the link as text ( which is what a # symlink actually is ) - open(fake_symlink_path, 'rb').read() == link_target + with open(fake_symlink_path, 'rt') as fd: + self.assertEqual(fd.read(), link_target) else: - assert S_ISLNK(os.lstat(fake_symlink_path)[ST_MODE]) + self.assertTrue(S_ISLNK(os.lstat(fake_symlink_path)[ST_MODE])) # TEST RENAMING def assert_mv_rval(rval): @@ -651,7 +659,7 @@ def assert_mv_rval(rval): # files into directory - dry run paths = ['LICENSE', 'VERSION', 'doc'] rval = index.move(paths, dry_run=True) - assert len(rval) == 2 + self.assertEqual(len(rval), 2) assert os.path.exists(paths[0]) # again, no dry run @@ -681,7 +689,8 @@ def make_paths(): for fid in range(3): fname = 'newfile%i' % fid - open(fname, 'wb').write(b"abcd") + with open(fname, 'wb') as fd: + fd.write(b"abcd") yield Blob(rw_repo, Blob.NULL_BIN_SHA, 0o100644, fname) # END for each new file # END path producer @@ -707,7 +716,7 @@ def make_paths(): assert fkey not in index.entries index.add(files, write=True) - if os.name != 'nt': + if is_win: hp = hook_path('pre-commit', index.repo.git_dir) hpd = os.path.dirname(hp) if not os.path.isdir(hpd): @@ -715,15 +724,22 @@ def make_paths(): with open(hp, "wt") as fp: fp.write("#!/usr/bin/env sh\necho stdout; echo stderr 1>&2; exit 1") # end - os.chmod(hp, 0o544) + os.chmod(hp, 0o744) try: index.commit("This should fail") except HookExecutionError as err: - assert err.status == 1 - assert err.command == hp - assert err.stdout == 'stdout\n' - assert err.stderr == 'stderr\n' - assert str(err) + if is_win: + self.assertIsInstance(err.status, OSError) + self.assertEqual(err.command, [hp]) + self.assertEqual(err.stdout, '') + self.assertEqual(err.stderr, '') + assert str(err) + else: + self.assertEqual(err.status, 1) + self.assertEqual(err.command, hp) + self.assertEqual(err.stdout, 'stdout\n') + self.assertEqual(err.stderr, 'stderr\n') + assert str(err) else: raise AssertionError("Should have cought a HookExecutionError") # end exception handling @@ -763,7 +779,7 @@ def test_compare_write_tree(self, rw_repo): count += 1 index = rw_repo.index.reset(commit) orig_tree = commit.tree - assert index.write_tree() == orig_tree + self.assertEqual(index.write_tree(), orig_tree) # END for each commit def test_index_new(self): diff --git a/git/test/test_reflog.py b/git/test/test_reflog.py index 3571e0839..dffedf3b6 100644 --- a/git/test/test_reflog.py +++ b/git/test/test_reflog.py @@ -7,11 +7,10 @@ RefLogEntry, RefLog ) -from git.util import Actor +from git.util import Actor, rmtree from gitdb.util import hex_to_bin import tempfile -import shutil import os @@ -104,4 +103,4 @@ def test_base(self): # END for each reflog # finally remove our temporary data - shutil.rmtree(tdir) + rmtree(tdir) diff --git a/git/test/test_remote.py b/git/test/test_remote.py index 3c2e622d7..b99e49cfa 100644 --- a/git/test/test_remote.py +++ b/git/test/test_remote.py @@ -25,10 +25,9 @@ Remote, GitCommandError ) -from git.util import IterableList +from git.util import IterableList, rmtree from git.compat import string_types import tempfile -import shutil import os import random @@ -101,9 +100,13 @@ def assert_received_message(self): class TestRemote(TestBase): + def tearDown(self): + import gc + gc.collect() + def _print_fetchhead(self, repo): - fp = open(os.path.join(repo.git_dir, "FETCH_HEAD")) - fp.close() + with open(os.path.join(repo.git_dir, "FETCH_HEAD")): + pass def _do_test_fetch_result(self, results, remote): # self._print_fetchhead(remote.repo) @@ -281,7 +284,7 @@ def get_info(res, remote, name): # and only provides progress information to ttys res = fetch_and_test(other_origin) finally: - shutil.rmtree(other_repo_dir) + rmtree(other_repo_dir) # END test and cleanup def _assert_push_and_pull(self, remote, rw_repo, remote_repo): @@ -403,7 +406,7 @@ def test_base(self, rw_repo, remote_repo): # OPTIONS # cannot use 'fetch' key anymore as it is now a method - for opt in ("url", ): + for opt in ("url",): val = getattr(remote, opt) reader = remote.config_reader assert reader.get(opt) == val diff --git a/git/test/test_repo.py b/git/test/test_repo.py index d04a0f66f..349d955e7 100644 --- a/git/test/test_repo.py +++ b/git/test/test_repo.py @@ -4,18 +4,15 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +import glob +from io import BytesIO +import itertools +import functools as fnt +import os import pickle +import sys +import tempfile -from git.test.lib import ( - patch, - TestBase, - with_rw_repo, - fixture, - assert_false, - assert_equal, - assert_true, - raises -) from git import ( InvalidGitRepositoryError, Repo, @@ -33,24 +30,33 @@ BadName, GitCommandError ) -from git.repo.fun import touch -from git.util import join_path_native +from git.compat import ( + PY3, + is_win, + string_types, + win_encode, +) from git.exc import ( BadObject, ) +from git.repo.fun import touch +from git.test.lib import ( + patch, + TestBase, + with_rw_repo, + fixture, + assert_false, + assert_equal, + assert_true, + raises +) +from git.test.lib import with_rw_directory +from git.util import join_path_native, rmtree, rmfile from gitdb.util import bin_to_hex -from git.compat import string_types -from gitdb.test.lib import with_rw_directory - -import os -import sys -import tempfile -import shutil -import itertools -from io import BytesIO - from nose import SkipTest +import os.path as osp + def iter_flatten(lol): for items in lol: @@ -62,8 +68,26 @@ def flatten(lol): return list(iter_flatten(lol)) +_tc_lock_fpaths = osp.join(osp.dirname(__file__), '../../.git/*.lock') + + +def _rm_lock_files(): + for lfp in glob.glob(_tc_lock_fpaths): + rmfile(lfp) + + class TestRepo(TestBase): + def setUp(self): + _rm_lock_files() + + def tearDown(self): + for lfp in glob.glob(_tc_lock_fpaths): + if osp.isfile(lfp): + raise AssertionError('Previous TC left hanging git-lock file: %s', lfp) + import gc + gc.collect() + @raises(InvalidGitRepositoryError) def test_new_should_raise_on_invalid_repo_location(self): Repo(tempfile.gettempdir()) @@ -75,10 +99,10 @@ def test_new_should_raise_on_non_existant_path(self): @with_rw_repo('0.3.2.1') def test_repo_creation_from_different_paths(self, rw_repo): r_from_gitdir = Repo(rw_repo.git_dir) - assert r_from_gitdir.git_dir == rw_repo.git_dir + self.assertEqual(r_from_gitdir.git_dir, rw_repo.git_dir) assert r_from_gitdir.git_dir.endswith('.git') assert not rw_repo.git.working_dir.endswith('.git') - assert r_from_gitdir.git.working_dir == rw_repo.git.working_dir + self.assertEqual(r_from_gitdir.git.working_dir, rw_repo.git.working_dir) def test_description(self): txt = "Test repository" @@ -92,33 +116,33 @@ def test_heads_should_return_array_of_head_objects(self): def test_heads_should_populate_head_data(self): for head in self.rorepo.heads: assert head.name - assert isinstance(head.commit, Commit) + self.assertIsInstance(head.commit, Commit) # END for each head - assert isinstance(self.rorepo.heads.master, Head) - assert isinstance(self.rorepo.heads['master'], Head) + self.assertIsInstance(self.rorepo.heads.master, Head) + self.assertIsInstance(self.rorepo.heads['master'], Head) def test_tree_from_revision(self): tree = self.rorepo.tree('0.1.6') - assert len(tree.hexsha) == 40 - assert tree.type == "tree" - assert self.rorepo.tree(tree) == tree + self.assertEqual(len(tree.hexsha), 40) + self.assertEqual(tree.type, "tree") + self.assertEqual(self.rorepo.tree(tree), tree) # try from invalid revision that does not exist self.failUnlessRaises(BadName, self.rorepo.tree, 'hello world') - + def test_pickleable(self): pickle.loads(pickle.dumps(self.rorepo)) def test_commit_from_revision(self): commit = self.rorepo.commit('0.1.4') - assert commit.type == 'commit' - assert self.rorepo.commit(commit) == commit + self.assertEqual(commit.type, 'commit') + self.assertEqual(self.rorepo.commit(commit), commit) def test_commits(self): mc = 10 commits = list(self.rorepo.iter_commits('0.1.6', max_count=mc)) - assert len(commits) == mc + self.assertEqual(len(commits), mc) c = commits[0] assert_equal('9a4b1d4d11eee3c5362a4152216376e634bd14cf', c.hexsha) @@ -135,23 +159,23 @@ def test_commits(self): assert_equal("Bumped version 0.1.6\n", c.message) c = commits[1] - assert isinstance(c.parents, tuple) + self.assertIsInstance(c.parents, tuple) def test_trees(self): mc = 30 num_trees = 0 for tree in self.rorepo.iter_trees('0.1.5', max_count=mc): num_trees += 1 - assert isinstance(tree, Tree) + self.assertIsInstance(tree, Tree) # END for each tree - assert num_trees == mc + self.assertEqual(num_trees, mc) def _assert_empty_repo(self, repo): # test all kinds of things with an empty, freshly initialized repo. # It should throw good errors # entries should be empty - assert len(repo.index.entries) == 0 + self.assertEqual(len(repo.index.entries), 0) # head is accessible assert repo.head @@ -183,7 +207,7 @@ def test_init(self): # with specific path for path in (git_dir_rela, git_dir_abs): r = Repo.init(path=path, bare=True) - assert isinstance(r, Repo) + self.assertIsInstance(r, Repo) assert r.bare is True assert not r.has_separate_working_tree() assert os.path.isdir(r.git_dir) @@ -196,7 +220,7 @@ def test_init(self): self._assert_empty_repo(rc) try: - shutil.rmtree(clone_path) + rmtree(clone_path) except OSError: # when relative paths are used, the clone may actually be inside # of the parent directory @@ -207,9 +231,9 @@ def test_init(self): rc = Repo.clone_from(r.git_dir, clone_path) self._assert_empty_repo(rc) - shutil.rmtree(git_dir_abs) + rmtree(git_dir_abs) try: - shutil.rmtree(clone_path) + rmtree(clone_path) except OSError: # when relative paths are used, the clone may actually be inside # of the parent directory @@ -227,7 +251,7 @@ def test_init(self): self._assert_empty_repo(r) finally: try: - shutil.rmtree(del_dir_abs) + rmtree(del_dir_abs) except OSError: pass os.chdir(prev_cwd) @@ -239,18 +263,18 @@ def test_bare_property(self): def test_daemon_export(self): orig_val = self.rorepo.daemon_export self.rorepo.daemon_export = not orig_val - assert self.rorepo.daemon_export == (not orig_val) + self.assertEqual(self.rorepo.daemon_export, (not orig_val)) self.rorepo.daemon_export = orig_val - assert self.rorepo.daemon_export == orig_val + self.assertEqual(self.rorepo.daemon_export, orig_val) def test_alternates(self): cur_alternates = self.rorepo.alternates # empty alternates self.rorepo.alternates = [] - assert self.rorepo.alternates == [] + self.assertEqual(self.rorepo.alternates, []) alts = ["other/location", "this/location"] self.rorepo.alternates = alts - assert alts == self.rorepo.alternates + self.assertEqual(alts, self.rorepo.alternates) self.rorepo.alternates = cur_alternates def test_repr(self): @@ -295,25 +319,27 @@ def test_is_dirty_with_path(self, rwrepo): assert rwrepo.is_dirty(untracked_files=True, path="doc") is True def test_head(self): - assert self.rorepo.head.reference.object == self.rorepo.active_branch.object + self.assertEqual(self.rorepo.head.reference.object, self.rorepo.active_branch.object) def test_index(self): index = self.rorepo.index - assert isinstance(index, IndexFile) + self.assertIsInstance(index, IndexFile) def test_tag(self): assert self.rorepo.tag('refs/tags/0.1.5').commit def test_archive(self): tmpfile = tempfile.mktemp(suffix='archive-test') - stream = open(tmpfile, 'wb') - self.rorepo.archive(stream, '0.1.6', path='doc') - assert stream.tell() - stream.close() + with open(tmpfile, 'wb') as stream: + self.rorepo.archive(stream, '0.1.6', path='doc') + assert stream.tell() os.remove(tmpfile) @patch.object(Git, '_call_process') def test_should_display_blame_information(self, git): + if sys.version_info < (2, 7): + ## Skipped, not `assertRaisesRegexp` in py2.6 + return git.return_value = fixture('blame') b = self.rorepo.blame('master', 'lib/git.py') assert_equal(13, len(b)) @@ -341,7 +367,7 @@ def test_should_display_blame_information(self, git): # BINARY BLAME git.return_value = fixture('blame_binary') blames = self.rorepo.blame('master', 'rps') - assert len(blames) == 2 + self.assertEqual(len(blames), 2) def test_blame_real(self): c = 0 @@ -361,32 +387,35 @@ def test_blame_incremental(self, git): git.return_value = fixture('blame_incremental') blame_output = self.rorepo.blame_incremental('9debf6b0aafb6f7781ea9d1383c86939a1aacde3', 'AUTHORS') blame_output = list(blame_output) - assert len(blame_output) == 5 + self.assertEqual(len(blame_output), 5) # Check all outputted line numbers ranges = flatten([entry.linenos for entry in blame_output]) - assert ranges == flatten([range(2, 3), range(14, 15), range(1, 2), range(3, 14), range(15, 17)]), str(ranges) + self.assertEqual(ranges, flatten([range(2, 3), range(14, 15), range(1, 2), range(3, 14), range(15, 17)])) commits = [entry.commit.hexsha[:7] for entry in blame_output] - assert commits == ['82b8902', '82b8902', 'c76852d', 'c76852d', 'c76852d'], str(commits) + self.assertEqual(commits, ['82b8902', '82b8902', 'c76852d', 'c76852d', 'c76852d']) # Original filenames - assert all([entry.orig_path == u'AUTHORS' for entry in blame_output]) + self.assertSequenceEqual([entry.orig_path for entry in blame_output], [u'AUTHORS'] * len(blame_output)) # Original line numbers orig_ranges = flatten([entry.orig_linenos for entry in blame_output]) - assert orig_ranges == flatten([range(2, 3), range(14, 15), range(1, 2), range(2, 13), range(13, 15)]), str(orig_ranges) # noqa + self.assertEqual(orig_ranges, flatten([range(2, 3), range(14, 15), range(1, 2), range(2, 13), range(13, 15)])) # noqa E501 @patch.object(Git, '_call_process') def test_blame_complex_revision(self, git): git.return_value = fixture('blame_complex_revision') res = self.rorepo.blame("HEAD~10..HEAD", "README.md") - assert len(res) == 1 - assert len(res[0][1]) == 83, "Unexpected amount of parsed blame lines" + self.assertEqual(len(res), 1) + self.assertEqual(len(res[0][1]), 83, "Unexpected amount of parsed blame lines") @with_rw_repo('HEAD', bare=False) def test_untracked_files(self, rwrepo): - for (run, repo_add) in enumerate((rwrepo.index.add, rwrepo.git.add)): + for run, (repo_add, is_invoking_git) in enumerate(( + (rwrepo.index.add, False), + (rwrepo.git.add, True), + )): base = rwrepo.working_tree_dir files = (join_path_native(base, u"%i_test _myfile" % run), join_path_native(base, "%i_test_other_file" % run), @@ -395,9 +424,8 @@ def test_untracked_files(self, rwrepo): num_recently_untracked = 0 for fpath in files: - fd = open(fpath, "wb") - fd.close() - # END for each filename + with open(fpath, "wb"): + pass untracked_files = rwrepo.untracked_files num_recently_untracked = len(untracked_files) @@ -405,10 +433,15 @@ def test_untracked_files(self, rwrepo): num_test_untracked = 0 for utfile in untracked_files: num_test_untracked += join_path_native(base, utfile) in files - assert len(files) == num_test_untracked + self.assertEqual(len(files), num_test_untracked) + if is_win and not PY3 and is_invoking_git: + ## On Windows, shell needed when passing unicode cmd-args. + # + repo_add = fnt.partial(repo_add, shell=True) + untracked_files = [win_encode(f) for f in untracked_files] repo_add(untracked_files) - assert len(rwrepo.untracked_files) == (num_recently_untracked - len(files)) + self.assertEqual(len(rwrepo.untracked_files), (num_recently_untracked - len(files))) # end for each run def test_config_reader(self): @@ -420,19 +453,16 @@ def test_config_reader(self): def test_config_writer(self): for config_level in self.rorepo.config_level: try: - writer = self.rorepo.config_writer(config_level) - assert not writer.read_only - writer.release() + with self.rorepo.config_writer(config_level) as writer: + self.assertFalse(writer.read_only) except IOError: # its okay not to get a writer for some configuration files if we # have no permissions pass - # END for each config level def test_config_level_paths(self): for config_level in self.rorepo.config_level: assert self.rorepo._get_config_path(config_level) - # end for each config level def test_creation_deletion(self): # just a very quick test to assure it generally works. There are @@ -442,15 +472,16 @@ def test_creation_deletion(self): tag = self.rorepo.create_tag("new_tag", "HEAD~2") self.rorepo.delete_tag(tag) - writer = self.rorepo.config_writer() - writer.release() + with self.rorepo.config_writer(): + pass remote = self.rorepo.create_remote("new_remote", "git@server:repo.git") self.rorepo.delete_remote(remote) def test_comparison_and_hash(self): # this is only a preliminary test, more testing done in test_index - assert self.rorepo == self.rorepo and not (self.rorepo != self.rorepo) - assert len(set((self.rorepo, self.rorepo))) == 1 + self.assertEqual(self.rorepo, self.rorepo) + self.assertFalse(self.rorepo != self.rorepo) + self.assertEqual(len(set((self.rorepo, self.rorepo))), 1) @with_rw_directory def test_tilde_and_env_vars_in_repo_path(self, rw_dir): @@ -489,57 +520,59 @@ def mktiny(): # readlines no limit s = mkfull() lines = s.readlines() - assert len(lines) == 3 and lines[-1].endswith(b'\n') - assert s._stream.tell() == len(d) # must have scrubbed to the end + self.assertEqual(len(lines), 3) + self.assertTrue(lines[-1].endswith(b'\n'), lines[-1]) + self.assertEqual(s._stream.tell(), len(d)) # must have scrubbed to the end # realines line limit s = mkfull() lines = s.readlines(5) - assert len(lines) == 1 + self.assertEqual(len(lines), 1) # readlines on tiny sections s = mktiny() lines = s.readlines() - assert len(lines) == 1 and lines[0] == l1p - assert s._stream.tell() == ts + 1 + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0], l1p) + self.assertEqual(s._stream.tell(), ts + 1) # readline no limit s = mkfull() - assert s.readline() == l1 - assert s.readline() == l2 - assert s.readline() == l3 - assert s.readline() == b'' - assert s._stream.tell() == len(d) + self.assertEqual(s.readline(), l1) + self.assertEqual(s.readline(), l2) + self.assertEqual(s.readline(), l3) + self.assertEqual(s.readline(), b'') + self.assertEqual(s._stream.tell(), len(d)) # readline limit s = mkfull() - assert s.readline(5) == l1p - assert s.readline() == l1[5:] + self.assertEqual(s.readline(5), l1p) + self.assertEqual(s.readline(), l1[5:]) # readline on tiny section s = mktiny() - assert s.readline() == l1p - assert s.readline() == b'' - assert s._stream.tell() == ts + 1 + self.assertEqual(s.readline(), l1p) + self.assertEqual(s.readline(), b'') + self.assertEqual(s._stream.tell(), ts + 1) # read no limit s = mkfull() - assert s.read() == d[:-1] - assert s.read() == b'' - assert s._stream.tell() == len(d) + self.assertEqual(s.read(), d[:-1]) + self.assertEqual(s.read(), b'') + self.assertEqual(s._stream.tell(), len(d)) # read limit s = mkfull() - assert s.read(5) == l1p - assert s.read(6) == l1[5:] - assert s._stream.tell() == 5 + 6 # its not yet done + self.assertEqual(s.read(5), l1p) + self.assertEqual(s.read(6), l1[5:]) + self.assertEqual(s._stream.tell(), 5 + 6) # its not yet done # read tiny s = mktiny() - assert s.read(2) == l1[:2] - assert s._stream.tell() == 2 - assert s.read() == l1[2:ts] - assert s._stream.tell() == ts + 1 + self.assertEqual(s.read(2), l1[:2]) + self.assertEqual(s._stream.tell(), 2) + self.assertEqual(s.read(), l1[2:ts]) + self.assertEqual(s._stream.tell(), ts + 1) def _assert_rev_parse_types(self, name, rev_obj): rev_parse = self.rorepo.rev_parse @@ -549,11 +582,12 @@ def _assert_rev_parse_types(self, name, rev_obj): # tree and blob type obj = rev_parse(name + '^{tree}') - assert obj == rev_obj.tree + self.assertEqual(obj, rev_obj.tree) obj = rev_parse(name + ':CHANGES') - assert obj.type == 'blob' and obj.path == 'CHANGES' - assert rev_obj.tree['CHANGES'] == obj + self.assertEqual(obj.type, 'blob') + self.assertEqual(obj.path, 'CHANGES') + self.assertEqual(rev_obj.tree['CHANGES'], obj) def _assert_rev_parse(self, name): """tries multiple different rev-parse syntaxes with the given name @@ -569,7 +603,7 @@ def _assert_rev_parse(self, name): # try history rev = name + "~" obj2 = rev_parse(rev) - assert obj2 == obj.parents[0] + self.assertEqual(obj2, obj.parents[0]) self._assert_rev_parse_types(rev, obj2) # history with number @@ -582,20 +616,20 @@ def _assert_rev_parse(self, name): for pn in range(11): rev = name + "~%i" % (pn + 1) obj2 = rev_parse(rev) - assert obj2 == history[pn] + self.assertEqual(obj2, history[pn]) self._assert_rev_parse_types(rev, obj2) # END history check # parent ( default ) rev = name + "^" obj2 = rev_parse(rev) - assert obj2 == obj.parents[0] + self.assertEqual(obj2, obj.parents[0]) self._assert_rev_parse_types(rev, obj2) # parent with number for pn, parent in enumerate(obj.parents): rev = name + "^%i" % (pn + 1) - assert rev_parse(rev) == parent + self.assertEqual(rev_parse(rev), parent) self._assert_rev_parse_types(rev, parent) # END for each parent @@ -611,7 +645,7 @@ def test_rev_parse(self): rev_parse = self.rorepo.rev_parse # try special case: This one failed at some point, make sure its fixed - assert rev_parse("33ebe").hexsha == "33ebe7acec14b25c5f84f35a664803fcab2f7781" + self.assertEqual(rev_parse("33ebe").hexsha, "33ebe7acec14b25c5f84f35a664803fcab2f7781") # start from reference num_resolved = 0 @@ -622,7 +656,7 @@ def test_rev_parse(self): path_section = '/'.join(path_tokens[-(pt + 1):]) try: obj = self._assert_rev_parse(path_section) - assert obj.type == ref.object.type + self.assertEqual(obj.type, ref.object.type) num_resolved += 1 except (BadName, BadObject): print("failed on %s" % path_section) @@ -637,31 +671,31 @@ def test_rev_parse(self): # it works with tags ! tag = self._assert_rev_parse('0.1.4') - assert tag.type == 'tag' + self.assertEqual(tag.type, 'tag') # try full sha directly ( including type conversion ) - assert tag.object == rev_parse(tag.object.hexsha) + self.assertEqual(tag.object, rev_parse(tag.object.hexsha)) self._assert_rev_parse_types(tag.object.hexsha, tag.object) # multiple tree types result in the same tree: HEAD^{tree}^{tree}:CHANGES rev = '0.1.4^{tree}^{tree}' - assert rev_parse(rev) == tag.object.tree - assert rev_parse(rev + ':CHANGES') == tag.object.tree['CHANGES'] + self.assertEqual(rev_parse(rev), tag.object.tree) + self.assertEqual(rev_parse(rev + ':CHANGES'), tag.object.tree['CHANGES']) # try to get parents from first revision - it should fail as no such revision # exists first_rev = "33ebe7acec14b25c5f84f35a664803fcab2f7781" commit = rev_parse(first_rev) - assert len(commit.parents) == 0 - assert commit.hexsha == first_rev + self.assertEqual(len(commit.parents), 0) + self.assertEqual(commit.hexsha, first_rev) self.failUnlessRaises(BadName, rev_parse, first_rev + "~") self.failUnlessRaises(BadName, rev_parse, first_rev + "^") # short SHA1 commit2 = rev_parse(first_rev[:20]) - assert commit2 == commit + self.assertEqual(commit2, commit) commit2 = rev_parse(first_rev[:5]) - assert commit2 == commit + self.assertEqual(commit2, commit) # todo: dereference tag into a blob 0.1.7^{blob} - quite a special one # needs a tag which points to a blob @@ -669,13 +703,13 @@ def test_rev_parse(self): # ref^0 returns commit being pointed to, same with ref~0, and ^{} tag = rev_parse('0.1.4') for token in (('~0', '^0', '^{}')): - assert tag.object == rev_parse('0.1.4%s' % token) + self.assertEqual(tag.object, rev_parse('0.1.4%s' % token)) # END handle multiple tokens # try partial parsing max_items = 40 for i, binsha in enumerate(self.rorepo.odb.sha_iter()): - assert rev_parse(bin_to_hex(binsha)[:8 - (i % 2)].decode('ascii')).binsha == binsha + self.assertEqual(rev_parse(bin_to_hex(binsha)[:8 - (i % 2)].decode('ascii')).binsha, binsha) if i > max_items: # this is rather slow currently, as rev_parse returns an object # which requires accessing packs, it has some additional overhead @@ -696,13 +730,13 @@ def test_rev_parse(self): self.failUnlessRaises(BadObject, rev_parse, "%s@{0}" % head.commit.hexsha) # uses HEAD.ref by default - assert rev_parse('@{0}') == head.commit + self.assertEqual(rev_parse('@{0}'), head.commit) if not head.is_detached: refspec = '%s@{0}' % head.ref.name - assert rev_parse(refspec) == head.ref.commit + self.assertEqual(rev_parse(refspec), head.ref.commit) # all additional specs work as well - assert rev_parse(refspec + "^{tree}") == head.commit.tree - assert rev_parse(refspec + ":CHANGES").type == 'blob' + self.assertEqual(rev_parse(refspec + "^{tree}"), head.commit.tree) + self.assertEqual(rev_parse(refspec + ":CHANGES").type, 'blob') # END operate on non-detached head # position doesn't exist @@ -718,13 +752,13 @@ def test_repo_odbtype(self): target_type = GitCmdObjectDB if sys.version_info[:2] < (2, 5): target_type = GitCmdObjectDB - assert isinstance(self.rorepo.odb, target_type) + self.assertIsInstance(self.rorepo.odb, target_type) def test_submodules(self): - assert len(self.rorepo.submodules) == 1 # non-recursive - assert len(list(self.rorepo.iter_submodules())) >= 2 + self.assertEqual(len(self.rorepo.submodules), 1) # non-recursive + self.assertGreaterEqual(len(list(self.rorepo.iter_submodules())), 2) - assert isinstance(self.rorepo.submodule("gitdb"), Submodule) + self.assertIsInstance(self.rorepo.submodule("gitdb"), Submodule) self.failUnlessRaises(ValueError, self.rorepo.submodule, "doesn't exist") @with_rw_repo('HEAD', bare=False) @@ -737,7 +771,7 @@ def test_submodule_update(self, rwrepo): # test create submodule sm = rwrepo.submodules[0] sm = rwrepo.create_submodule("my_new_sub", "some_path", join_path_native(self.rorepo.working_tree_dir, sm.path)) - assert isinstance(sm, Submodule) + self.assertIsInstance(sm, Submodule) # note: the rest of this functionality is tested in test_submodule @@ -747,16 +781,18 @@ def test_git_file(self, rwrepo): real_path_abs = os.path.abspath(join_path_native(rwrepo.working_tree_dir, '.real')) os.rename(rwrepo.git_dir, real_path_abs) git_file_path = join_path_native(rwrepo.working_tree_dir, '.git') - open(git_file_path, 'wb').write(fixture('git_file')) + with open(git_file_path, 'wb') as fp: + fp.write(fixture('git_file')) # Create a repo and make sure it's pointing to the relocated .git directory. git_file_repo = Repo(rwrepo.working_tree_dir) - assert os.path.abspath(git_file_repo.git_dir) == real_path_abs + self.assertEqual(os.path.abspath(git_file_repo.git_dir), real_path_abs) # Test using an absolute gitdir path in the .git file. - open(git_file_path, 'wb').write(('gitdir: %s\n' % real_path_abs).encode('ascii')) + with open(git_file_path, 'wb') as fp: + fp.write(('gitdir: %s\n' % real_path_abs).encode('ascii')) git_file_repo = Repo(rwrepo.working_tree_dir) - assert os.path.abspath(git_file_repo.git_dir) == real_path_abs + self.assertEqual(os.path.abspath(git_file_repo.git_dir), real_path_abs) def test_file_handle_leaks(self): def last_commit(repo, rev, path): @@ -777,7 +813,7 @@ def last_commit(repo, rev, path): def test_remote_method(self): self.failUnlessRaises(ValueError, self.rorepo.remote, 'foo-blue') - assert isinstance(self.rorepo.remote(name='origin'), Remote) + self.assertIsInstance(self.rorepo.remote(name='origin'), Remote) @with_rw_directory def test_empty_repo(self, rw_dir): @@ -785,7 +821,7 @@ def test_empty_repo(self, rw_dir): r = Repo.init(rw_dir, mkdir=False) # It's ok not to be able to iterate a commit, as there is none self.failUnlessRaises(ValueError, r.iter_commits) - assert r.active_branch.name == 'master' + self.assertEqual(r.active_branch.name, 'master') assert not r.active_branch.is_valid(), "Branch is yet to be born" # actually, when trying to create a new branch without a commit, git itself fails @@ -825,12 +861,15 @@ def test_merge_base(self): # two commit merge-base res = repo.merge_base(c1, c2) - assert isinstance(res, list) and len(res) == 1 and isinstance(res[0], Commit) - assert res[0].hexsha.startswith('3936084') + self.assertIsInstance(res, list) + self.assertEqual(len(res), 1) + self.assertIsInstance(res[0], Commit) + self.assertTrue(res[0].hexsha.startswith('3936084')) for kw in ('a', 'all'): res = repo.merge_base(c1, c2, c3, **{kw: True}) - assert isinstance(res, list) and len(res) == 1 + self.assertIsInstance(res, list) + self.assertEqual(len(res), 1) # end for each keyword signalling all merge-bases to be returned # Test for no merge base - can't do as we have diff --git a/git/test/test_submodule.py b/git/test/test_submodule.py index 17ce605a4..8e2829b2d 100644 --- a/git/test/test_submodule.py +++ b/git/test/test_submodule.py @@ -9,7 +9,7 @@ TestBase, with_rw_repo ) -from gitdb.test.lib import with_rw_directory +from git.test.lib import with_rw_directory from git.exc import ( InvalidGitRepositoryError, RepositoryDirtyError @@ -17,7 +17,7 @@ from git.objects.submodule.base import Submodule from git.objects.submodule.root import RootModule, RootUpdateProgress from git.util import to_native_path_linux, join_path_native -from git.compat import string_types +from git.compat import string_types, is_win from git.repo.fun import ( find_git_dir, touch @@ -26,7 +26,7 @@ # Change the configuration if possible to prevent the underlying memory manager # to keep file handles open. On windows we get problems as they are not properly # closed due to mmap bugs on windows (as it appears) -if sys.platform == 'win32': +if is_win: try: import smmap.util smmap.util.MapRegion._test_read_into_memory = True @@ -49,6 +49,10 @@ def update(self, op, cur_count, max_count, message=''): class TestSubmodule(TestBase): + def tearDown(self): + import gc + gc.collect() + k_subm_current = "c15a6e1923a14bc760851913858a3942a4193cdb" k_subm_changed = "394ed7006ee5dc8bddfd132b64001d5dfc0ffdd3" k_no_subm_tag = "0.1.6" @@ -305,7 +309,8 @@ def _do_base_tests(self, rwrepo): # but ... we have untracked files in the child submodule fn = join_path_native(csm.module().working_tree_dir, "newfile") - open(fn, 'w').write("hi") + with open(fn, 'w') as fd: + fd.write("hi") self.failUnlessRaises(InvalidGitRepositoryError, sm.remove) # forcibly delete the child repository diff --git a/git/test/test_util.py b/git/test/test_util.py index c6ca6920b..36fb5be3a 100644 --- a/git/test/test_util.py +++ b/git/test/test_util.py @@ -24,7 +24,7 @@ parse_date, ) from git.cmd import dashify -from git.compat import string_types +from git.compat import string_types, is_win import time @@ -90,7 +90,11 @@ def test_blocking_lock_file(self): wait_lock = BlockingLockFile(my_file, 0.05, wait_time) self.failUnlessRaises(IOError, wait_lock._obtain_lock) elapsed = time.time() - start - assert elapsed <= wait_time + 0.02 # some extra time it may cost + extra_time = 0.02 + if is_win: + # for Appveyor + extra_time *= 6 # NOTE: Indeterministic failures here... + self.assertLess(elapsed, wait_time + extra_time) def test_user_id(self): assert '@' in get_user_id() diff --git a/git/util.py b/git/util.py index f5c692315..814cd7f46 100644 --- a/git/util.py +++ b/git/util.py @@ -3,30 +3,18 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +from __future__ import unicode_literals +import getpass +import logging import os +import platform import re -import sys -import time -import stat import shutil -import platform -import getpass -import threading -import logging - -# NOTE: Some of the unused imports might be used/imported by others. -# Handle once test-cases are back up and running. -from .exc import InvalidGitRepositoryError - -from .compat import ( - MAXSIZE, - defenc, - PY3 -) +import stat +import time -# Most of these are unused here, but are for use by git-python modules so these -# don't see gitdb all the time. Flake of course doesn't like it. +from git.compat import is_win from gitdb.util import ( # NOQA make_sha, LockedFD, @@ -36,10 +24,24 @@ to_bin_sha ) +import os.path as osp + +from .compat import ( + MAXSIZE, + defenc, + PY3 +) +from .exc import InvalidGitRepositoryError + + +# NOTE: Some of the unused imports might be used/imported by others. +# Handle once test-cases are back up and running. +# Most of these are unused here, but are for use by git-python modules so these +# don't see gitdb all the time. Flake of course doesn't like it. __all__ = ("stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux", "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList", "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists', - 'RemoteProgress', 'CallableRemoteProgress', 'rmtree', 'WaitGroup', 'unbare_repo') + 'RemoteProgress', 'CallableRemoteProgress', 'rmtree', 'unbare_repo') #{ Utility Methods @@ -63,17 +65,23 @@ def rmtree(path): :note: we use shutil rmtree but adjust its behaviour to see whether files that couldn't be deleted are read-only. Windows will not remove them in that case""" + def onerror(func, path, exc_info): - if not os.access(path, os.W_OK): - # Is the error an access error ? - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise - # END end onerror + # Is the error an access error ? + os.chmod(path, stat.S_IWUSR) + func(path) # Will scream if still not possible to delete. + return shutil.rmtree(path, False, onerror) +def rmfile(path): + """Ensure file deleted also on *Windows* where read-only files need special treatment.""" + if osp.isfile(path): + if is_win: + os.chmod(path, 0o777) + os.remove(path) + + def stream_copy(source, destination, chunk_size=512 * 1024): """Copy all data from the source stream into the destination stream in chunks of size chunk_size @@ -107,7 +115,7 @@ def join_path(a, *p): return path -if sys.platform.startswith('win'): +if is_win: def to_native_path_windows(path): return path.replace('/', '\\') @@ -152,6 +160,7 @@ def get_user_id(): def finalize_process(proc, **kwargs): """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly""" + ## TODO: No close proc-streams?? proc.wait(**kwargs) #} END utilities @@ -324,12 +333,12 @@ def update(self, op_code, cur_count, max_count=None, message=''): You may read the contents of the current line in self._cur_line""" pass - + class CallableRemoteProgress(RemoteProgress): """An implementation forwarding updates to any callable""" __slots__ = ('_callable') - + def __init__(self, fn): self._callable = fn super(CallableRemoteProgress, self).__init__() @@ -565,7 +574,10 @@ def _obtain_lock_or_raise(self): (self._file_path, lock_file)) try: - fd = os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if is_win: + flags |= os.O_SHORT_LIVED + fd = os.open(lock_file, flags, 0) os.close(fd) except OSError as e: raise IOError(str(e)) @@ -586,12 +598,7 @@ def _release_lock(self): # instead of failing, to make it more usable. lfp = self._lock_file_path() try: - # on bloody windows, the file needs write permissions to be removable. - # Why ... - if os.name == 'nt': - os.chmod(lfp, 0o777) - # END handle win32 - os.remove(lfp) + rmfile(lfp) except OSError: pass self._owns_lock = False @@ -754,35 +761,6 @@ def iter_items(cls, repo, *args, **kwargs): #} END classes -class WaitGroup(object): - """WaitGroup is like Go sync.WaitGroup. - - Without all the useful corner cases. - By Peter Teichman, taken from https://gist.github.com/pteichman/84b92ae7cef0ab98f5a8 - """ - def __init__(self): - self.count = 0 - self.cv = threading.Condition() - - def add(self, n): - self.cv.acquire() - self.count += n - self.cv.release() - - def done(self): - self.cv.acquire() - self.count -= 1 - if self.count == 0: - self.cv.notify_all() - self.cv.release() - - def wait(self, stderr=b''): - self.cv.acquire() - while self.count > 0: - self.cv.wait() - self.cv.release() - - class NullHandler(logging.Handler): def emit(self, record): pass diff --git a/requirements.txt b/requirements.txt index 2316b96ec..85d25511e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ gitdb>=0.6.4 +ddt +mock \ No newline at end of file diff --git a/setup.py b/setup.py index 35b111537..c7dd25fcc 100755 --- a/setup.py +++ b/setup.py @@ -15,9 +15,8 @@ import sys from os import path -v = open(path.join(path.dirname(__file__), 'VERSION')) -VERSION = v.readline().strip() -v.close() +with open(path.join(path.dirname(__file__), 'VERSION')) as v: + VERSION = v.readline().strip() with open('requirements.txt') as reqs_file: requirements = reqs_file.read().splitlines() @@ -50,22 +49,18 @@ def make_release_tree(self, base_dir, files): def _stamp_version(filename): found, out = False, list() try: - f = open(filename, 'r') + with open(filename, 'r') as f: + for line in f: + if '__version__ =' in line: + line = line.replace("'git'", "'%s'" % VERSION) + found = True + out.append(line) except (IOError, OSError): print("Couldn't find file %s to stamp version" % filename, file=sys.stderr) - return - # END handle error, usually happens during binary builds - for line in f: - if '__version__ =' in line: - line = line.replace("'git'", "'%s'" % VERSION) - found = True - out.append(line) - f.close() if found: - f = open(filename, 'w') - f.writelines(out) - f.close() + with open(filename, 'w') as f: + f.writelines(out) else: print("WARNING: Couldn't find version line in file %s" % filename, file=sys.stderr) @@ -73,6 +68,9 @@ def _stamp_version(filename): extras_require = { ':python_version == "2.6"': ['ordereddict'], } +test_requires = ['ddt'] +if sys.version_info[:2] < (2, 7): + test_requires.append('mock') try: if 'bdist_wheel' not in sys.argv: @@ -104,10 +102,9 @@ def _stamp_version(filename): license="BSD License", requires=['gitdb (>=0.6.4)'], install_requires=install_requires, - test_requirements=['mock', 'nose'] + install_requires, + test_requirements=test_requires + install_requires, zip_safe=False, - long_description="""\ -GitPython is a python library used to interact with Git repositories""", + long_description="""GitPython is a python library used to interact with Git repositories""", classifiers=[ # Picked from # http://pypi.python.org/pypi?:action=list_classifiers